> ## Documentation Index
> Fetch the complete documentation index at: https://docs.archil.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Logging

> Using Archil to send application logs directly to S3 without code changes

Configure your applications with robust logging using Archil disks that automatically flow to Amazon S3 storage. Simply mount an Archil disk and point your regular application loggers to it - no S3 APIs required.

<CardGroup cols={2}>
  <Card title="Transparent S3 Integration" icon="cloud-arrow-up">
    Write to normal filesystem paths and logs automatically sync to S3
  </Card>

  <Card title="Instance Isolation" icon="folder-tree">
    Each instance creates its own directory for proper ownership separation
  </Card>

  <Card title="Infinite Capacity" icon="infinity">
    Never run out of capacity because your logs are going to S3
  </Card>

  <Card title="Standard Logging Libraries" icon="code">
    Use your existing logging setup - Python logging, winston, etc.
  </Card>
</CardGroup>

## Create and mount an Archil disk

First, create an Archil disk for your logging infrastructure. Follow the [Quickstart Guide](/getting-started/quickstart) to create a disk, then mount it in shared mode to allow multiple instances to write logs simultaneously.

### Mount the logging disk

```bash theme={null}
# Create the mount directory
sudo mkdir -p /mnt/logs

# Mount the Archil disk in shared mode
sudo archil mount <disk-name> /mnt/logs --region aws-us-east-1 --shared
```

### Create instance-specific directories

Each application instance should create its own directory within the mounted filesystem. We recommend doing this as part of a startup script or on application startup:

```bash theme={null}
# Create instance-specific log directory using hostname
INSTANCE_DIR="/mnt/logs/$(hostname)"
mkdir -p "$INSTANCE_DIR"

# Or use a custom instance identifier
INSTANCE_ID="${INSTANCE_ID:-$(date +%s)-$$}"
INSTANCE_DIR="/mnt/logs/instance-${INSTANCE_ID}"
mkdir -p "$INSTANCE_DIR"
```

## Application configuration

Configure your application loggers to write to the mounted Archil filesystem. Your logs will automatically flow to S3 without any S3 API calls.

<Tabs>
  <Tab title="Python">
    ```python theme={null}
    import logging
    import os
    import socket
    import threading
    import time
    from datetime import datetime

    class MinuteRotatingFileHandler(logging.FileHandler):
        def __init__(self, log_dir, service_name):
            self.log_dir = log_dir
            self.service_name = service_name
            self.current_minute = None
            super().__init__(self._get_current_filename())
            
        def _get_current_filename(self):
            now = datetime.utcnow()
            timestamp = now.strftime('%Y%m%dT%H_%M_00Z')
            return f"{self.log_dir}/{self.service_name}-{timestamp}.log"
            
        def emit(self, record):
            now = datetime.utcnow()
            current_minute = now.strftime('%Y%m%dT%H_%M')
            
            if self.current_minute != current_minute:
                self.close()
                self.baseFilename = self._get_current_filename()
                self.stream = self._open()
                self.current_minute = current_minute
                
            super().emit(record)

    def setup_logging():
        # Create instance-specific directory
        instance_id = socket.gethostname()
        log_dir = f"/mnt/logs/{instance_id}"
        os.makedirs(log_dir, exist_ok=True)
        
        # Configure standard Python logging
        logger = logging.getLogger('app')
        logger.setLevel(logging.INFO)
        
        # Create minute-rotating file handler
        file_handler = MinuteRotatingFileHandler(log_dir, 'app')
        file_handler.setFormatter(
            logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        )
        
        logger.addHandler(file_handler)
        return logger

    # Usage
    logger = setup_logging()
    logger.info("Application started - logs automatically flowing to S3")
    logger.error("Sample error message")
    ```
  </Tab>

  <Tab title="Node.js">
    ```javascript theme={null}
    const winston = require('winston');
    const fs = require('fs');
    const os = require('os');
    const path = require('path');

    class MinuteRotatingTransport extends winston.transports.File {
      constructor(options) {
        super(options);
        this.logDir = options.logDir;
        this.serviceName = options.serviceName;
        this.currentMinute = null;
        this.updateFilename();
        
        // Check for rotation every 10 seconds
        setInterval(() => {
          this.checkRotation();
        }, 10000);
      }
      
      updateFilename() {
        const now = new Date();
        const timestamp = now.toISOString()
          .replace(/[-:]/g, '')
          .replace(/\.\d{3}Z$/, '_00Z')
          .replace(/T(\d{2})(\d{2})/, 'T$1_$2');
        this.filename = path.join(this.logDir, `${this.serviceName}-${timestamp}.log`);
        this.currentMinute = timestamp.slice(0, -4); // Remove '_00Z' suffix
      }
      
      checkRotation() {
        const now = new Date();
        const currentMinute = now.toISOString()
          .replace(/[-:]/g, '')
          .replace(/\.\d{3}Z$/, '')
          .replace(/T(\d{2})(\d{2})/, 'T$1_$2')
          .slice(0, -2); // Remove seconds
          
        if (this.currentMinute !== currentMinute) {
          this.close();
          this.updateFilename();
          this.open();
        }
      }
    }

    function setupLogging() {
      // Create instance-specific directory
      const instanceId = os.hostname();
      const logDir = `/mnt/logs/${instanceId}`;
      
      if (!fs.existsSync(logDir)) {
        fs.mkdirSync(logDir, { recursive: true });
      }
      
      // Configure Winston logger with minute rotation
      const logger = winston.createLogger({
        level: 'info',
        format: winston.format.combine(
          winston.format.timestamp(),
          winston.format.printf(({ timestamp, level, message }) => {
            return `${timestamp} - ${level}: ${message}`;
          })
        ),
        transports: [
          new winston.transports.Console(),
          new MinuteRotatingTransport({ 
            logDir: logDir,
            serviceName: 'app'
          })
        ]
      });
      
      return logger;
    }

    // Usage
    const logger = setupLogging();
    logger.info('Application started - logs automatically flowing to S3');
    logger.error('Sample error message');
    ```
  </Tab>

  <Tab title="Go">
    ```go theme={null}
    package main

    import (
        "fmt"
        "log"
        "os"
        "path/filepath"
        "sync"
        "time"
    )

    type MinuteRotatingLogger struct {
        logDir      string
        serviceName string
        currentFile *os.File
        currentMinute string
        logger      *log.Logger
        mutex       sync.Mutex
    }

    func NewMinuteRotatingLogger(logDir, serviceName string) *MinuteRotatingLogger {
        mrl := &MinuteRotatingLogger{
            logDir:      logDir,
            serviceName: serviceName,
        }
        mrl.rotateIfNeeded()
        
        return mrl
    }

    func (mrl *MinuteRotatingLogger) rotateIfNeeded() {
        mrl.mutex.Lock()
        defer mrl.mutex.Unlock()
        
        now := time.Now().UTC()
        currentMinute := now.Format("20060102T15_04")
        
        if mrl.currentMinute != currentMinute {
            if mrl.currentFile != nil {
                mrl.currentFile.Close()
            }
            
            timestamp := now.Format("20060102T15_04_00Z")
            logFile := filepath.Join(mrl.logDir, fmt.Sprintf("%s-%s.log", mrl.serviceName, timestamp))
            
            file, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
            if err != nil {
                log.Fatalf("Failed to open log file: %v", err)
            }
            
            mrl.currentFile = file
            mrl.currentMinute = currentMinute
            mrl.logger = log.New(file, "", log.LstdFlags|log.Lshortfile)
        }
    }

    func (mrl *MinuteRotatingLogger) Printf(format string, v ...interface{}) {
        mrl.mutex.Lock()
        defer mrl.mutex.Unlock()
        mrl.rotateIfNeeded()
        mrl.logger.Printf(format, v...)
    }

    func (mrl *MinuteRotatingLogger) Println(v ...interface{}) {
        mrl.mutex.Lock()
        defer mrl.mutex.Unlock()
        mrl.rotateIfNeeded()
        mrl.logger.Println(v...)
    }

    func setupLogging() *MinuteRotatingLogger {
        // Create instance-specific directory
        hostname, _ := os.Hostname()
        logDir := filepath.Join("/mnt/logs", hostname)
        os.MkdirAll(logDir, 0755)
        
        return NewMinuteRotatingLogger(logDir, "app")
    }

    func main() {
        logger := setupLogging()
        logger.Println("Application started - logs automatically flowing to S3")
        logger.Println("Sample log message")
    }
    ```
  </Tab>

  <Tab title="Java">
    ```java theme={null}
    import java.io.IOException;
    import java.net.InetAddress;
    import java.nio.file.Files;
    import java.nio.file.Path;
    import java.nio.file.Paths;
    import java.time.LocalDateTime;
    import java.time.ZoneOffset;
    import java.time.format.DateTimeFormatter;
    import java.util.concurrent.Executors;
    import java.util.concurrent.ScheduledExecutorService;
    import java.util.concurrent.TimeUnit;
    import java.util.logging.FileHandler;
    import java.util.logging.Handler;
    import java.util.logging.Logger;
    import java.util.logging.LogRecord;
    import java.util.logging.SimpleFormatter;

    class MinuteRotatingFileHandler extends Handler {
        private final Path logDir;
        private final String serviceName;
        private FileHandler currentHandler;
        private String currentMinute;
        private final ScheduledExecutorService scheduler;

        public MinuteRotatingFileHandler(Path logDir, String serviceName) throws IOException {
            this.logDir = logDir;
            this.serviceName = serviceName;
            this.scheduler = Executors.newScheduledThreadPool(1);
            
            rotateIfNeeded();
            
            // Check for rotation every 10 seconds
            scheduler.scheduleAtFixedRate(this::rotateIfNeeded, 10, 10, TimeUnit.SECONDS);
        }

        private synchronized void rotateIfNeeded() {
            try {
                LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC);
                String minute = now.format(DateTimeFormatter.ofPattern("yyyyMMdd'T'HH_mm"));
                
                if (!minute.equals(currentMinute)) {
                    if (currentHandler != null) {
                        currentHandler.close();
                    }
                    
                    String timestamp = now.format(DateTimeFormatter.ofPattern("yyyyMMdd'T'HH_mm_00'Z'"));
                    String logFile = logDir.resolve(serviceName + "-" + timestamp + ".log").toString();
                    
                    currentHandler = new FileHandler(logFile, true);
                    currentHandler.setFormatter(new SimpleFormatter());
                    currentMinute = minute;
                }
            } catch (IOException e) {
                System.err.println("Failed to rotate log file: " + e.getMessage());
            }
        }

        @Override
        public synchronized void publish(LogRecord record) {
            rotateIfNeeded();
            if (currentHandler != null) {
                currentHandler.publish(record);
            }
        }

        @Override
        public void flush() {
            if (currentHandler != null) {
                currentHandler.flush();
            }
        }

        @Override
        public void close() throws SecurityException {
            if (currentHandler != null) {
                currentHandler.close();
            }
            scheduler.shutdown();
        }
    }

    public class LoggingSetup {
        public static Logger setupLogging() throws IOException {
            // Create instance-specific directory
            String hostname = InetAddress.getLocalHost().getHostName();
            Path logDir = Paths.get("/mnt/logs", hostname);
            Files.createDirectories(logDir);
            
            // Configure Java logging with minute rotation
            Logger logger = Logger.getLogger("AppLogger");
            MinuteRotatingFileHandler fileHandler = new MinuteRotatingFileHandler(logDir, "app");
            logger.addHandler(fileHandler);
            
            return logger;
        }
        
        public static void main(String[] args) throws IOException {
            Logger logger = setupLogging();
            logger.info("Application started - logs automatically flowing to S3");
            logger.severe("Sample error message");
        }
    }
    ```
  </Tab>
</Tabs>

## Log organization and management

With Archil, you don't need traditional log rotation because logs are automatically offloaded to S3, where there's infinite capacity. The standardized logging approach implements automatic minute-based rotation across all languages, creating new log files every minute with the consistent naming pattern `service_YYYYMMDDTHH_MM_00Z.log` for natural chronological organization while maintaining optimal query performance.

### S3 lifecycle management

One of the key advantages of using Archil for logging is that your logs automatically flow to S3, giving you access to all of S3's built-in cost optimization features. You can leverage S3's intelligent tiering and lifecycle policies to automatically move your log data to cheaper storage classes over time without any application changes.

Configure lifecycle policies on your S3 bucket to automatically transition logs to cheaper storage classes:

```json theme={null}
{
  "Rules": [
    {
      "ID": "LogRetentionPolicy",
      "Status": "Enabled",
      "Filter": {
        "Prefix": "logs/"
      },
      "Transitions": [
        {
          "Days": 30,
          "StorageClass": "STANDARD_IA"
        },
        {
          "Days": 90,
          "StorageClass": "GLACIER"
        }
      ],
      "Expiration": {
        "Days": 365
      }
    }
  ]
}
```

## Querying logs with Amazon Athena

Once your logs are flowing to S3 through Archil, you can use Amazon Athena to run SQL queries directly against your log data without needing to load it into a database.

### Setting up Athena for log analysis

Create a table in Athena that leverages the standardized filename format for efficient time-based partitioning. The standardized `service_YYYYMMDDTHH_MM_00Z.log` format enables Athena to automatically partition data by time:

```sql theme={null}
CREATE EXTERNAL TABLE application_logs (
  timestamp string,
  level string,
  message string,
  module string,
  function string,
  line int
)
PARTITIONED BY (
  year string,
  month string,
  day string,
  hour string,
  minute string,
  instance_id string
)
STORED AS TEXTFILE
LOCATION 's3://your-bucket/logs/'
TBLPROPERTIES (
  'projection.enabled' = 'true',
  'projection.year.type' = 'integer',
  'projection.year.range' = '2020,2030',
  'projection.year.interval' = '1',
  'projection.month.type' = 'integer', 
  'projection.month.range' = '1,12',
  'projection.month.interval' = '1',
  'projection.day.type' = 'integer',
  'projection.day.range' = '1,31', 
  'projection.day.interval' = '1',
  'projection.hour.type' = 'integer',
  'projection.hour.range' = '0,23',
  'projection.hour.interval' = '1',
  'projection.minute.type' = 'integer',
  'projection.minute.range' = '0,59',
  'projection.minute.interval' = '1',
  'projection.instance_id.type' = 'injected',
  'storage.location.template' = 's3://your-bucket/logs/${instance_id}/app-${year}${month}${day}T${hour}_${minute}_00Z.log'
)
```

This time-based partitioning approach provides several advantages:

* **Efficient time-range queries**: Athena can skip entire partitions when filtering by time
* **Automatic partition discovery**: No need to manually add partitions as new log files are created
* **Cost optimization**: Only scan the data you need for time-based analysis
* **Scalability**: Performance remains consistent as log volume grows

For simpler use cases where you don't need fine-grained time partitioning, you can use a non-partitioned table:

```sql theme={null}
-- Non-partitioned approach (simpler setup, less efficient for large datasets)
CREATE EXTERNAL TABLE application_logs_simple (
  timestamp string,
  level string,
  message string,
  instance_id string,
  module string,
  function string,
  line int
)
STORED AS TEXTFILE
LOCATION 's3://your-bucket/logs/'
```

### Example Athena queries

<Tabs>
  <Tab title="Error Analysis">
    ```sql theme={null}
    -- Find all errors in the last 24 hours (leverages time partitioning)
    SELECT timestamp, instance_id, message
    FROM application_logs
    WHERE level = 'ERROR'
      AND year = '2024' AND month = '07' AND day = '29'
      AND timestamp >= date_format(date_add('hour', -24, now()), '%Y-%m-%d %H:%i:%s')
    ORDER BY timestamp DESC
    LIMIT 100;

    -- Count errors by instance (with partition pruning)
    SELECT instance_id, COUNT(*) as error_count
    FROM application_logs
    WHERE level = 'ERROR'
      AND year = '2024' AND month = '07' AND day >= '28'
      AND timestamp >= date_format(date_add('day', -1, now()), '%Y-%m-%d')
    GROUP BY instance_id
    ORDER BY error_count DESC;
    ```
  </Tab>

  <Tab title="Performance Monitoring">
    ```sql theme={null}
    -- Analyze log volume by hour
    SELECT 
      date_format(from_iso8601_timestamp(timestamp), '%Y-%m-%d %H:00:00') as hour,
      COUNT(*) as log_count,
      COUNT(DISTINCT instance_id) as active_instances
    FROM application_logs
    WHERE timestamp >= date_format(date_add('day', -7, now()), '%Y-%m-%d')
    GROUP BY date_format(from_iso8601_timestamp(timestamp), '%Y-%m-%d %H:00:00')
    ORDER BY hour;

    -- Find instances with unusual activity
    SELECT 
      instance_id,
      COUNT(*) as total_logs,
      COUNT(CASE WHEN level = 'ERROR' THEN 1 END) as error_logs,
      COUNT(CASE WHEN level = 'WARN' THEN 1 END) as warn_logs
    FROM application_logs
    WHERE timestamp >= date_format(date_add('hour', -1, now()), '%Y-%m-%d %H:%i:%s')
    GROUP BY instance_id
    HAVING COUNT(*) > 1000 OR COUNT(CASE WHEN level = 'ERROR' THEN 1 END) > 10
    ORDER BY total_logs DESC;
    ```
  </Tab>

  <Tab title="Application Insights">
    ```sql theme={null}
    -- Search for specific patterns in log messages
    SELECT timestamp, instance_id, level, message
    FROM application_logs
    WHERE message LIKE '%timeout%'
      OR message LIKE '%connection%'
      OR message LIKE '%failed%'
    AND timestamp >= date_format(date_add('hour', -6, now()), '%Y-%m-%d %H:%i:%s')
    ORDER BY timestamp DESC;

    -- Analyze function call patterns
    SELECT 
      module,
      function,
      COUNT(*) as call_count,
      COUNT(CASE WHEN level = 'ERROR' THEN 1 END) as error_count
    FROM application_logs
    WHERE timestamp >= date_format(date_add('day', -1, now()), '%Y-%m-%d')
    GROUP BY module, function
    HAVING COUNT(*) > 100
    ORDER BY call_count DESC;
    ```
  </Tab>
</Tabs>