Logging

Configure your applications with robust logging using Archil disks that automatically flow to Amazon S3 storage. Simply mount an Archil disk and point your regular application loggers to it - no S3 APIs required.

Transparent S3 Integration

Write to normal filesystem paths and logs automatically sync to S3

Instance Isolation

Each instance creates its own directory for proper ownership separation

Infinite Capacity

Never run out of capacity because your logs are going to S3

Standard Logging Libraries

Use your existing logging setup - Python logging, winston, etc.

Create and mount an Archil disk

First, create an Archil disk for your logging infrastructure. Follow the Quickstart Guide to create a disk, then mount it in shared mode to allow multiple instances to write logs simultaneously.

Mount the logging disk

# Create the mount directory
sudo mkdir -p /mnt/logs

# Mount the Archil disk in shared mode
sudo archil mount <disk-name> /mnt/logs --region aws-us-east-1 --shared

Create instance-specific directories

Each application instance should create its own directory within the mounted filesystem. We recommend doing this as part of a startup script or on application startup:

# Create instance-specific log directory using hostname
INSTANCE_DIR="/mnt/logs/$(hostname)"
mkdir -p "$INSTANCE_DIR"

# Or use a custom instance identifier
INSTANCE_ID="${INSTANCE_ID:-$(date +%s)-$$}"
INSTANCE_DIR="/mnt/logs/instance-${INSTANCE_ID}"
mkdir -p "$INSTANCE_DIR"

Application configuration

Configure your application loggers to write to the mounted Archil filesystem. Your logs will automatically flow to S3 without any S3 API calls.

Python
Node.js
Go
Java

import logging
import os
import socket
import threading
import time
from datetime import datetime

class MinuteRotatingFileHandler(logging.FileHandler):
    def __init__(self, log_dir, service_name):
        self.log_dir = log_dir
        self.service_name = service_name
        self.current_minute = None
        super().__init__(self._get_current_filename())
        
    def _get_current_filename(self):
        now = datetime.utcnow()
        timestamp = now.strftime('%Y%m%dT%H_%M_00Z')
        return f"{self.log_dir}/{self.service_name}-{timestamp}.log"
        
    def emit(self, record):
        now = datetime.utcnow()
        current_minute = now.strftime('%Y%m%dT%H_%M')
        
        if self.current_minute != current_minute:
            self.close()
            self.baseFilename = self._get_current_filename()
            self.stream = self._open()
            self.current_minute = current_minute
            
        super().emit(record)

def setup_logging():
    # Create instance-specific directory
    instance_id = socket.gethostname()
    log_dir = f"/mnt/logs/{instance_id}"
    os.makedirs(log_dir, exist_ok=True)
    
    # Configure standard Python logging
    logger = logging.getLogger('app')
    logger.setLevel(logging.INFO)
    
    # Create minute-rotating file handler
    file_handler = MinuteRotatingFileHandler(log_dir, 'app')
    file_handler.setFormatter(
        logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    )
    
    logger.addHandler(file_handler)
    return logger

# Usage
logger = setup_logging()
logger.info("Application started - logs automatically flowing to S3")
logger.error("Sample error message")

const winston = require('winston');
const fs = require('fs');
const os = require('os');
const path = require('path');

class MinuteRotatingTransport extends winston.transports.File {
  constructor(options) {
    super(options);
    this.logDir = options.logDir;
    this.serviceName = options.serviceName;
    this.currentMinute = null;
    this.updateFilename();
    
    // Check for rotation every 10 seconds
    setInterval(() => {
      this.checkRotation();
    }, 10000);
  }
  
  updateFilename() {
    const now = new Date();
    const timestamp = now.toISOString()
      .replace(/[-:]/g, '')
      .replace(/\.\d{3}Z$/, '_00Z')
      .replace(/T(\d{2})(\d{2})/, 'T$1_$2');
    this.filename = path.join(this.logDir, `${this.serviceName}-${timestamp}.log`);
    this.currentMinute = timestamp.slice(0, -4); // Remove '_00Z' suffix
  }
  
  checkRotation() {
    const now = new Date();
    const currentMinute = now.toISOString()
      .replace(/[-:]/g, '')
      .replace(/\.\d{3}Z$/, '')
      .replace(/T(\d{2})(\d{2})/, 'T$1_$2')
      .slice(0, -2); // Remove seconds
      
    if (this.currentMinute !== currentMinute) {
      this.close();
      this.updateFilename();
      this.open();
    }
  }
}

function setupLogging() {
  // Create instance-specific directory
  const instanceId = os.hostname();
  const logDir = `/mnt/logs/${instanceId}`;
  
  if (!fs.existsSync(logDir)) {
    fs.mkdirSync(logDir, { recursive: true });
  }
  
  // Configure Winston logger with minute rotation
  const logger = winston.createLogger({
    level: 'info',
    format: winston.format.combine(
      winston.format.timestamp(),
      winston.format.printf(({ timestamp, level, message }) => {
        return `${timestamp} - ${level}: ${message}`;
      })
    ),
    transports: [
      new winston.transports.Console(),
      new MinuteRotatingTransport({ 
        logDir: logDir,
        serviceName: 'app'
      })
    ]
  });
  
  return logger;
}

// Usage
const logger = setupLogging();
logger.info('Application started - logs automatically flowing to S3');
logger.error('Sample error message');

package main

import (
    "fmt"
    "log"
    "os"
    "path/filepath"
    "sync"
    "time"
)

type MinuteRotatingLogger struct {
    logDir      string
    serviceName string
    currentFile *os.File
    currentMinute string
    logger      *log.Logger
    mutex       sync.Mutex
}

func NewMinuteRotatingLogger(logDir, serviceName string) *MinuteRotatingLogger {
    mrl := &MinuteRotatingLogger{
        logDir:      logDir,
        serviceName: serviceName,
    }
    mrl.rotateIfNeeded()
    
    return mrl
}

func (mrl *MinuteRotatingLogger) rotateIfNeeded() {
    mrl.mutex.Lock()
    defer mrl.mutex.Unlock()
    
    now := time.Now().UTC()
    currentMinute := now.Format("20060102T15_04")
    
    if mrl.currentMinute != currentMinute {
        if mrl.currentFile != nil {
            mrl.currentFile.Close()
        }
        
        timestamp := now.Format("20060102T15_04_00Z")
        logFile := filepath.Join(mrl.logDir, fmt.Sprintf("%s-%s.log", mrl.serviceName, timestamp))
        
        file, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
        if err != nil {
            log.Fatalf("Failed to open log file: %v", err)
        }
        
        mrl.currentFile = file
        mrl.currentMinute = currentMinute
        mrl.logger = log.New(file, "", log.LstdFlags|log.Lshortfile)
    }
}

func (mrl *MinuteRotatingLogger) Printf(format string, v ...interface{}) {
    mrl.mutex.Lock()
    defer mrl.mutex.Unlock()
    mrl.rotateIfNeeded()
    mrl.logger.Printf(format, v...)
}

func (mrl *MinuteRotatingLogger) Println(v ...interface{}) {
    mrl.mutex.Lock()
    defer mrl.mutex.Unlock()
    mrl.rotateIfNeeded()
    mrl.logger.Println(v...)
}

func setupLogging() *MinuteRotatingLogger {
    // Create instance-specific directory
    hostname, _ := os.Hostname()
    logDir := filepath.Join("/mnt/logs", hostname)
    os.MkdirAll(logDir, 0755)
    
    return NewMinuteRotatingLogger(logDir, "app")
}

func main() {
    logger := setupLogging()
    logger.Println("Application started - logs automatically flowing to S3")
    logger.Println("Sample log message")
}

import java.io.IOException;
import java.net.InetAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.logging.FileHandler;
import java.util.logging.Handler;
import java.util.logging.Logger;
import java.util.logging.LogRecord;
import java.util.logging.SimpleFormatter;

class MinuteRotatingFileHandler extends Handler {
    private final Path logDir;
    private final String serviceName;
    private FileHandler currentHandler;
    private String currentMinute;
    private final ScheduledExecutorService scheduler;

    public MinuteRotatingFileHandler(Path logDir, String serviceName) throws IOException {
        this.logDir = logDir;
        this.serviceName = serviceName;
        this.scheduler = Executors.newScheduledThreadPool(1);
        
        rotateIfNeeded();
        
        // Check for rotation every 10 seconds
        scheduler.scheduleAtFixedRate(this::rotateIfNeeded, 10, 10, TimeUnit.SECONDS);
    }

    private synchronized void rotateIfNeeded() {
        try {
            LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC);
            String minute = now.format(DateTimeFormatter.ofPattern("yyyyMMdd'T'HH_mm"));
            
            if (!minute.equals(currentMinute)) {
                if (currentHandler != null) {
                    currentHandler.close();
                }
                
                String timestamp = now.format(DateTimeFormatter.ofPattern("yyyyMMdd'T'HH_mm_00'Z'"));
                String logFile = logDir.resolve(serviceName + "-" + timestamp + ".log").toString();
                
                currentHandler = new FileHandler(logFile, true);
                currentHandler.setFormatter(new SimpleFormatter());
                currentMinute = minute;
            }
        } catch (IOException e) {
            System.err.println("Failed to rotate log file: " + e.getMessage());
        }
    }

    @Override
    public synchronized void publish(LogRecord record) {
        rotateIfNeeded();
        if (currentHandler != null) {
            currentHandler.publish(record);
        }
    }

    @Override
    public void flush() {
        if (currentHandler != null) {
            currentHandler.flush();
        }
    }

    @Override
    public void close() throws SecurityException {
        if (currentHandler != null) {
            currentHandler.close();
        }
        scheduler.shutdown();
    }
}

public class LoggingSetup {
    public static Logger setupLogging() throws IOException {
        // Create instance-specific directory
        String hostname = InetAddress.getLocalHost().getHostName();
        Path logDir = Paths.get("/mnt/logs", hostname);
        Files.createDirectories(logDir);
        
        // Configure Java logging with minute rotation
        Logger logger = Logger.getLogger("AppLogger");
        MinuteRotatingFileHandler fileHandler = new MinuteRotatingFileHandler(logDir, "app");
        logger.addHandler(fileHandler);
        
        return logger;
    }
    
    public static void main(String[] args) throws IOException {
        Logger logger = setupLogging();
        logger.info("Application started - logs automatically flowing to S3");
        logger.severe("Sample error message");
    }
}

Log organization and management

With Archil, you don’t need traditional log rotation because logs are automatically offloaded to S3, where there’s infinite capacity. The standardized logging approach implements automatic minute-based rotation across all languages, creating new log files every minute with the consistent naming pattern service_YYYYMMDDTHH_MM_00Z.log for natural chronological organization while maintaining optimal query performance.

S3 lifecycle management

One of the key advantages of using Archil for logging is that your logs automatically flow to S3, giving you access to all of S3’s built-in cost optimization features. You can leverage S3’s intelligent tiering and lifecycle policies to automatically move your log data to cheaper storage classes over time without any application changes. Configure lifecycle policies on your S3 bucket to automatically transition logs to cheaper storage classes:

{
  "Rules": [
    {
      "ID": "LogRetentionPolicy",
      "Status": "Enabled",
      "Filter": {
        "Prefix": "logs/"
      },
      "Transitions": [
        {
          "Days": 30,
          "StorageClass": "STANDARD_IA"
        },
        {
          "Days": 90,
          "StorageClass": "GLACIER"
        }
      ],
      "Expiration": {
        "Days": 365
      }
    }
  ]
}

Querying logs with Amazon Athena

Once your logs are flowing to S3 through Archil, you can use Amazon Athena to run SQL queries directly against your log data without needing to load it into a database.

Setting up Athena for log analysis

Create a table in Athena that leverages the standardized filename format for efficient time-based partitioning. The standardized service_YYYYMMDDTHH_MM_00Z.log format enables Athena to automatically partition data by time:

CREATE EXTERNAL TABLE application_logs (
  timestamp string,
  level string,
  message string,
  module string,
  function string,
  line int
)
PARTITIONED BY (
  year string,
  month string,
  day string,
  hour string,
  minute string,
  instance_id string
)
STORED AS TEXTFILE
LOCATION 's3://your-bucket/logs/'
TBLPROPERTIES (
  'projection.enabled' = 'true',
  'projection.year.type' = 'integer',
  'projection.year.range' = '2020,2030',
  'projection.year.interval' = '1',
  'projection.month.type' = 'integer', 
  'projection.month.range' = '1,12',
  'projection.month.interval' = '1',
  'projection.day.type' = 'integer',
  'projection.day.range' = '1,31', 
  'projection.day.interval' = '1',
  'projection.hour.type' = 'integer',
  'projection.hour.range' = '0,23',
  'projection.hour.interval' = '1',
  'projection.minute.type' = 'integer',
  'projection.minute.range' = '0,59',
  'projection.minute.interval' = '1',
  'projection.instance_id.type' = 'injected',
  'storage.location.template' = 's3://your-bucket/logs/${instance_id}/app-${year}${month}${day}T${hour}_${minute}_00Z.log'
)

This time-based partitioning approach provides several advantages:

Efficient time-range queries: Athena can skip entire partitions when filtering by time
Automatic partition discovery: No need to manually add partitions as new log files are created
Cost optimization: Only scan the data you need for time-based analysis
Scalability: Performance remains consistent as log volume grows

For simpler use cases where you don’t need fine-grained time partitioning, you can use a non-partitioned table:

-- Non-partitioned approach (simpler setup, less efficient for large datasets)
CREATE EXTERNAL TABLE application_logs_simple (
  timestamp string,
  level string,
  message string,
  instance_id string,
  module string,
  function string,
  line int
)
STORED AS TEXTFILE
LOCATION 's3://your-bucket/logs/'

Example Athena queries

Error Analysis
Performance Monitoring
Application Insights

-- Find all errors in the last 24 hours (leverages time partitioning)
SELECT timestamp, instance_id, message
FROM application_logs
WHERE level = 'ERROR'
  AND year = '2024' AND month = '07' AND day = '29'
  AND timestamp >= date_format(date_add('hour', -24, now()), '%Y-%m-%d %H:%i:%s')
ORDER BY timestamp DESC
LIMIT 100;

-- Count errors by instance (with partition pruning)
SELECT instance_id, COUNT(*) as error_count
FROM application_logs
WHERE level = 'ERROR'
  AND year = '2024' AND month = '07' AND day >= '28'
  AND timestamp >= date_format(date_add('day', -1, now()), '%Y-%m-%d')
GROUP BY instance_id
ORDER BY error_count DESC;

-- Analyze log volume by hour
SELECT 
  date_format(from_iso8601_timestamp(timestamp), '%Y-%m-%d %H:00:00') as hour,
  COUNT(*) as log_count,
  COUNT(DISTINCT instance_id) as active_instances
FROM application_logs
WHERE timestamp >= date_format(date_add('day', -7, now()), '%Y-%m-%d')
GROUP BY date_format(from_iso8601_timestamp(timestamp), '%Y-%m-%d %H:00:00')
ORDER BY hour;

-- Find instances with unusual activity
SELECT 
  instance_id,
  COUNT(*) as total_logs,
  COUNT(CASE WHEN level = 'ERROR' THEN 1 END) as error_logs,
  COUNT(CASE WHEN level = 'WARN' THEN 1 END) as warn_logs
FROM application_logs
WHERE timestamp >= date_format(date_add('hour', -1, now()), '%Y-%m-%d %H:%i:%s')
GROUP BY instance_id
HAVING COUNT(*) > 1000 OR COUNT(CASE WHEN level = 'ERROR' THEN 1 END) > 10
ORDER BY total_logs DESC;

-- Search for specific patterns in log messages
SELECT timestamp, instance_id, level, message
FROM application_logs
WHERE message LIKE '%timeout%'
  OR message LIKE '%connection%'
  OR message LIKE '%failed%'
AND timestamp >= date_format(date_add('hour', -6, now()), '%Y-%m-%d %H:%i:%s')
ORDER BY timestamp DESC;

-- Analyze function call patterns
SELECT 
  module,
  function,
  COUNT(*) as call_count,
  COUNT(CASE WHEN level = 'ERROR' THEN 1 END) as error_count
FROM application_logs
WHERE timestamp >= date_format(date_add('day', -1, now()), '%Y-%m-%d')
GROUP BY module, function
HAVING COUNT(*) > 100
ORDER BY call_count DESC;

Compute

AI

Data Science

Databases

Observability

Virtualization

Transparent S3 Integration

Instance Isolation

Infinite Capacity

Standard Logging Libraries

Create and mount an Archil disk

Mount the logging disk

Create instance-specific directories

Application configuration

Log organization and management

S3 lifecycle management

Querying logs with Amazon Athena

Setting up Athena for log analysis

Example Athena queries

Transparent S3 Integration

Instance Isolation

Infinite Capacity

Standard Logging Libraries

​Create and mount an Archil disk

​Mount the logging disk

​Create instance-specific directories

​Application configuration

​Log organization and management

​S3 lifecycle management

​Querying logs with Amazon Athena

​Setting up Athena for log analysis

​Example Athena queries

Create and mount an Archil disk

Mount the logging disk

Create instance-specific directories

Application configuration

Log organization and management

S3 lifecycle management

Querying logs with Amazon Athena

Setting up Athena for log analysis

Example Athena queries