Configure logrotate with Elasticsearch and Kibana integration for automated log management

Install required packages

Install logrotate and Elasticsearch tools for log management automation.

sudo apt update
sudo apt install -y logrotate curl jq
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-8.x.list
sudo apt update
sudo apt install -y elasticsearch kibana

sudo dnf update -y
sudo dnf install -y logrotate curl jq
sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
echo '[elasticsearch]
name=Elasticsearch repository for 8.x packages
baseurl=https://artifacts.elastic.co/packages/8.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=0
autorefresh=1
type=rpm-md' | sudo tee /etc/yum.repos.d/elasticsearch.repo
sudo dnf install -y --enablerepo=elasticsearch elasticsearch kibana

Configure Elasticsearch with security

Set up Elasticsearch with authentication and configure cluster settings for log management.

cluster.name: log-management-cluster
node.name: node-1
network.host: 127.0.0.1
http.port: 9200
discovery.type: single-node

# Security settings
xpack.security.enabled: true
xpack.security.enrollment.enabled: true
xpack.security.http.ssl.enabled: false
xpack.security.transport.ssl.enabled: false

# Index lifecycle management
action.destructive_requires_name: true
cluster.routing.allocation.disk.watermark.low: 85%
cluster.routing.allocation.disk.watermark.high: 90%
cluster.routing.allocation.disk.watermark.flood_stage: 95%

Start Elasticsearch and configure passwords

Enable Elasticsearch service and set up authentication credentials.

sudo systemctl enable --now elasticsearch
sudo systemctl status elasticsearch

# Wait for Elasticsearch to start
sleep 30

# Set passwords for built-in users
sudo /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto

Configure Kibana integration

Set up Kibana to connect to Elasticsearch with authentication.

server.port: 5601
server.host: "127.0.0.1"
server.name: log-management-kibana

elasticsearch.hosts: ["http://127.0.0.1:9200"]
elasticsearch.username: "kibana_system"
elasticsearch.password: "YOUR_KIBANA_SYSTEM_PASSWORD"

logging.appenders.file.type: file
logging.appenders.file.fileName: /var/log/kibana/kibana.log
logging.appenders.file.layout.type: json

logging.root.appenders: ["default", "file"]
logging.root.level: info

sudo mkdir -p /var/log/kibana
sudo chown kibana:kibana /var/log/kibana
sudo systemctl enable --now kibana
sudo systemctl status kibana

Create Elasticsearch index lifecycle policy

Configure ILM policy to automatically manage log indices based on age and size.

curl -X PUT "127.0.0.1:9200/_ilm/policy/logs-policy" \
  -H "Content-Type: application/json" \
  -u elastic:YOUR_ELASTIC_PASSWORD \
  -d '{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_size": "5GB",
            "max_age": "7d",
            "max_docs": 1000000
          }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "allocate": {
            "number_of_replicas": 0
          },
          "shrink": {
            "number_of_shards": 1
          }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "allocate": {
            "number_of_replicas": 0
          }
        }
      },
      "delete": {
        "min_age": "90d"
      }
    }
  }
}'

Create index template for automated log ingestion

Set up index template to apply ILM policy to all log indices automatically.

curl -X PUT "127.0.0.1:9200/_index_template/logs-template" \
  -H "Content-Type: application/json" \
  -u elastic:YOUR_ELASTIC_PASSWORD \
  -d '{
  "index_patterns": ["logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 1,
      "number_of_replicas": 0,
      "index.lifecycle.name": "logs-policy",
      "index.lifecycle.rollover_alias": "logs"
    },
    "mappings": {
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "level": {
          "type": "keyword"
        },
        "message": {
          "type": "text"
        },
        "service": {
          "type": "keyword"
        },
        "host": {
          "type": "keyword"
        }
      }
    }
  },
  "priority": 200,
  "version": 1
}'

Configure logrotate for application logs

Set up logrotate to manage application log files and integrate with Elasticsearch indexing.

/var/log/application/*.log {
    daily
    rotate 7
    compress
    delaycompress
    missingok
    notifempty
    create 644 www-data www-data
    sharedscripts
    postrotate
        # Send rotated logs to Elasticsearch
        /usr/local/bin/send-logs-to-elasticsearch.sh "$1"
        # Restart application if needed
        systemctl reload nginx 2>/dev/null || true
    endscript
}

Create log shipping script

Build script to automatically index rotated logs in Elasticsearch with proper formatting.

#!/bin/bash

# Configuration
ELASTIC_HOST="127.0.0.1:9200"
ELASTIC_USER="elastic"
ELASTIC_PASS="YOUR_ELASTIC_PASSWORD"
INDEX_PREFIX="logs"
SERVICE_NAME="application"
HOSTNAME=$(hostname)

# Function to send log entry to Elasticsearch
send_log_entry() {
    local log_file="$1"
    local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ")
    local index_name="${INDEX_PREFIX}-$(date +%Y.%m.%d)"
    
    while IFS= read -r line; do
        if [[ -n "$line" ]]; then
            # Parse log level (INFO, ERROR, WARN, DEBUG)
            local level="INFO"
            if [[ "$line" =~ \[([A-Z]+)\] ]]; then
                level="${BASH_REMATCH[1]}"
            fi
            
            # Create JSON document
            local doc=$(jq -n \
                --arg timestamp "$timestamp" \
                --arg level "$level" \
                --arg message "$line" \
                --arg service "$SERVICE_NAME" \
                --arg host "$HOSTNAME" \
                '{
                    "@timestamp": $timestamp,
                    "level": $level,
                    "message": $message,
                    "service": $service,
                    "host": $host
                }')
            
            # Send to Elasticsearch
            curl -s -X POST "${ELASTIC_HOST}/${index_name}/_doc" \
                -H "Content-Type: application/json" \
                -u "${ELASTIC_USER}:${ELASTIC_PASS}" \
                -d "$doc" > /dev/null
        fi
    done < "$log_file"
}

# Process log file if provided
if [[ -n "$1" && -f "$1" ]]; then
    send_log_entry "$1"
fi

sudo chmod +x /usr/local/bin/send-logs-to-elasticsearch.sh
sudo mkdir -p /var/log/application
sudo chown www-data:www-data /var/log/application

Configure system log rotation

Extend logrotate configuration for system logs with Elasticsearch integration.

/var/log/syslog {
    daily
    rotate 14
    compress
    delaycompress
    missingok
    notifempty
    postrotate
        # Index system logs in Elasticsearch
        /usr/local/bin/index-system-logs.sh
        /bin/kill -HUP `cat /var/run/rsyslogd.pid 2> /dev/null` 2> /dev/null || true
    endscript
}

/var/log/auth.log {
    daily
    rotate 30
    compress
    delaycompress
    missingok
    notifempty
    postrotate
        /usr/local/bin/index-auth-logs.sh
        /bin/kill -HUP `cat /var/run/rsyslogd.pid 2> /dev/null` 2> /dev/null || true
    endscript
}

Create system log indexing scripts

Build specialized scripts for different log types with appropriate field mapping.

#!/bin/bash

ELASTIC_HOST="127.0.0.1:9200"
ELASTIC_USER="elastic"
ELASTIC_PASS="YOUR_ELASTIC_PASSWORD"
LOG_FILE="/var/log/syslog.1"
INDEX_NAME="system-logs-$(date +%Y.%m.%d)"

if [[ -f "$LOG_FILE" ]]; then
    # Process compressed or uncompressed log files
    if [[ "$LOG_FILE" =~ \.gz$ ]]; then
        zcat "$LOG_FILE" | while IFS= read -r line; do
            process_syslog_entry "$line"
        done
    else
        while IFS= read -r line; do
            process_syslog_entry "$line"
        done < "$LOG_FILE"
    fi
fi

process_syslog_entry() {
    local line="$1"
    local timestamp=$(echo "$line" | awk '{print $1" "$2" "$3}')
    local hostname=$(echo "$line" | awk '{print $4}')
    local process=$(echo "$line" | awk '{print $5}' | sed 's/://')
    local message=$(echo "$line" | cut -d' ' -f6-)
    
    local doc=$(jq -n \
        --arg timestamp "$timestamp" \
        --arg hostname "$hostname" \
        --arg process "$process" \
        --arg message "$message" \
        --arg type "system" \
        '{
            "@timestamp": $timestamp,
            "host": $hostname,
            "process": $process,
            "message": $message,
            "log_type": $type
        }')
    
    curl -s -X POST "${ELASTIC_HOST}/${INDEX_NAME}/_doc" \
        -H "Content-Type: application/json" \
        -u "${ELASTIC_USER}:${ELASTIC_PASS}" \
        -d "$doc" > /dev/null
}

#!/bin/bash

ELASTIC_HOST="127.0.0.1:9200"
ELASTIC_USER="elastic"
ELASTIC_PASS="YOUR_ELASTIC_PASSWORD"
LOG_FILE="/var/log/auth.log.1"
INDEX_NAME="security-logs-$(date +%Y.%m.%d)"

if [[ -f "$LOG_FILE" ]]; then
    while IFS= read -r line; do
        # Parse authentication events
        local event_type="unknown"
        if [[ "$line" =~ "Accepted" ]]; then
            event_type="login_success"
        elif [[ "$line" =~ "Failed" ]]; then
            event_type="login_failure"
        elif [[ "$line" =~ "sudo" ]]; then
            event_type="sudo_command"
        fi
        
        local timestamp=$(echo "$line" | awk '{print $1" "$2" "$3}')
        local hostname=$(echo "$line" | awk '{print $4}')
        local message=$(echo "$line" | cut -d' ' -f6-)
        
        local doc=$(jq -n \
            --arg timestamp "$timestamp" \
            --arg hostname "$hostname" \
            --arg message "$message" \
            --arg event_type "$event_type" \
            --arg log_type "security" \
            '{
                "@timestamp": $timestamp,
                "host": $hostname,
                "message": $message,
                "event_type": $event_type,
                "log_type": $log_type
            }')
        
        curl -s -X POST "${ELASTIC_HOST}/${INDEX_NAME}/_doc" \
            -H "Content-Type: application/json" \
            -u "${ELASTIC_USER}:${ELASTIC_PASS}" \
            -d "$doc" > /dev/null
    done < "$LOG_FILE"
fi

sudo chmod +x /usr/local/bin/index-system-logs.sh
sudo chmod +x /usr/local/bin/index-auth-logs.sh

Configure automated log rotation schedule

Set up cron jobs to ensure regular log rotation and Elasticsearch maintenance.

sudo crontab -e

# Run logrotate hourly for high-volume logs
0 * * * * /usr/sbin/logrotate -f /etc/logrotate.d/application-logs

# Run system log rotation daily
0 2 * * * /usr/sbin/logrotate -f /etc/logrotate.d/elasticsearch-system

# Clean up old Elasticsearch indices weekly
0 3 * * 0 /usr/local/bin/cleanup-old-indices.sh

# Monitor disk space and send alerts
*/15 * * * * /usr/local/bin/disk-space-monitor.sh

Create maintenance and monitoring scripts

Build scripts for automatic cleanup and monitoring of the log management system.

#!/bin/bash

ELASTIC_HOST="127.0.0.1:9200"
ELASTIC_USER="elastic"
ELASTIC_PASS="YOUR_ELASTIC_PASSWORD"
RETENTION_DAYS=90

# Get indices older than retention period
old_indices=$(curl -s -u "${ELASTIC_USER}:${ELASTIC_PASS}" \
    "${ELASTIC_HOST}/_cat/indices?h=index&s=index" | \
    grep -E "logs-[0-9]{4}\.[0-9]{2}\.[0-9]{2}" | \
    while read index; do
        index_date=$(echo "$index" | grep -o '[0-9]\{4\}\.[0-9]\{2\}\.[0-9]\{2\}')
        index_timestamp=$(date -d "$index_date" +%s)
        cutoff_timestamp=$(date -d "$RETENTION_DAYS days ago" +%s)
        
        if [[ $index_timestamp -lt $cutoff_timestamp ]]; then
            echo "$index"
        fi
    done)

# Delete old indices
for index in $old_indices; do
    echo "Deleting index: $index"
    curl -X DELETE -s -u "${ELASTIC_USER}:${ELASTIC_PASS}" \
        "${ELASTIC_HOST}/${index}"
done

# Force merge recent indices for better performance
recent_indices=$(curl -s -u "${ELASTIC_USER}:${ELASTIC_PASS}" \
    "${ELASTIC_HOST}/_cat/indices?h=index&s=index" | \
    grep -E "logs-[0-9]{4}\.[0-9]{2}\.[0-9]{2}" | \
    tail -7)

for index in $recent_indices; do
    curl -X POST -s -u "${ELASTIC_USER}:${ELASTIC_PASS}" \
        "${ELASTIC_HOST}/${index}/_forcemerge?max_num_segments=1"
done

#!/bin/bash

THRESHOLD=85
EMAIL="admin@example.com"

# Check disk usage
usage=$(df /var/log | awk 'NR==2 {print $5}' | sed 's/%//')

if [[ $usage -gt $THRESHOLD ]]; then
    message="Warning: Log disk usage is at ${usage}% on $(hostname)"
    echo "$message" | mail -s "High Disk Usage Alert" "$EMAIL"
    
    # Emergency log cleanup
    find /var/log -name "*.log.[0-9]*" -mtime +7 -delete
    find /var/log -name "*.gz" -mtime +14 -delete
fi

# Check Elasticsearch cluster health
health=$(curl -s -u "${ELASTIC_USER}:${ELASTIC_PASS}" \
    "127.0.0.1:9200/_cluster/health" | jq -r '.status')

if [[ "$health" != "green" ]]; then
    echo "Elasticsearch cluster status: $health" | \
        mail -s "Elasticsearch Health Alert" "$EMAIL"
fi

sudo chmod +x /usr/local/bin/cleanup-old-indices.sh
sudo chmod +x /usr/local/bin/disk-space-monitor.sh

Create Kibana dashboards

Access Kibana and create dashboards for log analysis and monitoring visualization.

# Access Kibana at http://127.0.0.1:5601
# Login with elastic user and password

# Create index pattern for logs
curl -X POST "127.0.0.1:5601/api/saved_objects/index-pattern/logs-*" \
  -H "Content-Type: application/json" \
  -H "kbn-xsrf: true" \
  -u elastic:YOUR_ELASTIC_PASSWORD \
  -d '{
    "attributes": {
      "title": "logs-*",
      "timeFieldName": "@timestamp"
    }
  }'

# Create dashboard configuration
curl -X POST "127.0.0.1:5601/api/saved_objects/dashboard/log-management-dashboard" \
  -H "Content-Type: application/json" \
  -H "kbn-xsrf: true" \
  -u elastic:YOUR_ELASTIC_PASSWORD \
  -d '{
    "attributes": {
      "title": "Log Management Dashboard",
      "description": "Automated log rotation and analytics",
      "panelsJSON": "[]",
      "version": 1
    }
  }'

Set up Elasticsearch monitoring

Enable monitoring for cluster health and performance metrics.

# Enable monitoring collection
curl -X PUT "127.0.0.1:9200/_cluster/settings" \
  -H "Content-Type: application/json" \
  -u elastic:YOUR_ELASTIC_PASSWORD \
  -d '{
    "persistent": {
      "xpack.monitoring.collection.enabled": true
    }
  }'

# Create watcher for disk space alerts
curl -X PUT "127.0.0.1:9200/_watcher/watch/disk_space_alert" \
  -H "Content-Type: application/json" \
  -u elastic:YOUR_ELASTIC_PASSWORD \
  -d '{
    "trigger": {
      "schedule": {
        "interval": "5m"
      }
    },
    "input": {
      "http": {
        "request": {
          "host": "127.0.0.1",
          "port": 9200,
          "path": "/_nodes/stats/fs"
        }
      }
    },
    "condition": {
      "script": {
        "source": "ctx.payl