Setup log aggregation with rsyslog and logrotate for centralized system monitoring

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Default values
CENTRAL_SERVER_IP=""
MODE="server"

# Usage function
usage() {
    echo "Usage: $0 [server|client] [central_server_ip]"
    echo "  server: Configure as centralized log server"
    echo "  client: Configure as log forwarding client"
    echo "  central_server_ip: IP address of central server (required for client mode)"
    exit 1
}

# Logging functions
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Cleanup function
cleanup() {
    log_error "Installation failed. Cleaning up..."
    systemctl stop rsyslog 2>/dev/null || true
    exit 1
}

trap cleanup ERR

# Parse arguments
if [ $# -eq 0 ]; then
    usage
fi

MODE="$1"
if [ "$MODE" = "client" ] && [ $# -lt 2 ]; then
    log_error "Central server IP required for client mode"
    usage
fi

if [ "$MODE" = "client" ]; then
    CENTRAL_SERVER_IP="$2"
fi

if [ "$MODE" != "server" ] && [ "$MODE" != "client" ]; then
    log_error "Invalid mode: $MODE"
    usage
fi

# Check if running as root
if [ "$EUID" -ne 0 ]; then
    log_error "Please run as root or with sudo"
    exit 1
fi

echo "[1/8] Detecting distribution and package manager..."

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update && apt upgrade -y"
            FIREWALL_CMD="ufw"
            RSYSLOG_USER="syslog"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            FIREWALL_CMD="firewall-cmd"
            RSYSLOG_USER="syslog"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            FIREWALL_CMD="firewall-cmd"
            RSYSLOG_USER="syslog"
            ;;
        *) 
            log_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    log_error "Cannot detect distribution"
    exit 1
fi

log_info "Detected: $PRETTY_NAME using $PKG_MGR"

echo "[2/8] Updating system packages..."
$PKG_UPDATE

echo "[3/8] Installing rsyslog and logrotate..."
$PKG_INSTALL rsyslog logrotate

echo "[4/8] Configuring rsyslog..."

if [ "$MODE" = "server" ]; then
    # Configure rsyslog server
    cat > /etc/rsyslog.d/10-centralized-server.conf << 'EOF'
# Enable UDP syslog reception
module(load="imudp")
input(type="imudp" port="514" address="0.0.0.0")

# Enable TCP syslog reception
module(load="imtcp")
input(type="imtcp" port="514" address="0.0.0.0")

# Create log directories based on hostname
$template DynFile,"/var/log/centralized/%HOSTNAME%/%PROGRAMNAME%.log"
$template LogFormat,"%TIMESTAMP% %HOSTNAME% %syslogtag% %msg%\n"

# Store remote logs in separate files by hostname
*.* ?DynFile;LogFormat

# Filter by program name
:programname, isequal, "sshd" /var/log/centralized/ssh.log
:programname, isequal, "nginx" /var/log/centralized/nginx.log
:programname, isequal, "mysql" /var/log/centralized/mysql.log

# Filter by facility and severity
kern.crit /var/log/centralized/kernel-critical.log
auth.info /var/log/centralized/auth.log
mail.* /var/log/centralized/mail.log

# Filter messages containing specific strings
:msg, contains, "Failed password" /var/log/centralized/failed-logins.log
:msg, contains, "connection refused" /var/log/centralized/connection-errors.log
:msg, contains, "Out of memory" /var/log/centralized/memory-issues.log

# Discard debug messages to reduce log volume
kern.debug stop
*.debug stop

# Stop processing after writing to dynamic file
& stop
EOF

    # Create log directory structure
    mkdir -p /var/log/centralized
    chown $RSYSLOG_USER:adm /var/log/centralized
    chmod 755 /var/log/centralized

else
    # Configure rsyslog client
    cat > /etc/rsyslog.d/10-centralized-client.conf << EOF
# Forward all logs to central server
*.* @@${CENTRAL_SERVER_IP}:514

# Forward specific facilities with different priorities
auth,authpriv.* @@${CENTRAL_SERVER_IP}:514
mail.* @@${CENTRAL_SERVER_IP}:514
cron.* @@${CENTRAL_SERVER_IP}:514

# Local logging with filtering
local0.info /var/log/application.log
local1.warn /var/log/security.log

# Stop processing forwarded messages locally
& stop
EOF
fi

echo "[5/8] Configuring firewall..."

if [ "$MODE" = "server" ]; then
    case "$FIREWALL_CMD" in
        "ufw")
            ufw --force enable
            ufw allow 514/udp comment 'rsyslog UDP'
            ufw allow 514/tcp comment 'rsyslog TCP'
            ufw reload
            ;;
        "firewall-cmd")
            systemctl enable firewalld
            systemctl start firewalld
            firewall-cmd --permanent --add-port=514/udp --add-port=514/tcp
            firewall-cmd --reload
            ;;
    esac
fi

echo "[6/8] Configuring logrotate..."

if [ "$MODE" = "server" ]; then
    cat > /etc/logrotate.d/centralized-logs << 'EOF'
/var/log/centralized/*/*.log {
    daily
    rotate 30
    compress
    delaycompress
    missingok
    notifempty
    create 644 syslog adm
    postrotate
        systemctl reload rsyslog
    endscript
}

/var/log/centralized/ssh.log {
    daily
    rotate 90
    compress
    delaycompress
    missingok
    notifempty
    create 644 syslog adm
    postrotate
        systemctl reload rsyslog
    endscript
}

/var/log/centralized/failed-logins.log {
    daily
    rotate 180
    compress
    delaycompress
    missingok
    notifempty
    create 644 syslog adm
    postrotate
        systemctl reload rsyslog
    endscript
}
EOF
fi

echo "[7/8] Starting and enabling services..."
systemctl enable rsyslog
systemctl restart rsyslog
systemctl enable logrotate.timer
systemctl start logrotate.timer

echo "[8/8] Verifying installation..."

# Check if rsyslog is running
if systemctl is-active --quiet rsyslog; then
    log_info "rsyslog service is running"
else
    log_error "rsyslog service failed to start"
    exit 1
fi

# Check configuration syntax
if rsyslogd -N1 &>/dev/null; then
    log_info "rsyslog configuration syntax is valid"
else
    log_error "rsyslog configuration has syntax errors"
    exit 1
fi

# Test log generation
logger "Test log entry from rsyslog setup script"

if [ "$MODE" = "server" ]; then
    # Check if listening on ports
    if ss -tulpn | grep -q ':514'; then
        log_info "rsyslog is listening on port 514"
    else
        log_warn "rsyslog may not be listening on port 514"
    fi
    
    log_info "Server configuration complete!"
    log_info "Centralized logs will be stored in /var/log/centralized/"
    log_info "Configure clients to forward logs to this server's IP address"
else
    log_info "Client configuration complete!"
    log_info "Logs are being forwarded to $CENTRAL_SERVER_IP"
fi

log_info "Installation completed successfully!"