Configure Linux system logging with rsyslog and journald for centralized log management

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Configuration variables
LOG_SERVER=""
REMOTE_LOGGING=false

# Cleanup function
cleanup() {
    echo -e "${RED}[ERROR] Installation failed. Restoring original configurations...${NC}"
    
    if [ -f /etc/systemd/journald.conf.backup ]; then
        mv /etc/systemd/journald.conf.backup /etc/systemd/journald.conf
    fi
    
    if [ -f /etc/rsyslog.conf.backup ]; then
        mv /etc/rsyslog.conf.backup /etc/rsyslog.conf
    fi
    
    rm -f /etc/rsyslog.d/99-remote.conf
    
    echo -e "${YELLOW}[WARNING] Cleanup completed. Original configurations restored.${NC}"
}

trap cleanup ERR

# Usage message
show_usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -s SERVER    Configure remote log server (optional)"
    echo "  -h           Show this help message"
    echo ""
    echo "Examples:"
    echo "  $0                                    # Local logging only"
    echo "  $0 -s log.example.com                # With remote logging"
}

# Parse arguments
while getopts "s:h" opt; do
    case $opt in
        s)
            LOG_SERVER="$OPTARG"
            REMOTE_LOGGING=true
            ;;
        h)
            show_usage
            exit 0
            ;;
        \?)
            echo -e "${RED}Invalid option: -$OPTARG${NC}" >&2
            show_usage
            exit 1
            ;;
    esac
done

# Check prerequisites
echo -e "${BLUE}[0/8] Checking prerequisites...${NC}"

if [ "$EUID" -ne 0 ]; then
    echo -e "${RED}[ERROR] This script must be run as root${NC}"
    exit 1
fi

# Auto-detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update && apt upgrade -y"
            PKG_INSTALL="apt install -y"
            SYSLOG_USER="syslog"
            SYSLOG_GROUP="adm"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            SYSLOG_USER="root"
            SYSLOG_GROUP="root"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            SYSLOG_USER="root"
            SYSLOG_GROUP="root"
            ;;
        *)
            echo -e "${RED}[ERROR] Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}[ERROR] Cannot detect distribution${NC}"
    exit 1
fi

echo -e "${GREEN}[SUCCESS] Detected distribution: $PRETTY_NAME${NC}"

# Update system packages
echo -e "${BLUE}[1/8] Updating system packages...${NC}"
$PKG_UPDATE

# Install required packages
echo -e "${BLUE}[2/8] Installing logging components...${NC}"
$PKG_INSTALL rsyslog logrotate

# Try to install rsyslog-relp if available
if $REMOTE_LOGGING; then
    if $PKG_INSTALL rsyslog-relp 2>/dev/null; then
        echo -e "${GREEN}[SUCCESS] rsyslog-relp installed${NC}"
    else
        echo -e "${YELLOW}[WARNING] rsyslog-relp not available, using UDP transport${NC}"
    fi
fi

# Create backup of journald configuration
echo -e "${BLUE}[3/8] Configuring journald...${NC}"
cp /etc/systemd/journald.conf /etc/systemd/journald.conf.backup

# Configure journald
cat > /etc/systemd/journald.conf << 'EOF'
[Journal]
Storage=persistent
Compress=yes
Seal=yes
SplitMode=uid
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=10000
SystemMaxUse=4G
SystemKeepFree=1G
SystemMaxFileSize=128M
SystemMaxFiles=100
MaxRetentionSec=2month
MaxFileSec=1month
ForwardToSyslog=yes
ForwardToKMsg=no
ForwardToConsole=no
ForwardToWall=yes
EOF

# Create journald storage directory
echo -e "${BLUE}[4/8] Setting up journald storage...${NC}"
mkdir -p /var/log/journal
chown root:systemd-journal /var/log/journal
chmod 2755 /var/log/journal

# Create backup of rsyslog configuration
echo -e "${BLUE}[5/8] Configuring rsyslog...${NC}"
cp /etc/rsyslog.conf /etc/rsyslog.conf.backup

# Configure main rsyslog
cat > /etc/rsyslog.conf << EOF
# Global configuration modules
\$ModLoad imuxsock
\$ModLoad imklog
\$ModLoad imjournal
\$ModLoad immark

# High precision timestamps
\$template HighPrecision,"%timegenerated:::date-rfc3339% %HOSTNAME% %syslogtag%%msg%\\n"
\$ActionFileDefaultTemplate HighPrecision

# Work directory
\$WorkDirectory /var/spool/rsyslog

# Include all config files in /etc/rsyslog.d/
\$IncludeConfig /etc/rsyslog.d/*.conf

# Configuration settings
\$OmitLocalLogging on
\$ActionFileEnableSync on

# Set permissions and security
\$FileOwner $SYSLOG_USER
\$FileGroup $SYSLOG_GROUP
\$FileCreateMode 0640
\$DirCreateMode 0755
\$Umask 0022

# Drop privileges (Debian/Ubuntu only)
EOF

if [ "$PKG_MGR" = "apt" ]; then
    cat >> /etc/rsyslog.conf << EOF
\$PrivDropToUser syslog
\$PrivDropToGroup syslog

EOF
fi

cat >> /etc/rsyslog.conf << 'EOF'
# Standard log files
*.*;auth,authpriv.none          -/var/log/syslog
auth,authpriv.*                 /var/log/auth.log
*.*;auth,authpriv.none          -/var/log/syslog
cron.*                          /var/log/cron.log
daemon.*                        -/var/log/daemon.log
kern.*                          -/var/log/kern.log
lpr.*                           -/var/log/lpr.log
mail.*                          -/var/log/mail.log
user.*                          -/var/log/user.log

# Emergency messages
*.emerg                         :omusrmsg:*

# News and mail logging
mail.info                       -/var/log/mail.info
mail.warn                       -/var/log/mail.warn
mail.err                        /var/log/mail.err
EOF

# Configure remote logging if requested
echo -e "${BLUE}[6/8] Setting up logging destinations...${NC}"
if $REMOTE_LOGGING && [ -n "$LOG_SERVER" ]; then
    cat > /etc/rsyslog.d/99-remote.conf << EOF
# Load RELP output module (if available)
\$ModLoad omrelp

# Remote logging configuration
\$template RemoteHost,"/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log"

# Forward all logs to remote server
*.* @@$LOG_SERVER:514

# Also try RELP if module is loaded
*.* :omrelp:$LOG_SERVER:2514
EOF
    echo -e "${GREEN}[SUCCESS] Remote logging configured to $LOG_SERVER${NC}"
else
    echo -e "${YELLOW}[INFO] Local logging only${NC}"
fi

# Create necessary directories
mkdir -p /var/spool/rsyslog /var/log/remote
chown $SYSLOG_USER:$SYSLOG_GROUP /var/spool/rsyslog
chmod 755 /var/spool/rsyslog

# Restart and enable services
echo -e "${BLUE}[7/8] Restarting logging services...${NC}"
systemctl restart systemd-journald
systemctl restart rsyslog
systemctl enable systemd-journald
systemctl enable rsyslog

# Verify configuration
echo -e "${BLUE}[8/8] Verifying installation...${NC}"

# Check service status
if systemctl is-active --quiet systemd-journald; then
    echo -e "${GREEN}[SUCCESS] systemd-journald is running${NC}"
else
    echo -e "${RED}[ERROR] systemd-journald is not running${NC}"
    exit 1
fi

if systemctl is-active --quiet rsyslog; then
    echo -e "${GREEN}[SUCCESS] rsyslog is running${NC}"
else
    echo -e "${RED}[ERROR] rsyslog is not running${NC}"
    exit 1
fi

# Check log directories
if [ -d /var/log/journal ]; then
    echo -e "${GREEN}[SUCCESS] Journal persistent storage configured${NC}"
else
    echo -e "${RED}[ERROR] Journal directory not found${NC}"
    exit 1
fi

# Test logging
logger "Test message from rsyslog configuration script"
sleep 2

if grep -q "Test message from rsyslog configuration script" /var/log/syslog 2>/dev/null; then
    echo -e "${GREEN}[SUCCESS] Log message routing verified${NC}"
else
    echo -e "${YELLOW}[WARNING] Could not verify log routing${NC}"
fi

# Clean up trap
trap - ERR

echo -e "${GREEN}[COMPLETE] Linux logging configuration completed successfully!${NC}"
echo ""
echo -e "${BLUE}Configuration Summary:${NC}"
echo "- journald: Persistent storage enabled with 2-month retention"
echo "- rsyslog: High precision timestamps and structured logging"
echo "- Log rotation: Managed by logrotate"
if $REMOTE_LOGGING; then
    echo "- Remote logging: Configured to $LOG_SERVER"
fi
echo ""
echo -e "${YELLOW}Next steps:${NC}"
echo "- Review log files in /var/log/"
echo "- Monitor journald with: journalctl -f"
echo "- Check rsyslog status: systemctl status rsyslog"
if $REMOTE_LOGGING; then
    echo "- Verify remote log delivery to $LOG_SERVER"
fi