Configure Linux system monitoring with iostat and disk performance analysis

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Script info
SCRIPT_NAME="iostat-monitoring-installer"
LOG_FILE="/var/log/${SCRIPT_NAME}.log"

# Cleanup function for rollback
cleanup() {
    local exit_code=$?
    if [ $exit_code -ne 0 ]; then
        echo -e "${RED}[ERROR] Installation failed. Check ${LOG_FILE} for details${NC}"
        # Rollback actions if needed
        if systemctl is-enabled sysstat >/dev/null 2>&1; then
            systemctl disable sysstat >/dev/null 2>&1 || true
        fi
    fi
    exit $exit_code
}

trap cleanup ERR

# Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S'): $1" | tee -a "${LOG_FILE}"
}

# Print colored output
print_status() {
    local color=$1
    local message=$2
    echo -e "${color}${message}${NC}"
    log "$message"
}

# Check if running as root or with sudo
check_privileges() {
    if [[ $EUID -ne 0 ]] && ! sudo -n true 2>/dev/null; then
        print_status "$RED" "[ERROR] This script requires root privileges or sudo access"
        exit 1
    fi
}

# Auto-detect distribution
detect_distro() {
    if [ ! -f /etc/os-release ]; then
        print_status "$RED" "[ERROR] Cannot detect Linux distribution"
        exit 1
    fi
    
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_UPGRADE="apt upgrade -y"
            PKG_INSTALL="apt install -y"
            SYSSTAT_CONFIG="/etc/default/sysstat"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf check-update || true"
            PKG_UPGRADE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            SYSSTAT_CONFIG=""
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum check-update || true"
            PKG_UPGRADE="yum update -y"
            PKG_INSTALL="yum install -y"
            SYSSTAT_CONFIG=""
            ;;
        *)
            print_status "$RED" "[ERROR] Unsupported distribution: $ID"
            exit 1
            ;;
    esac
    
    print_status "$GREEN" "Detected distribution: $PRETTY_NAME"
}

# Update system packages
update_system() {
    print_status "$YELLOW" "[1/6] Updating system packages..."
    
    $PKG_UPDATE >> "${LOG_FILE}" 2>&1
    $PKG_UPGRADE >> "${LOG_FILE}" 2>&1
    
    print_status "$GREEN" "System packages updated successfully"
}

# Install sysstat package
install_sysstat() {
    print_status "$YELLOW" "[2/6] Installing sysstat package..."
    
    if command -v iostat >/dev/null 2>&1; then
        print_status "$YELLOW" "sysstat already installed, checking version..."
        iostat -V >> "${LOG_FILE}" 2>&1 || true
    fi
    
    $PKG_INSTALL sysstat >> "${LOG_FILE}" 2>&1
    
    # Verify installation
    if ! command -v iostat >/dev/null 2>&1; then
        print_status "$RED" "[ERROR] iostat command not found after installation"
        exit 1
    fi
    
    print_status "$GREEN" "sysstat package installed successfully"
}

# Configure sysstat service
configure_sysstat() {
    print_status "$YELLOW" "[3/6] Configuring sysstat service..."
    
    # Enable data collection for Debian-based systems
    if [[ -n "$SYSSTAT_CONFIG" && -f "$SYSSTAT_CONFIG" ]]; then
        cp "$SYSSTAT_CONFIG" "${SYSSTAT_CONFIG}.backup"
        sed -i 's/ENABLED="false"/ENABLED="true"/' "$SYSSTAT_CONFIG"
        print_status "$GREEN" "Enabled sysstat data collection in $SYSSTAT_CONFIG"
    fi
    
    # Enable and start sysstat service
    systemctl enable sysstat >> "${LOG_FILE}" 2>&1
    systemctl start sysstat >> "${LOG_FILE}" 2>&1
    
    print_status "$GREEN" "sysstat service configured and started"
}

# Create monitoring scripts
create_monitoring_scripts() {
    print_status "$YELLOW" "[4/6] Creating monitoring scripts..."
    
    # Create monitoring directory
    mkdir -p /opt/iostat-monitoring
    
    # Create disk monitoring script
    cat > /opt/iostat-monitoring/check-disk-performance.sh << 'EOF'
#!/bin/bash
# Disk performance monitoring script

THRESHOLD_UTIL=80
THRESHOLD_AWAIT=20
LOG_FILE="/var/log/disk-performance.log"

echo "$(date): Checking disk performance..." >> "$LOG_FILE"

# Check for high disk utilization
iostat -x 1 1 | awk -v threshold="$THRESHOLD_UTIL" '
/^[a-z]/ && $10 > threshold {
    print "WARNING: Device " $1 " has " $10 "% utilization (threshold: " threshold "%)"
}' >> "$LOG_FILE"

# Check for high await times
iostat -x 1 1 | awk -v threshold="$THRESHOLD_AWAIT" '
/^[a-z]/ && $10 > 50 && $9 > threshold {
    print "WARNING: Device " $1 " has " $9 "ms average wait time (threshold: " threshold "ms)"
}' >> "$LOG_FILE"

# Log current I/O wait
IOWAIT=$(iostat -c 1 2 | tail -1 | awk '{print $4}')
if (( $(echo "$IOWAIT > 10" | bc -l) )); then
    echo "WARNING: High I/O wait: ${IOWAIT}%" >> "$LOG_FILE"
fi
EOF

    chmod 755 /opt/iostat-monitoring/check-disk-performance.sh
    chown root:root /opt/iostat-monitoring/check-disk-performance.sh
    
    print_status "$GREEN" "Monitoring scripts created"
}

# Setup systemd timer for automated monitoring
setup_systemd_timer() {
    print_status "$YELLOW" "[5/6] Setting up automated monitoring..."
    
    # Create systemd service
    cat > /etc/systemd/system/disk-monitoring.service << EOF
[Unit]
Description=Disk Performance Monitoring
After=multi-user.target

[Service]
Type=oneshot
ExecStart=/opt/iostat-monitoring/check-disk-performance.sh
User=root
StandardOutput=journal
StandardError=journal
EOF

    # Create systemd timer
    cat > /etc/systemd/system/disk-monitoring.timer << EOF
[Unit]
Description=Run disk monitoring every 5 minutes
Requires=disk-monitoring.service

[Timer]
OnCalendar=*:0/5
Persistent=true

[Install]
WantedBy=timers.target
EOF

    # Set correct permissions
    chmod 644 /etc/systemd/system/disk-monitoring.service
    chmod 644 /etc/systemd/system/disk-monitoring.timer
    
    # Reload systemd and enable timer
    systemctl daemon-reload >> "${LOG_FILE}" 2>&1
    systemctl enable disk-monitoring.timer >> "${LOG_FILE}" 2>&1
    systemctl start disk-monitoring.timer >> "${LOG_FILE}" 2>&1
    
    print_status "$GREEN" "Automated monitoring configured"
}

# Verify installation
verify_installation() {
    print_status "$YELLOW" "[6/6] Verifying installation..."
    
    # Test iostat command
    if ! iostat 1 1 >> "${LOG_FILE}" 2>&1; then
        print_status "$RED" "[ERROR] iostat command failed"
        exit 1
    fi
    
    # Check sysstat service status
    if ! systemctl is-active --quiet sysstat; then
        print_status "$YELLOW" "[WARNING] sysstat service is not active"
    fi
    
    # Check timer status
    if ! systemctl is-active --quiet disk-monitoring.timer; then
        print_status "$RED" "[ERROR] disk-monitoring timer is not active"
        exit 1
    fi
    
    # Check if bc is available for monitoring script
    if ! command -v bc >/dev/null 2>&1; then
        print_status "$YELLOW" "Installing bc for monitoring calculations..."
        $PKG_INSTALL bc >> "${LOG_FILE}" 2>&1
    fi
    
    print_status "$GREEN" "Installation verified successfully"
}

# Main installation function
main() {
    print_status "$GREEN" "Starting iostat monitoring installation..."
    
    check_privileges
    detect_distro
    update_system
    install_sysstat
    configure_sysstat
    create_monitoring_scripts
    setup_systemd_timer
    verify_installation
    
    print_status "$GREEN" "Installation completed successfully!"
    echo
    echo "Usage examples:"
    echo "  iostat                    # Show current stats"
    echo "  iostat -x 1 5            # Extended stats, 1s interval, 5 times"
    echo "  iostat -xh 2             # Human-readable format, 2s interval"
    echo "  systemctl status disk-monitoring.timer  # Check monitoring status"
    echo "  tail -f /var/log/disk-performance.log   # View monitoring alerts"
}

main "$@"