Configure Linux CPU and process scheduling with systemd and nice priorities for workload optimization

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Global variables
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BACKUP_DIR="/tmp/cpu-scheduling-backup-$(date +%s)"
PKG_MGR=""
PKG_INSTALL=""
UPDATE_CMD=""

# Usage function
usage() {
    echo -e "${BLUE}Usage: $0 [OPTIONS]${NC}"
    echo ""
    echo "Configure Linux CPU and process scheduling optimization"
    echo ""
    echo "Options:"
    echo "  -h, --help     Show this help message"
    echo "  --skip-update  Skip package manager update"
    echo ""
    echo "Examples:"
    echo "  $0                # Full installation"
    echo "  $0 --skip-update  # Skip package updates"
}

# Logging functions
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1" >&2
}

log_step() {
    echo -e "${BLUE}$1${NC}"
}

# Error handling and cleanup
cleanup_on_error() {
    local exit_code=$?
    log_error "Installation failed with exit code $exit_code"
    
    if [[ -d "$BACKUP_DIR" ]]; then
        log_info "Backup directory available at: $BACKUP_DIR"
        log_info "You can manually restore configurations if needed"
    fi
    
    exit $exit_code
}

trap cleanup_on_error ERR

# Check if running as root or with sudo
check_privileges() {
    if [[ $EUID -ne 0 ]]; then
        log_error "This script must be run as root or with sudo"
        exit 1
    fi
}

# Detect Linux distribution and set package manager
detect_distro() {
    log_step "[1/8] Detecting Linux distribution..."
    
    if [[ ! -f /etc/os-release ]]; then
        log_error "/etc/os-release not found. Cannot detect distribution."
        exit 1
    fi
    
    source /etc/os-release
    
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            UPDATE_CMD="apt update"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            UPDATE_CMD="dnf makecache"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            UPDATE_CMD="yum makecache"
            ;;
        *)
            log_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
    
    log_info "Detected distribution: $PRETTY_NAME"
    log_info "Using package manager: $PKG_MGR"
}

# Update package manager cache
update_packages() {
    log_step "[2/8] Updating package manager cache..."
    
    if [[ "$SKIP_UPDATE" == "false" ]]; then
        $UPDATE_CMD
        log_info "Package cache updated successfully"
    else
        log_info "Skipping package cache update"
    fi
}

# Install required packages
install_packages() {
    log_step "[3/8] Installing scheduling and monitoring tools..."
    
    # Common packages for all distributions
    local packages="util-linux htop iotop"
    
    # Add schedutils if available (may not be available on all distros)
    if [[ "$PKG_MGR" == "apt" ]]; then
        packages="$packages schedutils"
    elif [[ "$PKG_MGR" == "dnf" ]] || [[ "$PKG_MGR" == "yum" ]]; then
        # schedutils might be part of util-linux or not available
        packages="$packages procps-ng"
    fi
    
    $PKG_INSTALL $packages
    log_info "Required packages installed successfully"
}

# Create backup directory
create_backup() {
    log_step "[4/8] Creating backup directory..."
    
    mkdir -p "$BACKUP_DIR"
    
    # Backup system limits
    if [[ -f /etc/security/limits.conf ]]; then
        cp /etc/security/limits.conf "$BACKUP_DIR/limits.conf.bak"
    fi
    
    # Backup sysctl configuration
    if [[ -f /etc/sysctl.conf ]]; then
        cp /etc/sysctl.conf "$BACKUP_DIR/sysctl.conf.bak"
    fi
    
    log_info "Backup created at: $BACKUP_DIR"
}

# Configure system limits
configure_limits() {
    log_step "[5/8] Configuring system resource limits..."
    
    local limits_file="/etc/security/limits.conf"
    
    # Add scheduling priority limits
    cat >> "$limits_file" << 'EOF'

# CPU Scheduling Configuration - Added by install script
# Allow users in wheel/sudo group to set process priorities
@wheel          soft    nice            -10
@wheel          hard    nice            -10
@sudo           soft    nice            -10  
@sudo           hard    nice            -10

# RT scheduling limits for system services
root            soft    rtprio          99
root            hard    rtprio          99

EOF
    
    log_info "System limits configured in $limits_file"
}

# Configure kernel parameters
configure_kernel_params() {
    log_step "[6/8] Configuring kernel scheduling parameters..."
    
    local sysctl_file="/etc/sysctl.d/99-cpu-scheduling.conf"
    
    cat > "$sysctl_file" << 'EOF'
# CPU Scheduling Optimization Parameters
# Reduce scheduler migration cost
kernel.sched_migration_cost_ns = 5000000

# Increase scheduler latency for better throughput
kernel.sched_min_granularity_ns = 10000000
kernel.sched_wakeup_granularity_ns = 15000000

# RT throttling - prevent RT tasks from starving system
kernel.sched_rt_runtime_us = 950000
kernel.sched_rt_period_us = 1000000

# Process limits
kernel.pid_max = 4194304
kernel.threads-max = 2097152

EOF
    
    # Apply the settings
    sysctl -p "$sysctl_file"
    
    log_info "Kernel scheduling parameters configured"
}

# Create scheduling management scripts
create_management_scripts() {
    log_step "[7/8] Creating process scheduling management scripts..."
    
    local bin_dir="/usr/local/bin"
    
    # Create priority management script
    cat > "$bin_dir/set-process-priority" << 'EOF'
#!/bin/bash
# Process Priority Management Script

usage() {
    echo "Usage: $0 <pid|command> <nice_value> [io_class] [io_priority]"
    echo "Nice values: -20 (highest) to +19 (lowest priority)"
    echo "I/O classes: 0=none, 1=rt, 2=best-effort, 3=idle"
    echo "I/O priority: 0-7 (0=highest, 7=lowest)"
    exit 1
}

[[ $# -lt 2 ]] && usage

PROCESS=$1
NICE_VAL=$2
IO_CLASS=${3:-2}
IO_PRIO=${4:-4}

# Check if it's a PID or command
if [[ "$PROCESS" =~ ^[0-9]+$ ]]; then
    # It's a PID
    renice "$NICE_VAL" -p "$PROCESS"
    ionice -c "$IO_CLASS" -n "$IO_PRIO" -p "$PROCESS"
    echo "Updated priority for PID $PROCESS"
else
    # It's a command
    nice -n "$NICE_VAL" ionice -c "$IO_CLASS" -n "$IO_PRIO" "$PROCESS" "${@:5}"
fi
EOF
    
    chmod 755 "$bin_dir/set-process-priority"
    
    # Create system monitoring script
    cat > "$bin_dir/check-scheduling" << 'EOF'
#!/bin/bash
# System Scheduling Status Checker

echo "=== Process Scheduling Overview ==="
echo
echo "Top processes by CPU usage and priority:"
ps -eo pid,ppid,ni,pri,psr,pcpu,comm --sort=-pcpu | head -15
echo
echo "=== Real-time processes ==="
chrt -m
echo
echo "Processes using real-time scheduling:"
ps -eo pid,ni,pri,rtprio,cls,comm | grep -v ' - '
echo
echo "=== System load ==="
uptime
echo
echo "=== I/O scheduling classes ==="
echo "PID    Class    Priority    Command"
for pid in $(ps -eo pid --no-headers | head -10); do
    io_info=$(ionice -p $pid 2>/dev/null || echo "none: prio 4")
    cmd=$(ps -p $pid -o comm --no-headers 2>/dev/null || echo "unknown")
    printf "%-6s %-8s %s\n" "$pid" "$io_info" "$cmd"
done
EOF
    
    chmod 755 "$bin_dir/check-scheduling"
    
    log_info "Management scripts created in $bin_dir"
}

# Verify installation
verify_installation() {
    log_step "[8/8] Verifying installation..."
    
    local errors=0
    
    # Check if tools are available
    for tool in nice renice chrt ionice htop; do
        if ! command -v "$tool" >/dev/null 2>&1; then
            log_error "Tool not found: $tool"
            ((errors++))
        fi
    done
    
    # Check if management scripts exist and are executable
    for script in set-process-priority check-scheduling; do
        if [[ ! -x "/usr/local/bin/$script" ]]; then
            log_error "Management script not executable: $script"
            ((errors++))
        fi
    done
    
    # Check if kernel parameters were applied
    if ! sysctl kernel.sched_migration_cost_ns >/dev/null 2>&1; then
        log_error "Kernel scheduling parameters not applied"
        ((errors++))
    fi
    
    if [[ $errors -eq 0 ]]; then
        log_info "Installation verified successfully"
        echo
        echo -e "${GREEN}=== CPU Scheduling Configuration Complete ===${NC}"
        echo
        echo "Available commands:"
        echo "  set-process-priority <pid> <nice_value>  - Set process priority"
        echo "  check-scheduling                         - View scheduling status"
        echo "  htop                                     - Interactive process viewer"
        echo
        echo "Examples:"
        echo "  set-process-priority 1234 10            - Lower priority for PID 1234"
        echo "  nice -n 19 find / -name '*.log'         - Run find with lowest priority"
        echo "  sudo chrt -f 50 critical-app            - Run app with RT FIFO priority"
        echo
        echo -e "${YELLOW}Warning:${NC} Use real-time scheduling policies carefully to avoid system lockup"
    else
        log_error "Installation completed with $errors errors"
        exit 1
    fi
}

# Main function
main() {
    local SKIP_UPDATE=false
    
    # Parse command line arguments
    while [[ $# -gt 0 ]]; do
        case $1 in
            -h|--help)
                usage
                exit 0
                ;;
            --skip-update)
                SKIP_UPDATE=true
                shift
                ;;
            *)
                log_error "Unknown option: $1"
                usage
                exit 1
                ;;
        esac
    done
    
    echo -e "${GREEN}CPU and Process Scheduling Configuration${NC}"
    echo "========================================"
    
    check_privileges
    detect_distro
    update_packages
    install_packages
    create_backup
    configure_limits
    configure_kernel_params
    create_management_scripts
    verify_installation
}

# Run main function
main "$@"