Configure automated system maintenance with advanced cron scheduling and shell scripts

Create maintenance directory structure

Organize maintenance scripts and logs in a dedicated directory structure with proper permissions.

sudo mkdir -p /opt/maintenance/{scripts,logs,config}
sudo mkdir -p /opt/maintenance/scripts/{daily,weekly,monthly}
sudo chmod 755 /opt/maintenance
sudo chmod 750 /opt/maintenance/scripts

Set up maintenance user and permissions

Create a dedicated user for maintenance tasks to improve security and logging.

sudo useradd -r -s /bin/bash -d /opt/maintenance -c "System Maintenance" maintenance
sudo chown -R maintenance:maintenance /opt/maintenance
sudo usermod -aG adm maintenance

Create maintenance configuration file

Define global settings for all maintenance scripts in a central configuration file.

# System Maintenance Configuration
MAINT_LOG_DIR="/opt/maintenance/logs"
MAINT_SCRIPT_DIR="/opt/maintenance/scripts"
MAINT_USER="maintenance"
MAINT_GROUP="maintenance"

# Email settings
ALERT_EMAIL="admin@example.com"
SMTP_SERVER="localhost"

# Retention policies
LOG_RETENTION_DAYS=30
BACKUP_RETENTION_DAYS=7
TEMP_CLEANUP_DAYS=7

# System thresholds
DISK_USAGE_THRESHOLD=85
MEMORY_USAGE_THRESHOLD=90
LOAD_AVERAGE_THRESHOLD=5.0

Install and configure cron service

Ensure cron is installed and running with proper logging enabled.

sudo apt update
sudo apt install -y cron rsyslog
sudo systemctl enable --now cron
sudo systemctl enable --now rsyslog

sudo dnf update -y
sudo dnf install -y cronie rsyslog
sudo systemctl enable --now crond
sudo systemctl enable --now rsyslog

Configure cron logging

Enable detailed cron logging to track job execution and troubleshoot issues.

# Cron logging configuration
cron.*                          /var/log/cron.log

# Separate maintenance cron jobs
:programname, isequal, "CRON" /var/log/maintenance-cron.log
& stop

sudo systemctl restart rsyslog
sudo touch /var/log/cron.log /var/log/maintenance-cron.log
sudo chown syslog:adm /var/log/cron.log /var/log/maintenance-cron.log

Create advanced cron schedule patterns

Set up sophisticated scheduling patterns for different maintenance tasks using cron expressions and systemd timers.

sudo -u maintenance crontab -e

# Advanced Cron Scheduling for System Maintenance
# Format: minute hour day month weekday command

# Daily maintenance at 2:15 AM
15 2 * * * /opt/maintenance/scripts/daily/daily-maintenance.sh >> /opt/maintenance/logs/daily.log 2>&1

# Weekly maintenance on Sundays at 3:30 AM
30 3 * * 0 /opt/maintenance/scripts/weekly/weekly-maintenance.sh >> /opt/maintenance/logs/weekly.log 2>&1

# Monthly maintenance on the 1st at 4:00 AM
0 4 1 * * /opt/maintenance/scripts/monthly/monthly-maintenance.sh >> /opt/maintenance/logs/monthly.log 2>&1

# Security updates check every 6 hours
0 */6 * * * /opt/maintenance/scripts/daily/security-updates.sh >> /opt/maintenance/logs/security.log 2>&1

# Disk space monitoring every 30 minutes during business hours
*/30 9-17 * * 1-5 /opt/maintenance/scripts/daily/disk-monitor.sh >> /opt/maintenance/logs/disk-monitor.log 2>&1

# System health check every 5 minutes
*/5 * * * * /opt/maintenance/scripts/daily/health-check.sh >> /opt/maintenance/logs/health.log 2>&1

# Log rotation at midnight
0 0 * * * /opt/maintenance/scripts/daily/log-rotation.sh >> /opt/maintenance/logs/log-rotation.log 2>&1

Configure systemd timers for critical tasks

Use systemd timers for more reliable scheduling of critical maintenance tasks with better logging and dependency management.

[Unit]
Description=Daily System Maintenance
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
User=maintenance
Group=maintenance
WorkingDirectory=/opt/maintenance
ExecStart=/opt/maintenance/scripts/daily/daily-maintenance.sh
StandardOutput=append:/opt/maintenance/logs/systemd-daily.log
StandardError=append:/opt/maintenance/logs/systemd-daily.log

[Unit]
Description=Run daily system maintenance
Requires=system-maintenance.service

[Timer]
OnCalendar=daily
RandomizedDelaySec=30min
Persistent=true

[Install]
WantedBy=timers.target

sudo systemctl daemon-reload
sudo systemctl enable --now system-maintenance.timer

Create master daily maintenance script

Develop a comprehensive daily maintenance script with error handling, logging, and modular task execution.

#!/bin/bash
# Daily System Maintenance Script
# Author: System Administrator
# Version: 1.0

set -euo pipefail

# Source configuration
source /opt/maintenance/config/maintenance.conf

# Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$MAINT_LOG_DIR/daily-$(date +%Y%m%d).log"
}

# Error handling
error_handler() {
    local line_no=$1
    log "ERROR: Script failed at line $line_no"
    echo "Daily maintenance failed at $(date)" | mail -s "Maintenance Alert: $(hostname)" "$ALERT_EMAIL"
    exit 1
}

trap 'error_handler ${LINENO}' ERR

log "Starting daily maintenance on $(hostname)"

# Update package lists
log "Updating package lists"
if command -v apt >/dev/null 2>&1; then
    sudo apt update -qq
elif command -v dnf >/dev/null 2>&1; then
    sudo dnf check-update -q || true
fi

# Clean temporary files older than threshold
log "Cleaning temporary files older than $TEMP_CLEANUP_DAYS days"
find /tmp -type f -mtime +$TEMP_CLEANUP_DAYS -delete 2>/dev/null || true
find /var/tmp -type f -mtime +$TEMP_CLEANUP_DAYS -delete 2>/dev/null || true

# Clean old log files
log "Rotating and compressing old logs"
find /var/log -name "*.log" -mtime +1 -exec gzip {} \;
find /var/log -name "*.gz" -mtime +$LOG_RETENTION_DAYS -delete

# Check disk usage
log "Checking disk usage"
disk_usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
if [ "$disk_usage" -gt "$DISK_USAGE_THRESHOLD" ]; then
    log "WARNING: Disk usage at $disk_usage%"
    echo "Disk usage critical on $(hostname): $disk_usage%" | mail -s "Disk Alert: $(hostname)" "$ALERT_EMAIL"
fi

# Update file database
log "Updating locate database"
sudo updatedb

# Clean package caches
log "Cleaning package caches"
if command -v apt >/dev/null 2>&1; then
    sudo apt autoremove -y -qq
    sudo apt autoclean -qq
elif command -v dnf >/dev/null 2>&1; then
    sudo dnf autoremove -y -q
    sudo dnf clean packages -q
fi

# Generate system summary
log "Generating system summary"
echo "=== Daily System Summary for $(hostname) ===" > "$MAINT_LOG_DIR/summary-$(date +%Y%m%d).txt"
echo "Date: $(date)" >> "$MAINT_LOG_DIR/summary-$(date +%Y%m%d).txt"
echo "Uptime: $(uptime)" >> "$MAINT_LOG_DIR/summary-$(date +%Y%m%d).txt"
echo "Disk Usage: $(df -h / | awk 'NR==2 {print $5}')" >> "$MAINT_LOG_DIR/summary-$(date +%Y%m%d).txt"
echo "Memory Usage: $(free -m | awk 'NR==2{printf "%.1f%%", $3*100/$2}')" >> "$MAINT_LOG_DIR/summary-$(date +%Y%m%d).txt"
echo "Load Average: $(cat /proc/loadavg | cut -d' ' -f1-3)" >> "$MAINT_LOG_DIR/summary-$(date +%Y%m%d).txt"

log "Daily maintenance completed successfully"

# Cleanup old maintenance logs
find "$MAINT_LOG_DIR" -name "daily-*.log" -mtime +$LOG_RETENTION_DAYS -delete
find "$MAINT_LOG_DIR" -name "summary-*.txt" -mtime +$LOG_RETENTION_DAYS -delete

chmod 750 /opt/maintenance/scripts/daily/daily-maintenance.sh
sudo chown maintenance:maintenance /opt/maintenance/scripts/daily/daily-maintenance.sh

Create security updates monitoring script

Develop a script to monitor and optionally install critical security updates automatically.

#!/bin/bash
# Security Updates Monitoring Script

set -euo pipefail

source /opt/maintenance/config/maintenance.conf

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$MAINT_LOG_DIR/security-$(date +%Y%m%d).log"
}

log "Checking for security updates"

if command -v apt >/dev/null 2>&1; then
    # Ubuntu/Debian security updates
    security_updates=$(apt list --upgradable 2>/dev/null | grep -E '(security|Ubuntu.*-security)' | wc -l)
    
    if [ "$security_updates" -gt 0 ]; then
        log "Found $security_updates security updates available"
        
        # List available security updates
        apt list --upgradable 2>/dev/null | grep -E '(security|Ubuntu.*-security)' > "$MAINT_LOG_DIR/security-updates-$(date +%Y%m%d).txt"
        
        # Install unattended-upgrades if not present
        if ! dpkg -l | grep -q unattended-upgrades; then
            log "Installing unattended-upgrades"
            sudo apt install -y unattended-upgrades
        fi
        
        # Configure automatic security updates
        echo 'Unattended-Upgrade::Allowed-Origins {
"${distro_id}:${distro_codename}-security";
"${distro_id} ESMApps:${distro_codename}-apps-security";
"${distro_id} ESM:${distro_codename}-infra-security";
};' | sudo tee /etc/apt/apt.conf.d/50unattended-upgrades
        
        # Send alert
        echo "Security updates available on $(hostname): $security_updates updates" | mail -s "Security Alert: $(hostname)" "$ALERT_EMAIL"
    else
        log "No security updates available"
    fi
    
elif command -v dnf >/dev/null 2>&1; then
    # RHEL/CentOS/AlmaLinux/Rocky security updates
    security_updates=$(dnf --security check-update 2>/dev/null | wc -l)
    
    if [ "$security_updates" -gt 0 ]; then
        log "Found $security_updates security updates available"
        
        # List available security updates
        dnf --security check-update > "$MAINT_LOG_DIR/security-updates-$(date +%Y%m%d).txt" 2>/dev/null
        
        # Configure automatic security updates
        if ! rpm -q dnf-automatic >/dev/null 2>&1; then
            log "Installing dnf-automatic"
            sudo dnf install -y dnf-automatic
        fi
        
        # Configure dnf-automatic for security updates only
        sudo sed -i 's/upgrade_type = default/upgrade_type = security/' /etc/dnf/automatic.conf
        sudo sed -i 's/apply_updates = no/apply_updates = yes/' /etc/dnf/automatic.conf
        sudo systemctl enable --now dnf-automatic-install.timer
        
        # Send alert
        echo "Security updates available on $(hostname): $security_updates updates" | mail -s "Security Alert: $(hostname)" "$ALERT_EMAIL"
    else
        log "No security updates available"
    fi
fi

log "Security update check completed"

chmod 750 /opt/maintenance/scripts/daily/security-updates.sh
sudo chown maintenance:maintenance /opt/maintenance/scripts/daily/security-updates.sh

Create system health monitoring script

Build a comprehensive health monitoring script that checks system resources and services.

#!/bin/bash
# System Health Check Script

set -euo pipefail

source /opt/maintenance/config/maintenance.conf

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
}

# Check disk usage
check_disk_usage() {
    local usage
    usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
    
    if [ "$usage" -gt "$DISK_USAGE_THRESHOLD" ]; then
        log "CRITICAL: Disk usage at $usage%"
        echo "Disk usage critical on $(hostname): $usage%" | mail -s "Critical Alert: Disk Usage" "$ALERT_EMAIL"
        return 1
    fi
    
    log "OK: Disk usage at $usage%"
    return 0
}

# Check memory usage
check_memory_usage() {
    local mem_usage
    mem_usage=$(free | awk 'NR==2{printf "%.0f", $3*100/$2}')
    
    if [ "$mem_usage" -gt "$MEMORY_USAGE_THRESHOLD" ]; then
        log "WARNING: Memory usage at $mem_usage%"
        echo "High memory usage on $(hostname): $mem_usage%" | mail -s "Warning: Memory Usage" "$ALERT_EMAIL"
        return 1
    fi
    
    log "OK: Memory usage at $mem_usage%"
    return 0
}

# Check load average
check_load_average() {
    local load_avg
    load_avg=$(cat /proc/loadavg | cut -d' ' -f1)
    
    if (( $(echo "$load_avg > $LOAD_AVERAGE_THRESHOLD" | bc -l) )); then
        log "WARNING: Load average at $load_avg"
        echo "High load average on $(hostname): $load_avg" | mail -s "Warning: Load Average" "$ALERT_EMAIL"
        return 1
    fi
    
    log "OK: Load average at $load_avg"
    return 0
}

# Check critical services
check_services() {
    local services=("ssh" "cron" "rsyslog")
    local failed_services=()
    
    for service in "${services[@]}"; do
        if ! systemctl is-active --quiet "$service"; then
            failed_services+=("$service")
            log "CRITICAL: Service $service is not running"
        else
            log "OK: Service $service is running"
        fi
    done
    
    if [ ${#failed_services[@]} -gt 0 ]; then
        echo "Critical services down on $(hostname): ${failed_services[*]}" | mail -s "Critical Alert: Services Down" "$ALERT_EMAIL"
        return 1
    fi
    
    return 0
}

# Check filesystem corruption
check_filesystem() {
    local errors
    errors=$(dmesg | grep -i "ext.*error" | tail -5)
    
    if [ -n "$errors" ]; then
        log "WARNING: Filesystem errors detected"
        echo "Filesystem errors on $(hostname): $errors" | mail -s "Warning: Filesystem Errors" "$ALERT_EMAIL"
        return 1
    fi
    
    log "OK: No filesystem errors detected"
    return 0
}

# Main health check execution
main() {
    local exit_code=0
    
    log "Starting system health check"
    
    check_disk_usage || exit_code=1
    check_memory_usage || exit_code=1
    check_load_average || exit_code=1
    check_services || exit_code=1
    check_filesystem || exit_code=1
    
    if [ $exit_code -eq 0 ]; then
        log "System health check completed - All OK"
    else
        log "System health check completed - Issues detected"
    fi
    
    return $exit_code
}

main "$@"

chmod 750 /opt/maintenance/scripts/daily/health-check.sh
sudo chown maintenance:maintenance /opt/maintenance/scripts/daily/health-check.sh

Create weekly maintenance script

Build a weekly maintenance script for more intensive tasks like system updates and database optimization.

#!/bin/bash
# Weekly System Maintenance Script

set -euo pipefail

source /opt/maintenance/config/maintenance.conf

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$MAINT_LOG_DIR/weekly-$(date +%Y%m%d).log"
}

log "Starting weekly maintenance on $(hostname)"

# Full system update (non-security packages)
log "Performing full system update"
if command -v apt >/dev/null 2>&1; then
    sudo apt update -q
    sudo apt upgrade -y -q
    sudo apt autoremove -y -q
    sudo apt autoclean -q
elif command -v dnf >/dev/null 2>&1; then
    sudo dnf upgrade -y -q
    sudo dnf autoremove -y -q
    sudo dnf clean all -q
fi

# Defragment and optimize databases if present
if command -v mysql >/dev/null 2>&1; then
    log "Optimizing MySQL databases"
    mysql -e "FLUSH LOGS; OPTIMIZE TABLE mysql.general_log; OPTIMIZE TABLE mysql.slow_log;"
fi

# Clean old kernels (keep last 2)
if command -v apt >/dev/null 2>&1; then
    log "Cleaning old kernels"
    sudo apt autoremove --purge -y
fi

# Update SSL certificates
if command -v certbot >/dev/null 2>&1; then
    log "Renewing SSL certificates"
    sudo certbot renew --quiet
fi

# Backup important configuration files
log "Backing up configuration files"
backup_dir="/opt/maintenance/backups/$(date +%Y%m%d)"
mkdir -p "$backup_dir"

tar -czf "$backup_dir/etc-backup.tar.gz" /etc/ 2>/dev/null || true
tar -czf "$backup_dir/opt-maintenance-backup.tar.gz" /opt/maintenance/ 2>/dev/null || true

# Clean old backups
find /opt/maintenance/backups -type d -mtime +$BACKUP_RETENTION_DAYS -exec rm -rf {} \;

# Generate weekly report
log "Generating weekly system report"
weekly_report="$MAINT_LOG_DIR/weekly-report-$(date +%Y%m%d).txt"
{
    echo "=== Weekly System Report for $(hostname) ==="
    echo "Report Date: $(date)"
    echo ""
    echo "System Information:"
    uname -a
    echo ""
    echo "Disk Usage:"
    df -h
    echo ""
    echo "Memory Information:"
    free -h
    echo ""
    echo "Runnin