Implement backup rotation policies with automated cleanup using systemd timers and shell scripts

Create the backup rotation directory structure

Set up dedicated directories for backup scripts, logs, and configuration files with proper permissions.

sudo mkdir -p /opt/backup-rotation/{scripts,config,logs}
sudo mkdir -p /var/log/backup-rotation
sudo chown -R root:root /opt/backup-rotation
sudo chmod 755 /opt/backup-rotation /opt/backup-rotation/scripts
sudo chmod 644 /opt/backup-rotation/config

Create the main rotation configuration file

Define retention policies for different backup types in a central configuration file that the scripts will read.

# Backup retention configuration
# Format: TYPE:PATH:DAILY:WEEKLY:MONTHLY
# DAILY = days to keep daily backups
# WEEKLY = weeks to keep weekly backups  
# MONTHLY = months to keep monthly backups

database:/var/backups/database:7:4:3
files:/var/backups/files:14:8:6
logs:/var/backups/logs:30:12:12
config:/var/backups/config:7:8:12

Create the backup rotation script

This script reads the retention configuration and removes old backups based on file age and naming patterns.

#!/bin/bash

set -euo pipefail

# Configuration
CONFIG_FILE="/opt/backup-rotation/config/retention.conf"
LOG_FILE="/var/log/backup-rotation/rotation.log"
LOCK_FILE="/var/run/backup-rotation.lock"

# Logging function
log_message() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

# Check if script is already running
if [ -f "$LOCK_FILE" ]; then
    log_message "ERROR: Backup rotation already running (lock file exists)"
    exit 1
fi

# Create lock file
echo $$ > "$LOCK_FILE"
trap 'rm -f "$LOCK_FILE"' EXIT

log_message "Starting backup rotation process"

# Read configuration and process each backup type
while IFS=':' read -r backup_type backup_path daily_keep weekly_keep monthly_keep; do
    # Skip comments and empty lines
    [[ "$backup_type" =~ ^#.*$ ]] && continue
    [[ -z "$backup_type" ]] && continue
    
    log_message "Processing $backup_type backups in $backup_path"
    
    if [ ! -d "$backup_path" ]; then
        log_message "WARNING: Backup directory $backup_path does not exist"
        continue
    fi
    
    # Remove daily backups older than specified days
    find "$backup_path" -name "*daily*" -type f -mtime +"$daily_keep" -exec rm -f {} \; -print | while read deleted_file; do
        log_message "Deleted daily backup: $deleted_file"
    done
    
    # Remove weekly backups older than specified weeks
    weekly_days=$((weekly_keep * 7))
    find "$backup_path" -name "*weekly*" -type f -mtime +"$weekly_days" -exec rm -f {} \; -print | while read deleted_file; do
        log_message "Deleted weekly backup: $deleted_file"
    done
    
    # Remove monthly backups older than specified months (approximate)
    monthly_days=$((monthly_keep * 30))
    find "$backup_path" -name "*monthly*" -type f -mtime +"$monthly_days" -exec rm -f {} \; -print | while read deleted_file; do
        log_message "Deleted monthly backup: $deleted_file"
    done
    
    # Calculate and log space usage
    if command -v du >/dev/null; then
        space_used=$(du -sh "$backup_path" | cut -f1)
        log_message "Current space usage for $backup_type: $space_used"
    fi
    
done < "$CONFIG_FILE"

log_message "Backup rotation completed successfully"

# Clean up old rotation logs (keep 30 days)
find /var/log/backup-rotation -name "*.log" -mtime +30 -delete

Make the rotation script executable

Set proper permissions on the backup rotation script so systemd can execute it.

sudo chmod 755 /opt/backup-rotation/scripts/rotate-backups.sh
sudo chown root:root /opt/backup-rotation/scripts/rotate-backups.sh

Create the systemd service unit

Define a systemd service that runs the backup rotation script with proper logging and error handling.

[Unit]
Description=Backup Rotation Service
After=network.target

[Service]
Type=oneshot
User=root
Group=root
ExecStart=/opt/backup-rotation/scripts/rotate-backups.sh
StandardOutput=journal
StandardError=journal
SyslogIdentifier=backup-rotation

# Resource limits
MemoryMax=512M
CPUQuota=25%

# Security hardening
PrivateTmp=true
ProtectSystem=strict
ReadWritePaths=/var/backups /var/log/backup-rotation /var/run
NoNewPrivileges=true
ProtectHome=true
ProtectControlGroups=true
ProtectKernelModules=true
ProtectKernelTunables=true

Create the systemd timer unit

Set up a timer that runs backup rotation daily at 2 AM with randomization to prevent resource conflicts.

[Unit]
Description=Daily Backup Rotation Timer
Requires=backup-rotation.service

[Timer]
OnCalendar=*-*-* 02:00:00
RandomizedDelaySec=1800
Persistent=true

[Install]
WantedBy=timers.target

Enable and start the systemd timer

Reload systemd configuration and enable the timer to start automatically on boot.

sudo systemctl daemon-reload
sudo systemctl enable backup-rotation.timer
sudo systemctl start backup-rotation.timer

Create a backup health monitoring script

This script checks backup status, disk usage, and rotation process health for monitoring integration.

#!/bin/bash

set -euo pipefail

# Configuration
CONFIG_FILE="/opt/backup-rotation/config/retention.conf"
LOG_FILE="/var/log/backup-rotation/health-check.log"
ALERT_THRESHOLD_PERCENT=85
MAX_AGE_HOURS=26

# Exit codes for monitoring systems
OK=0
WARNING=1
CRITICAL=2

log_message() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

exit_code=$OK
alert_messages=()

log_message "Starting backup health check"

# Check each backup location
while IFS=':' read -r backup_type backup_path daily_keep weekly_keep monthly_keep; do
    [[ "$backup_type" =~ ^#.*$ ]] && continue
    [[ -z "$backup_type" ]] && continue
    
    if [ ! -d "$backup_path" ]; then
        alert_messages+=("CRITICAL: Backup directory $backup_path missing")
        exit_code=$CRITICAL
        continue
    fi
    
    # Check disk usage
    usage_percent=$(df "$backup_path" | tail -1 | awk '{print int($5)}')
    if [ "$usage_percent" -gt "$ALERT_THRESHOLD_PERCENT" ]; then
        alert_messages+=("WARNING: $backup_type backup disk usage at ${usage_percent}%")
        [ $exit_code -eq $OK ] && exit_code=$WARNING
    fi
    
    # Check for recent backups
    recent_backup=$(find "$backup_path" -type f -mtime -1 | head -1)
    if [ -z "$recent_backup" ]; then
        alert_messages+=("WARNING: No recent backups found in $backup_path")
        [ $exit_code -eq $OK ] && exit_code=$WARNING
    fi
    
    # Check backup count
    backup_count=$(find "$backup_path" -type f | wc -l)
    log_message "$backup_type: $backup_count files, ${usage_percent}% disk usage"
    
done < "$CONFIG_FILE"

# Check rotation service status
if ! systemctl is-active --quiet backup-rotation.timer; then
    alert_messages+=("CRITICAL: Backup rotation timer not running")
    exit_code=$CRITICAL
fi

# Check for recent rotation log entries
last_rotation=$(find /var/log/backup-rotation -name "rotation.log" -mtime -1 2>/dev/null | head -1)
if [ -z "$last_rotation" ]; then
    alert_messages+=("WARNING: No recent rotation logs found")
    [ $exit_code -eq $OK ] && exit_code=$WARNING
fi

# Output results for monitoring systems
if [ ${#alert_messages[@]} -eq 0 ]; then
    echo "OK: All backup systems healthy"
    log_message "Health check passed"
else
    for message in "${alert_messages[@]}"; do
        echo "$message"
        log_message "$message"
    done
fi

exit $exit_code

Make the health check script executable

Set permissions on the monitoring script and create a systemd timer to run it regularly.

sudo chmod 755 /opt/backup-rotation/scripts/backup-health-check.sh
sudo chown root:root /opt/backup-rotation/scripts/backup-health-check.sh

Create health check systemd service and timer

Set up automated health checking that runs every hour to catch issues early.

[Unit]
Description=Backup Health Check
After=network.target

[Service]
Type=oneshot
User=root
Group=root
ExecStart=/opt/backup-rotation/scripts/backup-health-check.sh
StandardOutput=journal
StandardError=journal
SyslogIdentifier=backup-health

[Unit]
Description=Hourly Backup Health Check
Requires=backup-health-check.service

[Timer]
OnCalendar=hourly
RandomizedDelaySec=300
Persistent=true

[Install]
WantedBy=timers.target

Enable the health check timer

Start the health monitoring system alongside the backup rotation.

sudo systemctl daemon-reload
sudo systemctl enable backup-health-check.timer
sudo systemctl start backup-health-check.timer

Create example backup directories for testing

Set up test backup directories that match your retention configuration to verify the system works.

sudo mkdir -p /var/backups/{database,files,logs,config}
sudo chown root:root /var/backups/{database,files,logs,config}
sudo chmod 755 /var/backups/{database,files,logs,config}

Create test backup files

Generate sample backup files with different ages to test the rotation logic.

# Create test files with different naming patterns and ages
sudo touch /var/backups/database/db-backup-daily-$(date +%Y%m%d).sql
sudo touch /var/backups/database/db-backup-weekly-$(date +%Y%W).sql
sudo touch /var/backups/files/files-backup-daily-$(date +%Y%m%d).tar.gz

# Create an old file to test deletion (10 days old)
sudo touch -d "10 days ago" /var/backups/database/old-daily-backup.sql
sudo touch -d "60 days ago" /var/backups/database/old-weekly-backup.sql

Create size-based rotation script

Add a script that removes backups when directories exceed size limits, useful for high-volume backup systems.

#!/bin/bash

set -euo pipefail

# Configuration - sizes in GB
MAX_SIZE_GB=50
CONFIG_FILE="/opt/backup-rotation/config/retention.conf"
LOG_FILE="/var/log/backup-rotation/size-cleanup.log"

log_message() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

log_message "Starting size-based cleanup"

while IFS=':' read -r backup_type backup_path daily_keep weekly_keep monthly_keep; do
    [[ "$backup_type" =~ ^#.*$ ]] && continue
    [[ -z "$backup_type" ]] && continue
    
    if [ ! -d "$backup_path" ]; then
        continue
    fi
    
    # Get current size in GB
    current_size_kb=$(du -s "$backup_path" | cut -f1)
    current_size_gb=$((current_size_kb / 1024 / 1024))
    
    log_message "$backup_type current size: ${current_size_gb}GB (limit: ${MAX_SIZE_GB}GB)"
    
    if [ "$current_size_gb" -gt "$MAX_SIZE_GB" ]; then
        log_message "Size limit exceeded for $backup_type, removing oldest files"
        
        # Remove oldest files until under limit
        while [ "$current_size_gb" -gt "$MAX_SIZE_GB" ]; do
            oldest_file=$(find "$backup_path" -type f -printf '%T@ %p\n' | sort -n | head -1 | cut -d' ' -f2-)
            
            if [ -z "$oldest_file" ]; then
                log_message "No more files to remove in $backup_path"
                break
            fi
            
            rm -f "$oldest_file"
            log_message "Removed oldest file: $oldest_file"
            
            # Recalculate size
            current_size_kb=$(du -s "$backup_path" | cut -f1)
            current_size_gb=$((current_size_kb / 1024 / 1024))
        done
    fi
    
done < "$CONFIG_FILE"

log_message "Size-based cleanup completed"

Set up email alerts for backup failures

Configure email notifications when backup rotation or health checks fail. This requires a configured mail system.

#!/bin/bash

set -euo pipefail

# Configuration
ALERT_EMAIL="admin@example.com"
SMTP_SERVER="localhost"
HOSTNAME=$(hostname -f)

# Function to send email alert
send_alert() {
    local subject="$1"
    local message="$2"
    
    if command -v mail >/dev/null; then
        echo -e "$message\n\nHost: $HOSTNAME\nTime: $(date)" | mail -s "$subject" "$ALERT_EMAIL"
    elif command -v sendmail >/dev/null; then
        {
            echo "To: $ALERT_EMAIL"
            echo "Subject: $subject"
            echo ""
            echo -e "$message\n\nHost: $HOSTNAME\nTime: $(date)"
        } | sendmail "$ALERT_EMAIL"
    else
        echo "No mail command available" >&2
        return 1
    fi
}

# Check if health check failed
if ! /opt/backup-rotation/scripts/backup-health-check.sh >/dev/null 2>&1; then
    health_output=$(/opt/backup-rotation/scripts/backup-health-check.sh 2>&1)
    send_alert "Backup Health Check Failed - $HOSTNAME" "$health_output"
fi

Integrate with monitoring systems

Create a script that outputs metrics in Prometheus format for integration with monitoring stacks.

#!/bin/bash

set -euo pipefail

CONFIG_FILE="/opt/backup-rotation/config/retention.conf"
METRICS_FILE="/var/lib/node_exporter/textfile_collector/backup_metrics.prom"

# Ensure directory exists
sudo mkdir -p /var/lib/node_exporter/textfile_collector

# Generate Prometheus metrics
{
    echo "# HELP backup_files_total Total number of backup files"
    echo "# TYPE backup_files_total gauge"
    
    echo "# HELP backup_disk_usage_bytes Disk usage of backup directories"
    echo "# TYPE backup_disk_usage_bytes gauge"
    
    echo "# HELP backup_rotation_last_run_timestamp Last successful rotation run"
    echo "# TYPE backup_rotation_last_run_timestamp gauge"
    
    while IFS=':' read -r backup_type backup_path daily_keep weekly_keep monthly_keep; do
        [[ "$backup_type" =~ ^#.*$ ]] && continue
        [[ -z "$backup_type" ]] && continue
        
        if [ -d "$backup_path" ]; then
            file_count=$(find "$backup_path" -type f | wc -l)
            disk_usage=$(du -sb "$backup_path" | cut -f1)
            
            echo "backup_files_total{type=\"$backup_type\",path=\"$backup_path\"} $file_count"
            echo "backup_disk_usage_bytes{type=\"$backup_type\",path=\"$backup_path\"} $disk_usage"
        fi
    done < "$CONFIG_FILE"
    
    # Add timestamp of last rotation
    if [ -f "/var/log/backup-rotation/rotation.log" ]; then
        last_run=$(stat -c %Y /var/log/backup-rotation/rotation.log)
        echo "backup_rotation_last_run_timestamp $last_run"
    fi
    
} > "$METRICS_FILE.tmp" && mv "$METRICS_FILE.tmp" "$METRICS_FILE"

Implement compression for old backups

Automatically compress backups older than a certain age to save storage space without losing data.

#!/bin/bash

set -euo pipefail

CONFIG_FILE="/opt/backup-rotation/config/retention.conf"
LOG_FILE="/var/log/backup-rotation/compression.log"
COMPRESS_AFTER_DAYS=7

log_message() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

log_message "Starting backup compression for files older than $COMPRESS_AFTER_DAYS days"

while IFS=':' read -r backup_type backup_path daily_keep weekly_keep monthly_keep; do
    [[ "$backup_type" =~ ^#.*$ ]] && continue
    [[ -z "$backup_type" ]] && continue
    
    if [ ! -d "$backup_path" ]; then
        continue
    fi
    
    log_message "Processing compression for $backup_type in $backup_path"
    
    # Find uncompressed files older than specified days
    find "$backup_path" -type f ! -name "*.gz" ! -na*