Implement Redis backup automation with RDB and AOF persistence

Install Redis and backup tools

Install Redis server and required backup utilities for automation and monitoring.

sudo apt update
sudo apt install -y redis-server rsync gzip curl jq
sudo systemctl enable redis-server

sudo dnf update -y
sudo dnf install -y redis rsync gzip curl jq
sudo systemctl enable redis

Configure RDB snapshot automation

Set up automated RDB snapshots with optimized intervals and compression settings for efficient storage.

# RDB Snapshot Configuration
save 900 1      # Save if at least 1 key changed in 900 seconds
save 300 10     # Save if at least 10 keys changed in 300 seconds
save 60 10000   # Save if at least 10000 keys changed in 60 seconds

RDB file settings
dbfilename dump.rdb
dir /var/lib/redis
rdbcompression yes
rdbchecksum yes

Stop writes on RDB save errors
stop-writes-on-bgsave-error yes

RDB save process optimization
rdb-save-incremental-fsync yes

Enable AOF persistence logging

Configure Append-Only File persistence for granular data recovery with automatic rewriting optimization.

# AOF Configuration
appendonly yes
appendfilename "appendonly.aof"

AOF sync policies
appendfsync everysec    # Sync every second (balanced performance/safety)

AOF rewrite configuration
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb

AOF loading behavior
aof-load-truncated yes
aof-use-rdb-preamble yes

Disable fsync during rewrite
no-appendfsync-on-rewrite no

Create backup directory structure

Set up organized backup directories with proper permissions for automated backup storage.

sudo mkdir -p /var/backups/redis/{daily,weekly,monthly,logs}
sudo mkdir -p /var/backups/redis/rdb /var/backups/redis/aof
sudo chown -R redis:redis /var/backups/redis
sudo chmod -R 750 /var/backups/redis

Create RDB backup automation script

Develop a comprehensive RDB backup script with compression, rotation, and error handling.

#!/bin/bash

Redis RDB Backup Script
set -euo pipefail

Configuration
REDIS_CLI="/usr/bin/redis-cli"
BACKUP_DIR="/var/backups/redis"
RDB_SOURCE="/var/lib/redis/dump.rdb"
LOG_FILE="$BACKUP_DIR/logs/rdb-backup.log"
RETENTION_DAYS=30
DATE=$(date +%Y%m%d_%H%M%S)
HOSTNAME=$(hostname)

Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') [RDB-BACKUP] $1" | tee -a "$LOG_FILE"
}

Check Redis connectivity
check_redis() {
    if ! $REDIS_CLI ping > /dev/null 2>&1; then
        log "ERROR: Redis server is not responding"
        exit 1
    fi
    log "Redis connectivity verified"
}

Trigger RDB save
trigger_save() {
    log "Triggering RDB save..."
    $REDIS_CLI BGSAVE
    
    # Wait for save to complete
    while [ "$($REDIS_CLI LASTSAVE)" = "$($REDIS_CLI LASTSAVE)" ]; do
        sleep 1
    done
    log "RDB save completed"
}

Create backup with compression
create_backup() {
    local backup_file="$BACKUP_DIR/rdb/redis-rdb-$HOSTNAME-$DATE.rdb.gz"
    
    if [ ! -f "$RDB_SOURCE" ]; then
        log "ERROR: RDB file not found at $RDB_SOURCE"
        exit 1
    fi
    
    log "Creating compressed backup: $backup_file"
    gzip -c "$RDB_SOURCE" > "$backup_file"
    
    # Verify backup integrity
    if gzip -t "$backup_file"; then
        log "Backup created successfully: $backup_file"
        echo "$backup_file" > "$BACKUP_DIR/latest_rdb_backup.txt"
    else
        log "ERROR: Backup verification failed"
        rm -f "$backup_file"
        exit 1
    fi
}

Cleanup old backups
cleanup_old_backups() {
    log "Cleaning up RDB backups older than $RETENTION_DAYS days"
    find "$BACKUP_DIR/rdb" -name "*.rdb.gz" -mtime +$RETENTION_DAYS -delete
    log "Cleanup completed"
}

Generate backup report
generate_report() {
    local backup_count=$(find "$BACKUP_DIR/rdb" -name "*.rdb.gz" | wc -l)
    local total_size=$(du -sh "$BACKUP_DIR/rdb" | cut -f1)
    
    log "Backup Summary: $backup_count files, total size: $total_size"
}

Main execution
main() {
    log "Starting Redis RDB backup process"
    check_redis
    trigger_save
    create_backup
    cleanup_old_backups
    generate_report
    log "RDB backup process completed successfully"
}

main "$@"

Create AOF backup automation script

Implement AOF backup script with rewrite optimization and incremental backup support.

#!/bin/bash

Redis AOF Backup Script
set -euo pipefail

Configuration
REDIS_CLI="/usr/bin/redis-cli"
BACKUP_DIR="/var/backups/redis"
AOF_SOURCE="/var/lib/redis/appendonly.aof"
LOG_FILE="$BACKUP_DIR/logs/aof-backup.log"
RETENTION_DAYS=14
DATE=$(date +%Y%m%d_%H%M%S)
HOSTNAME=$(hostname)

Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') [AOF-BACKUP] $1" | tee -a "$LOG_FILE"
}

Check Redis connectivity
check_redis() {
    if ! $REDIS_CLI ping > /dev/null 2>&1; then
        log "ERROR: Redis server is not responding"
        exit 1
    fi
    log "Redis connectivity verified"
}

Trigger AOF rewrite
trigger_aof_rewrite() {
    log "Triggering AOF rewrite for optimization..."
    $REDIS_CLI BGREWRITEAOF
    
    # Wait for rewrite to complete
    while [ "$($REDIS_CLI INFO persistence | grep aof_rewrite_in_progress:1)" ]; do
        sleep 2
        log "AOF rewrite in progress..."
    done
    log "AOF rewrite completed"
}

Create AOF backup
create_aof_backup() {
    local backup_file="$BACKUP_DIR/aof/redis-aof-$HOSTNAME-$DATE.aof.gz"
    
    if [ ! -f "$AOF_SOURCE" ]; then
        log "ERROR: AOF file not found at $AOF_SOURCE"
        exit 1
    fi
    
    # Create backup during low activity
    log "Creating compressed AOF backup: $backup_file"
    gzip -c "$AOF_SOURCE" > "$backup_file"
    
    # Verify backup integrity
    if gzip -t "$backup_file"; then
        log "AOF backup created successfully: $backup_file"
        echo "$backup_file" > "$BACKUP_DIR/latest_aof_backup.txt"
    else
        log "ERROR: AOF backup verification failed"
        rm -f "$backup_file"
        exit 1
    fi
}

Cleanup old AOF backups
cleanup_old_backups() {
    log "Cleaning up AOF backups older than $RETENTION_DAYS days"
    find "$BACKUP_DIR/aof" -name "*.aof.gz" -mtime +$RETENTION_DAYS -delete
    log "AOF cleanup completed"
}

Generate AOF report
generate_report() {
    local backup_count=$(find "$BACKUP_DIR/aof" -name "*.aof.gz" | wc -l)
    local total_size=$(du -sh "$BACKUP_DIR/aof" | cut -f1)
    local aof_size=$($REDIS_CLI INFO persistence | grep aof_current_size | cut -d: -f2)
    
    log "AOF Backup Summary: $backup_count files, total size: $total_size, current AOF: ${aof_size} bytes"
}

Main execution
main() {
    log "Starting Redis AOF backup process"
    check_redis
    trigger_aof_rewrite
    create_aof_backup
    cleanup_old_backups
    generate_report
    log "AOF backup process completed successfully"
}

main "$@"

Create backup verification script

Implement comprehensive backup validation with integrity checks and restoration testing.

#!/bin/bash

Redis Backup Verification Script
set -euo pipefail

Configuration
BACKUP_DIR="/var/backups/redis"
TEST_DIR="/tmp/redis-backup-test"
LOG_FILE="$BACKUP_DIR/logs/backup-verify.log"
REDIS_CLI="/usr/bin/redis-cli"
TEST_PORT=6380

Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') [VERIFY] $1" | tee -a "$LOG_FILE"
}

Cleanup test environment
cleanup_test_env() {
    log "Cleaning up test environment"
    pkill -f "redis-server.*$TEST_PORT" 2>/dev/null || true
    rm -rf "$TEST_DIR"
}

Setup test environment
setup_test_env() {
    log "Setting up test environment"
    mkdir -p "$TEST_DIR"
    cleanup_test_env
}

Verify RDB backup
verify_rdb_backup() {
    local latest_rdb=$(cat "$BACKUP_DIR/latest_rdb_backup.txt" 2>/dev/null || echo "")
    
    if [ ! -f "$latest_rdb" ]; then
        log "ERROR: Latest RDB backup not found"
        return 1
    fi
    
    log "Verifying RDB backup: $latest_rdb"
    
    # Extract backup to test directory
    gzip -dc "$latest_rdb" > "$TEST_DIR/test.rdb"
    
    # Start test Redis instance
    redis-server --port $TEST_PORT --dir "$TEST_DIR" --dbfilename test.rdb --daemonize yes --logfile "$TEST_DIR/redis-test.log"
    sleep 2
    
    # Test data integrity
    local key_count=$(redis-cli -p $TEST_PORT DBSIZE)
    log "RDB backup verification: $key_count keys restored successfully"
    
    # Stop test instance
    redis-cli -p $TEST_PORT SHUTDOWN NOSAVE 2>/dev/null || true
    
    return 0
}

Verify AOF backup
verify_aof_backup() {
    local latest_aof=$(cat "$BACKUP_DIR/latest_aof_backup.txt" 2>/dev/null || echo "")
    
    if [ ! -f "$latest_aof" ]; then
        log "ERROR: Latest AOF backup not found"
        return 1
    fi
    
    log "Verifying AOF backup: $latest_aof"
    
    # Extract AOF backup
    gzip -dc "$latest_aof" > "$TEST_DIR/test.aof"
    
    # Verify AOF syntax
    redis-check-aof "$TEST_DIR/test.aof" > "$TEST_DIR/aof-check.log" 2>&1
    
    if [ $? -eq 0 ]; then
        log "AOF backup verification: Syntax and structure are valid"
        return 0
    else
        log "ERROR: AOF backup verification failed"
        cat "$TEST_DIR/aof-check.log" >> "$LOG_FILE"
        return 1
    fi
}

Generate verification report
generate_verification_report() {
    local rdb_count=$(find "$BACKUP_DIR/rdb" -name "*.rdb.gz" -mtime -1 | wc -l)
    local aof_count=$(find "$BACKUP_DIR/aof" -name "*.aof.gz" -mtime -1 | wc -l)
    local total_size=$(du -sh "$BACKUP_DIR" | cut -f1)
    
    log "Verification Summary:"
    log "  - RDB backups (24h): $rdb_count"
    log "  - AOF backups (24h): $aof_count"
    log "  - Total backup size: $total_size"
    
    # Create status file for monitoring
    cat > "$BACKUP_DIR/backup-status.json" << EOF
{
  "timestamp": "$(date -Iseconds)",
  "rdb_backups_24h": $rdb_count,
  "aof_backups_24h": $aof_count,
  "total_size": "$total_size",
  "last_verification": "$(date -Iseconds)",
  "status": "healthy"
}
EOF
}

Main execution
main() {
    log "Starting backup verification process"
    
    setup_test_env
    
    local rdb_status=0
    local aof_status=0
    
    verify_rdb_backup || rdb_status=1
    verify_aof_backup || aof_status=1
    
    cleanup_test_env
    generate_verification_report
    
    if [ $rdb_status -eq 0 ] && [ $aof_status -eq 0 ]; then
        log "All backup verifications passed"
        exit 0
    else
        log "Backup verification failed - check logs"
        exit 1
    fi
}

main "$@"

Set script permissions and ownership

Configure proper permissions for backup scripts to ensure security and execution access.

sudo chmod +x /usr/local/bin/redis-rdb-backup.sh
sudo chmod +x /usr/local/bin/redis-aof-backup.sh
sudo chmod +x /usr/local/bin/redis-backup-verify.sh
sudo chown root:redis /usr/local/bin/redis-*-backup.sh
sudo chown root:redis /usr/local/bin/redis-backup-verify.sh

Configure automated scheduling with systemd timers

Create systemd timer units for reliable backup scheduling with proper error handling and logging.

[Unit]
Description=Redis RDB Backup Service
After=redis-server.service
Requires=redis-server.service

[Service]
Type=oneshot
User=redis
Group=redis
ExecStart=/usr/local/bin/redis-rdb-backup.sh
StandardOutput=journal
StandardError=journal

Create RDB backup timer

Configure daily RDB backup scheduling with systemd timer for consistent execution.

[Unit]
Description=Daily Redis RDB Backup
Requires=redis-rdb-backup.service

[Timer]
OnCalendar=daily
RandomizedDelaySec=300
Persistent=true

[Install]
WantedBy=timers.target

Create AOF backup service and timer

Set up AOF backup scheduling for more frequent incremental backups.

[Unit]
Description=Redis AOF Backup Service
After=redis-server.service
Requires=redis-server.service

[Service]
Type=oneshot
User=redis
Group=redis
ExecStart=/usr/local/bin/redis-aof-backup.sh
StandardOutput=journal
StandardError=journal

[Unit]
Description=Redis AOF Backup Every 6 Hours
Requires=redis-aof-backup.service

[Timer]
OnCalendar=0/6:00:00
RandomizedDelaySec=180
Persistent=true

[Install]
WantedBy=timers.target

Create verification service and timer

Schedule automated backup verification to ensure backup integrity and recoverability.

[Unit]
Description=Redis Backup Verification Service
After=redis-server.service

[Service]
Type=oneshot
User=redis
Group=redis
ExecStart=/usr/local/bin/redis-backup-verify.sh
StandardOutput=journal
StandardError=journal

[Unit]
Description=Redis Backup Verification Every 12 Hours
Requires=redis-backup-verify.service

[Timer]
OnCalendar=0/12:00:00
RandomizedDelaySec=600
Persistent=true

[Install]
WantedBy=timers.target

Enable and start backup timers

Activate all backup automation timers and verify their scheduling status.

sudo systemctl daemon-reload
sudo systemctl enable --now redis-rdb-backup.timer
sudo systemctl enable --now redis-aof-backup.timer
sudo systemctl enable --now redis-backup-verify.timer

Restart Redis with new configuration

Apply the persistence configuration changes by restarting the Redis service.

sudo systemctl restart redis-server
sudo systemctl status redis-server

sudo systemctl restart redis
sudo systemctl status redis

Create monitoring and alerting script

Implement monitoring script with email alerts and integration with Prometheus monitoring.

#!/bin/bash

Redis Backup Monitoring Script
set -euo pipefail

Configuration
BACKUP_DIR="/var/backups/redis"
ALERT_EMAIL="admin@example.com"
STATUS_FILE="$BACKUP_DIR/backup-status.json"
LOG_FILE="$BACKUP_DIR/logs/monitor.log"
MAX_BACKUP_AGE_HOURS=25
MIN_BACKUP_SIZE_KB=1024

Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') [MONITOR] $1" | tee -a "$LOG_FILE"
}

Send alert email
send_alert() {
    local subject="$1"
    local message="$2"
    
    echo "$message" | mail -s "Redis Backup Alert: $subject" "$ALERT_EMAIL" 2>/dev/null || \
        log "Failed to send email alert: $subject"
}

Check backup freshness
check_backup_freshness() {
    local backup_type="$1"
    local backup_dir="$BACKUP_DIR/$backup_type"
    local latest_backup=$(find "$backup_dir" -name "*.$backup_type.gz" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-)
    
    if [ -z "$latest_backup" ]; then
        log "ERROR: No $backup_type backups found"
        send_alert "Missing $backup_type Backups" "No $backup_type backup files found in $backup_dir"
        return 1
    fi
    
    local backup_age_hours=$(( ($(date +%s) - $(stat -c %Y "$latest_backup")) / 3600 ))
    local backup_size_kb=$(du -k "$latest_backup" | cut -f1)
    
    if [ $backup_age_hours -gt $MAX_BACKUP_AGE_HOURS ]; then
        log "ERROR: $backup_type backup is too old ($backup_age_hours hours)"
        send_alert "$backup_type Backup Too Old" "Latest $backup_type backup is $backup_age_hours hours old. File: $latest_backup"
        return 1
    fi
    
    if [ $backup_size_kb -lt $MIN_BACKUP_SIZE_KB ]; then
        log "ERROR: $backup_type backup is too small ($backup_size_kb KB)"
        send_alert "$backup_type Backup Too Small" "Latest $backup_type backup is only $backup_size_kb KB. File: $latest_backup"
        return 1
    fi
    
    log "$backup_type backup check passed: $backup_age_hours hours old, $backup_size_kb KB"
    return 0
}

Check backup verification status
check_verification_status() {
    if [ ! -f "$STATUS_FILE" ]; then
        log "ERROR: Backup status file not found"
        send_alert "Backup Verification Missing" "Backup verification status file not found: $STATUS_FILE"
        return 1
    fi
    
    local last_verify=$(jq -r '.last_verification' "$STATUS_FILE" 2>/dev/null || echo "null")
    
    if [ "$last_verify" = "null" ]; then
        log "ERROR: Invalid backup status file"
        send_alert "Backup Status Invalid" "Cannot parse backup status file: $STATUS_FILE"
        return 1
    fi
    
    local verify_age_hours=$(( ($(date +%s) - $(date -d "$last_verify" +%s)) / 3600 ))
    
    if [ $verify_age_hours -gt 13 ]; then
        log "ERROR: Backup verification is too old ($verify_age_hours hours)"
        send_alert "Backup Verification Overdue" "Last backup verification was $verify_age_hours hours ago"
        return 1
    fi
    
    log "Backup verification check passed: $verify_age_hours hours ago"
    return 0
}

Generate metrics for Prometheus
generate_metrics() {
    local metrics_file="$BACKUP_DIR/metrics.prom"
    local rdb_count=$(find "$BACKUP_DIR/rdb" -name "*.rdb.gz" | wc -l)
    local aof_count=$(find "$BACKUP_DIR/aof" -name "*.aof.gz" | wc -l)
    local total_size_bytes=$(du -sb "$BACKUP_DIR" | cut -f1)
    local latest_rdb_age=0
    local latest_aof_age=0
    
    # Calculate backup ages
    if [ -f "$BACKUP_DIR/latest_rdb_backup.txt" ]; then
        local latest_rdb=$(cat "$BACKUP_DIR/latest_rdb_backup.txt")
        if [ -f "$latest_rdb" ]; then
            latest_rdb_age=$(( $(date +%s) - $(stat -c %Y "$latest_rdb") ))
        fi
    fi
    
    if [ -f "$BACKUP_DIR/latest_aof_backup.txt" ]; then
        local latest_aof=$(cat "$BACKUP_DIR/latest_aof_backup.txt")
        if [ -f "$latest_aof" ]; then
            latest_aof_age=$(( $(date +%s) - $(stat -c %Y "$latest_aof") ))
        fi
    fi
    
    cat > "$metrics_file" << EOF
HELP redis_backup_rdb_count Number of RDB backup files
TYPE redis_backup_rdb_count gauge
redis_backup_rdb_count $rdb_count

HELP redis_backup_aof_count Number of AOF backup files
TYPE redis_backup_aof_count gauge
redis_backup_aof_count $aof_count

HELP redis_backup_total_size_bytes Total backup directory size in bytes
TYPE redis_backup_total_size_bytes gauge
redis_backup_total_size_bytes $total_size_bytes

HELP redis_backup_latest_rdb_age_seconds Age of latest RDB backup in seconds
TYPE redis_backup_latest_rdb_age_seconds gauge
redis_backup_latest_rdb_age_seconds $latest_rdb_age

HELP redis_backup_latest_aof_age_seconds Age of latest AOF backup in seconds
TYPE redis_backup_latest_aof_age_seconds gauge
redis_backup_latest_aof_age_seconds $latest_aof_age
EOF
    
    log "Metrics updated: $metrics_file"
}

Main monitoring function
main() {
    log "Starting backup monitoring check"
    
    local rdb_status=0
    local aof_status=0
    local verify_status=0
    
    check_backup_freshness "rdb" || rdb_status=1
    check_backup_freshness "aof" || aof_status=1
    check_verification_status || verify_status=1
    
    generate_metrics
    
    if [ $rdb_status -eq 0 ] && [ $aof_status -eq 0 ] && [ $verify_status -eq 0 ]; then
        log "All backup monitoring checks passed"
        exit 0
    else
        log "Backup monitoring detected issues - alerts sent"
        exit 1
    fi
}

main "$@"

Configure monitoring timer and permissions

Set up monitoring automation with proper script permissions and scheduled execution.

sudo chmod +x /usr/local/bin/redis-backup-monitor.sh
sudo chown root:redis /usr/local/bin/redis-backup-monitor.sh

[Unit]
Description=Redis Backup Monitoring Service

[Service]
Type=oneshot
User=redis
Group=redis
ExecStart=/usr/local/bin/redis-backup-monitor.sh
StandardOutput=journal
StandardError=journal

[Unit]
Description=Redis Backup Monitoring Every 4 Hours
Requires=redis-backup-monitor.service

[Timer]
OnCalendar=0/4:00:00
Persistent=true

[Install]
WantedBy=timers.target

sudo systemctl daemon-reload
sudo systemctl enable --now redis-backup-monitor.timer