Configure MinIO backup and disaster recovery with automated snapshots and replication

Update system packages

Start by updating your package manager to ensure you have the latest security patches and package versions.

sudo apt update && sudo apt upgrade -y
sudo apt install -y curl wget gnupg2

sudo dnf update -y
sudo dnf install -y curl wget gnupg2

Install MinIO Client and backup tools

Install the MinIO client (mc) and additional tools needed for backup operations and encryption.

curl https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
chmod +x /usr/local/bin/mc
mc --version

sudo apt install -y rsync gzip tar cron

sudo dnf install -y rsync gzip tar cronie
sudo systemctl enable --now crond

Configure MinIO client connections

Set up connections to your primary MinIO instance and backup destinations. Replace the URLs and credentials with your actual MinIO server details.

mc alias set primary https://minio1.example.com:9000 minioadmin SecurePassword123!
mc alias set backup https://minio2.example.com:9000 minioadmin BackupPassword456!
mc alias ls

Create backup storage structure

Create dedicated buckets for storing backups with proper naming conventions and organization.

mc mb primary/backups
mc mb backup/disaster-recovery
mc mb backup/snapshots

Set versioning on backup buckets
mc version enable primary/backups
mc version enable backup/disaster-recovery
mc version enable backup/snapshots

Configure bucket replication

Set up cross-site replication between primary and backup MinIO instances for automatic data synchronization.

# Create replication configuration
mc replicate add primary/production backup/disaster-recovery --priority 1
mc replicate add primary/uploads backup/disaster-recovery --priority 2

Enable replication status tracking
mc replicate ls primary/production
mc replicate status primary/production

Create backup encryption keys

Generate encryption keys for securing backup data during transit and storage.

sudo mkdir -p /etc/minio/backup-keys
sudo chmod 700 /etc/minio/backup-keys

Generate encryption key
openssl rand -hex 32 | sudo tee /etc/minio/backup-keys/backup-encryption.key
sudo chmod 600 /etc/minio/backup-keys/backup-encryption.key
sudo chown root:root /etc/minio/backup-keys/backup-encryption.key

Create backup automation script

Create a comprehensive backup script that handles snapshots, encryption, and retention policies.

#!/bin/bash

MinIO Backup Script
set -euo pipefail

Configuration
PRIMARY_ALIAS="primary"
BACKUP_ALIAS="backup"
BACKUP_BUCKET="backups"
ENCRYPTION_KEY_FILE="/etc/minio/backup-keys/backup-encryption.key"
LOG_FILE="/var/log/minio-backup.log"
RETENTION_DAYS=30
DATE=$(date +%Y%m%d-%H%M%S)

Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

Function to backup bucket
backup_bucket() {
    local source_bucket=$1
    local backup_name="${source_bucket}-${DATE}"
    
    log "Starting backup of bucket: $source_bucket"
    
    # Create encrypted backup
    mc mirror --encrypt-key "${PRIMARY_ALIAS}/${source_bucket}=$(cat $ENCRYPTION_KEY_FILE)" \
        "${PRIMARY_ALIAS}/${source_bucket}" \
        "${BACKUP_ALIAS}/${BACKUP_BUCKET}/${backup_name}"
    
    # Verify backup integrity
    local source_count=$(mc ls --recursive "${PRIMARY_ALIAS}/${source_bucket}" | wc -l)
    local backup_count=$(mc ls --recursive "${BACKUP_ALIAS}/${BACKUP_BUCKET}/${backup_name}" | wc -l)
    
    if [ "$source_count" -eq "$backup_count" ]; then
        log "Backup verified successfully: $backup_name ($source_count files)"
    else
        log "ERROR: Backup verification failed for $backup_name"
        exit 1
    fi
}

Function to cleanup old backups
cleanup_old_backups() {
    log "Cleaning up backups older than $RETENTION_DAYS days"
    
    mc find "${BACKUP_ALIAS}/${BACKUP_BUCKET}" --name "*" --older-than "${RETENTION_DAYS}d" \
        --exec "mc rm --recursive --force {}" || true
}

Main backup process
main() {
    log "=== MinIO Backup Started ==="
    
    # List of buckets to backup (customize as needed)
    BUCKETS=("production" "uploads" "documents")
    
    for bucket in "${BUCKETS[@]}"; do
        if mc ls "${PRIMARY_ALIAS}/${bucket}" &>/dev/null; then
            backup_bucket "$bucket"
        else
            log "WARNING: Bucket $bucket not found, skipping"
        fi
    done
    
    # Cleanup old backups
    cleanup_old_backups
    
    log "=== MinIO Backup Completed ==="
}

Run main function
main "$@"

sudo chmod 755 /usr/local/bin/minio-backup.sh
sudo chown root:root /usr/local/bin/minio-backup.sh

Create disaster recovery script

Create a script to handle disaster recovery operations and failover procedures.

#!/bin/bash

MinIO Disaster Recovery Script
set -euo pipefail

Configuration
BACKUP_ALIAS="backup"
RECOVERY_ALIAS="recovery"
BACKUP_BUCKET="backups"
DR_BUCKET="disaster-recovery"
LOG_FILE="/var/log/minio-dr.log"

Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

Function to list available backups
list_backups() {
    log "Available backups:"
    mc ls "${BACKUP_ALIAS}/${BACKUP_BUCKET}/" | tee -a "$LOG_FILE"
}

Function to restore from backup
restore_from_backup() {
    local backup_name=$1
    local target_bucket=$2
    
    log "Starting restore of $backup_name to $target_bucket"
    
    # Verify backup exists
    if ! mc ls "${BACKUP_ALIAS}/${BACKUP_BUCKET}/${backup_name}" &>/dev/null; then
        log "ERROR: Backup $backup_name not found"
        exit 1
    fi
    
    # Create target bucket if it doesn't exist
    mc mb "${RECOVERY_ALIAS}/${target_bucket}" --ignore-existing
    
    # Restore data
    mc mirror "${BACKUP_ALIAS}/${BACKUP_BUCKET}/${backup_name}" \
        "${RECOVERY_ALIAS}/${target_bucket}"
    
    # Verify restoration
    local source_count=$(mc ls --recursive "${BACKUP_ALIAS}/${BACKUP_BUCKET}/${backup_name}" | wc -l)
    local target_count=$(mc ls --recursive "${RECOVERY_ALIAS}/${target_bucket}" | wc -l)
    
    if [ "$source_count" -eq "$target_count" ]; then
        log "Restore completed successfully: $target_bucket ($target_count files)"
    else
        log "ERROR: Restore verification failed"
        exit 1
    fi
}

Function to failover to backup site
failover_to_backup() {
    log "=== Starting Failover to Backup Site ==="
    
    # Check replication status
    mc replicate status primary/production || {
        log "Primary site unreachable, proceeding with backup site activation"
    }
    
    # Promote replica buckets to active
    mc ls "${BACKUP_ALIAS}/${DR_BUCKET}/" | tee -a "$LOG_FILE"
    
    log "Failover preparation completed. Manual verification required."
}

case "${1:-}" in
    "list")
        list_backups
        ;;
    "restore")
        if [ $# -ne 3 ]; then
            echo "Usage: $0 restore  "
            exit 1
        fi
        restore_from_backup "$2" "$3"
        ;;
    "failover")
        failover_to_backup
        ;;
    *)
        echo "Usage: $0 {list|restore|failover}"
        echo "  list                        - List available backups"
        echo "  restore     - Restore backup to bucket"
        echo "  failover                    - Failover to backup site"
        exit 1
        ;;
esac

sudo chmod 755 /usr/local/bin/minio-disaster-recovery.sh
sudo chown root:root /usr/local/bin/minio-disaster-recovery.sh

Configure automated backup scheduling

Set up cron jobs to run backups automatically on a regular schedule.

# Create log directory
sudo mkdir -p /var/log
sudo touch /var/log/minio-backup.log /var/log/minio-dr.log
sudo chmod 644 /var/log/minio-backup.log /var/log/minio-dr.log

sudo crontab -e

Add the following cron entries:

# MinIO backup schedule
Daily backups at 2 AM
0 2   * /usr/local/bin/minio-backup.sh >/dev/null 2>&1

Weekly verification at 3 AM on Sundays
0 3   0 /usr/local/bin/minio-disaster-recovery.sh list >/dev/null 2>&1

Monthly cleanup logs
0 4 1   find /var/log -name "minio-*.log" -size +100M -exec truncate -s 50M {} \;

Configure backup monitoring

Create monitoring scripts to track backup health and send alerts on failures.

#!/bin/bash

MinIO Backup Monitoring Script
set -euo pipefail

BACKUP_ALIAS="backup"
BACKUP_BUCKET="backups"
LOG_FILE="/var/log/minio-backup.log"
ALERT_EMAIL="admin@example.com"
MAX_BACKUP_AGE_HOURS=26  # Alert if no backup in 26 hours

Check if recent backups exist
check_recent_backups() {
    local cutoff_time=$(date -d "-${MAX_BACKUP_AGE_HOURS} hours" +%Y%m%d-%H%M%S)
    local recent_backups
    
    recent_backups=$(mc ls "${BACKUP_ALIAS}/${BACKUP_BUCKET}/" | \
        awk '{print $4}' | grep -E "[0-9]{8}-[0-9]{6}" | \
        sort -r | head -1)
    
    if [[ -z "$recent_backups" ]] || [[ "$recent_backups" < "$cutoff_time" ]]; then
        echo "CRITICAL: No recent backups found within $MAX_BACKUP_AGE_HOURS hours"
        return 1
    else
        echo "OK: Recent backup found: $recent_backups"
        return 0
    fi
}

Check backup log for errors
check_backup_logs() {
    if grep -q "ERROR" "$LOG_FILE" 2>/dev/null; then
        echo "WARNING: Errors found in backup log"
        tail -20 "$LOG_FILE" | grep "ERROR"
        return 1
    else
        echo "OK: No errors in recent backup logs"
        return 0
    fi
}

Check replication status
check_replication() {
    if ! mc replicate status primary/production >/dev/null 2>&1; then
        echo "WARNING: Replication status check failed"
        return 1
    else
        echo "OK: Replication status healthy"
        return 0
    fi
}

Main monitoring function
main() {
    local status=0
    local report="MinIO Backup Health Report $(date)\n\n"
    
    # Run checks
    if ! check_recent_backups; then
        status=1
    fi
    
    if ! check_backup_logs; then
        status=1
    fi
    
    if ! check_replication; then
        status=1
    fi
    
    # Send alert if issues found
    if [ $status -ne 0 ]; then
        echo "Issues detected in MinIO backup system" | \
            mail -s "MinIO Backup Alert" "$ALERT_EMAIL" 2>/dev/null || \
            logger "MinIO Backup Alert: Issues detected"
    fi
    
    exit $status
}

main

sudo chmod 755 /usr/local/bin/minio-backup-monitor.sh
sudo chown root:root /usr/local/bin/minio-backup-monitor.sh

Test backup and recovery procedures

Perform initial tests to verify your backup and disaster recovery setup works correctly.

# Test backup script
sudo /usr/local/bin/minio-backup.sh

Check backup was created
mc ls backup/backups/

Test monitoring script
sudo /usr/local/bin/minio-backup-monitor.sh

List available backups for recovery testing
/usr/local/bin/minio-disaster-recovery.sh list