Configure backup compression and deduplication with BorgBackup and rsync for optimal storage efficiency

Install BorgBackup and rsync

Install the backup tools and dependencies for compression algorithms.

sudo apt update
sudo apt install -y borgbackup rsync lz4 zstd python3-llfuse

sudo dnf update -y
sudo dnf install -y borgbackup rsync lz4 zstd python3-llfuse

Create backup directories and user

Set up dedicated backup storage locations and service user with proper permissions.

sudo mkdir -p /var/backups/borg /var/backups/rsync /var/log/backups
sudo useradd --system --home /var/backups --shell /bin/bash backup
sudo chown -R backup:backup /var/backups
sudo chmod 750 /var/backups /var/log/backups

Initialize BorgBackup repository

Create an encrypted Borg repository with optimal settings for deduplication.

sudo -u backup borg init --encryption=repokey-blake2 /var/backups/borg/main
sudo -u backup borg key export /var/backups/borg/main /var/backups/borg-key-backup.txt

Configure BorgBackup wrapper script

Create an automated backup script with compression and deduplication optimized for your data types.

#!/bin/bash

set -euo pipefail

# Configuration
REPO="/var/backups/borg/main"
BACKUP_DIRS="/etc /home /var/www /opt"
LOGFILE="/var/log/backups/borg-$(date +%Y%m%d-%H%M%S).log"
RETENTION_DAILY=7
RETENTION_WEEKLY=4
RETENTION_MONTHLY=6

# Export for Borg
export BORG_REPO="$REPO"
export BORG_PASSPHRASE="$(cat /var/backups/.borg-passphrase)"
export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes

# Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOGFILE"
}

# Start backup
log "Starting BorgBackup process"

# Create backup with zstd compression (good balance of speed/ratio)
log "Creating archive with compression"
borg create \
    --verbose \
    --filter AME \
    --list \
    --stats \
    --show-rc \
    --compression zstd,3 \
    --exclude-caches \
    --exclude '/home/*/.cache' \
    --exclude '/var/cache' \
    --exclude '/var/tmp' \
    --exclude '/tmp' \
    --exclude '/var/log/backups' \
    "::backup-{now}" \
    $BACKUP_DIRS 2>&1 | tee -a "$LOGFILE"

# Prune old backups
log "Pruning old archives"
borg prune \
    --verbose \
    --list \
    --prefix backup- \
    --show-rc \
    --keep-daily $RETENTION_DAILY \
    --keep-weekly $RETENTION_WEEKLY \
    --keep-monthly $RETENTION_MONTHLY 2>&1 | tee -a "$LOGFILE"

# Compact repository to reclaim space
log "Compacting repository"
borg compact --verbose 2>&1 | tee -a "$LOGFILE"

# Check repository integrity (weekly)
if [ "$(date +%u)" -eq 1 ]; then
    log "Running repository check (weekly)"
    borg check --verbose --verify-data --show-rc 2>&1 | tee -a "$LOGFILE"
fi

log "BorgBackup completed successfully"

# Send metrics to monitoring system (optional)
if command -v curl &> /dev/null && [ -n "${PROMETHEUS_GATEWAY:-}" ]; then
    REPO_SIZE=$(borg info --json | jq -r '.repository.total_size')
    curl -X POST "$PROMETHEUS_GATEWAY/metrics/job/borgbackup/instance/$(hostname)" \
         --data-binary "borgbackup_repo_size_bytes $REPO_SIZE"
fi

Set up BorgBackup passphrase

Create a secure passphrase file with restricted permissions.

sudo -u backup openssl rand -base64 32 > /var/backups/.borg-passphrase
sudo chmod 400 /var/backups/.borg-passphrase
sudo chown backup:backup /var/backups/.borg-passphrase

Make backup script executable

Set proper permissions on the BorgBackup wrapper script.

sudo chmod +x /usr/local/bin/borg-backup.sh
sudo chown backup:backup /usr/local/bin/borg-backup.sh

Configure rsync backup script

Create an rsync script optimized for incremental backups with compression and deduplication through hard links.

#!/bin/bash

set -euo pipefail

# Configuration
SOURCE_DIRS=("/var/www" "/home" "/opt")
BACKUP_ROOT="/var/backups/rsync"
DATE=$(date +%Y%m%d-%H%M%S)
CURRENT="$BACKUP_ROOT/current"
INCREMENT="$BACKUP_ROOT/increment-$DATE"
LOGFILE="/var/log/backups/rsync-$(date +%Y%m%d-%H%M%S).log"
RETENTION_DAYS=30

# Logging function
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOGFILE"
}

log "Starting rsync incremental backup"

# Create backup directories
mkdir -p "$BACKUP_ROOT" "$INCREMENT"

# Rsync with hard links for deduplication
for source in "${SOURCE_DIRS[@]}"; do
    if [ -d "$source" ]; then
        log "Backing up $source"
        
        # Create target directory structure
        target="$INCREMENT$(dirname "$source")"
        mkdir -p "$target"
        
        # Rsync with hard link deduplication
        rsync -avz \
            --compress \
            --compress-level=6 \
            --delete \
            --delete-excluded \
            --hard-links \
            --link-dest="$CURRENT$(dirname "$source")" \
            --exclude='*.tmp' \
            --exclude='*.log' \
            --exclude='.cache/' \
            --exclude='lost+found/' \
            --stats \
            "$source/" "$INCREMENT$source/" 2>&1 | tee -a "$LOGFILE"
    else
        log "Warning: Source directory $source does not exist, skipping"
    fi
done

# Update current symlink
log "Updating current backup symlink"
rm -f "$CURRENT"
ln -sf "increment-$DATE" "$CURRENT"

# Clean up old backups
log "Cleaning up backups older than $RETENTION_DAYS days"
find "$BACKUP_ROOT" -maxdepth 1 -name "increment-*" -type d -mtime +$RETENTION_DAYS -exec rm -rf {} \;

# Calculate storage efficiency
TOTAL_SIZE=$(du -sb "$BACKUP_ROOT" | cut -f1)
log "Total backup storage usage: $(numfmt --to=iec-i --suffix=B "$TOTAL_SIZE")"

log "Rsync backup completed successfully"

Set rsync script permissions

Make the rsync backup script executable with proper ownership.

sudo chmod +x /usr/local/bin/rsync-backup.sh
sudo chown backup:backup /usr/local/bin/rsync-backup.sh

Create systemd timer for BorgBackup

Set up automated scheduling for BorgBackup using systemd timers.

[Unit]
Description=BorgBackup automated backup
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
User=backup
Group=backup
ExecStart=/usr/local/bin/borg-backup.sh
StandardOutput=journal
StandardError=journal

# Security hardening
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/var/backups /var/log/backups
ProtectKernelTunables=yes
ProtectControlGroups=yes
RestrictRealtime=yes
MemoryDenyWriteExecute=yes
SystemCallFilter=@system-service
SystemCallErrorNumber=EPERM

Create systemd timer for rsync

Configure systemd timer for frequent rsync incremental backups.

[Unit]
Description=Rsync incremental backup
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
User=backup
Group=backup
ExecStart=/usr/local/bin/rsync-backup.sh
StandardOutput=journal
StandardError=journal

# Security hardening
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/var/backups /var/log/backups
ProtectKernelTunables=yes
ProtectControlGroups=yes
RestrictRealtime=yes
MemoryDenyWriteExecute=yes
SystemCallFilter=@system-service
SystemCallErrorNumber=EPERM

Configure backup timers

Create timer configurations for regular backup execution.

[Unit]
Description=Run BorgBackup daily
Requires=borgbackup.service

[Timer]
# Run daily at 2 AM
OnCalendar=daily
Persistent=true
RandomizedDelaySec=3600

[Install]
WantedBy=timers.target

[Unit]
Description=Run rsync backup every 4 hours
Requires=rsyncbackup.service

[Timer]
# Run every 4 hours
OnCalendar=*-*-* 0,4,8,12,16,20:00:00
Persistent=true
RandomizedDelaySec=900

[Install]
WantedBy=timers.target

Enable and start backup timers

Activate the systemd timers for automated backup execution.

sudo systemctl daemon-reload
sudo systemctl enable --now borgbackup.timer
sudo systemctl enable --now rsyncbackup.timer

Configure backup monitoring script

Create a monitoring script to track backup health and storage efficiency.

#!/bin/bash

set -euo pipefail

# Configuration
BORG_REPO="/var/backups/borg/main"
RSYNC_ROOT="/var/backups/rsync"
METRICS_FILE="/var/log/backups/metrics.txt"
ALERT_EMAIL="admin@example.com"
MAX_AGE_HOURS=26  # Alert if backup older than 26 hours

# Export for Borg
export BORG_REPO="$BORG_REPO"
export BORG_PASSPHRASE="$(cat /var/backups/.borg-passphrase)"

# Check BorgBackup status
check_borg() {
    echo "=== BorgBackup Status ===" > "$METRICS_FILE"
    
    if borg list --short --last 1 > /dev/null 2>&1; then
        LAST_BACKUP=$(borg list --short --last 1 | head -1 | cut -d' ' -f1)
        BACKUP_TIME=$(echo "$LAST_BACKUP" | sed 's/backup-//g')
        
        # Get repository info
        REPO_INFO=$(borg info --json 2>/dev/null)
        TOTAL_SIZE=$(echo "$REPO_INFO" | jq -r '.repository.total_size')
        UNIQUE_SIZE=$(echo "$REPO_INFO" | jq -r '.repository.unique_size')
        COMPRESSED_SIZE=$(echo "$REPO_INFO" | jq -r '.repository.compressed_size')
        
        # Calculate deduplication ratio
        if [ "$TOTAL_SIZE" -gt 0 ]; then
            DEDUP_RATIO=$(echo "scale=2; (1 - $UNIQUE_SIZE / $TOTAL_SIZE) * 100" | bc)
        else
            DEDUP_RATIO=0
        fi
        
        # Calculate compression ratio
        if [ "$UNIQUE_SIZE" -gt 0 ]; then
            COMP_RATIO=$(echo "scale=2; (1 - $COMPRESSED_SIZE / $UNIQUE_SIZE) * 100" | bc)
        else
            COMP_RATIO=0
        fi
        
        echo "Last backup: $LAST_BACKUP" >> "$METRICS_FILE"
        echo "Total size: $(numfmt --to=iec-i --suffix=B "$TOTAL_SIZE")" >> "$METRICS_FILE"
        echo "Unique size: $(numfmt --to=iec-i --suffix=B "$UNIQUE_SIZE")" >> "$METRICS_FILE"
        echo "Compressed size: $(numfmt --to=iec-i --suffix=B "$COMPRESSED_SIZE")" >> "$METRICS_FILE"
        echo "Deduplication ratio: ${DEDUP_RATIO}%" >> "$METRICS_FILE"
        echo "Compression ratio: ${COMP_RATIO}%" >> "$METRICS_FILE"
        
        # Check backup age
        BACKUP_TIMESTAMP=$(date -d "${BACKUP_TIME:0:4}-${BACKUP_TIME:4:2}-${BACKUP_TIME:6:2} ${BACKUP_TIME:9:2}:${BACKUP_TIME:11:2}:${BACKUP_TIME:13:2}" +%s)
        CURRENT_TIMESTAMP=$(date +%s)
        AGE_HOURS=$(( (CURRENT_TIMESTAMP - BACKUP_TIMESTAMP) / 3600 ))
        
        if [ "$AGE_HOURS" -gt "$MAX_AGE_HOURS" ]; then
            echo "WARNING: Last backup is $AGE_HOURS hours old" >> "$METRICS_FILE"
            echo "Backup alert: Last BorgBackup is $AGE_HOURS hours old" | mail -s "Backup Alert" "$ALERT_EMAIL" 2>/dev/null || true
        fi
        
    else
        echo "ERROR: Cannot access BorgBackup repository" >> "$METRICS_FILE"
        echo "Backup error: Cannot access BorgBackup repository" | mail -s "Backup Error" "$ALERT_EMAIL" 2>/dev/null || true
    fi
}

# Check rsync status
check_rsync() {
    echo "" >> "$METRICS_FILE"
    echo "=== Rsync Backup Status ===" >> "$METRICS_FILE"
    
    if [ -L "$RSYNC_ROOT/current" ]; then
        CURRENT_BACKUP=$(readlink "$RSYNC_ROOT/current")
        BACKUP_SIZE=$(du -sh "$RSYNC_ROOT/$CURRENT_BACKUP" 2>/dev/null | cut -f1)
        TOTAL_BACKUPS=$(ls -1d "$RSYNC_ROOT"/increment-* 2>/dev/null | wc -l)
        
        echo "Current backup: $CURRENT_BACKUP" >> "$METRICS_FILE"
        echo "Backup size: $BACKUP_SIZE" >> "$METRICS_FILE"
        echo "Total increments: $TOTAL_BACKUPS" >> "$METRICS_FILE"
        
        # Check hard link deduplication efficiency
        if command -v hardlink &> /dev/null; then
            SAVED_SPACE=$(hardlink -v -n "$RSYNC_ROOT" 2>&1 | grep "saved" | tail -1)
            echo "Hard link savings: $SAVED_SPACE" >> "$METRICS_FILE"
        fi
    else
        echo "ERROR: No current rsync backup found" >> "$METRICS_FILE"
    fi
}

# Run checks
check_borg
check_rsync

# Display results
cat "$METRICS_FILE"

echo "Backup monitoring completed at $(date)" >> "$METRICS_FILE"

Set monitoring script permissions

Make the monitoring script executable and set up dependencies.

sudo chmod +x /usr/local/bin/backup-monitor.sh
sudo chown backup:backup /usr/local/bin/backup-monitor.sh
sudo apt install -y bc jq hardlink mailutils

sudo chmod +x /usr/local/bin/backup-monitor.sh
sudo chown backup:backup /usr/local/bin/backup-monitor.sh
sudo dnf install -y bc jq hardlink mailx

Test initial backups

Run manual backup tests to verify configuration before automation kicks in.

# Test BorgBackup
sudo systemctl start borgbackup.service
sudo systemctl status borgbackup.service

# Test rsync backup
sudo systemctl start rsyncbackup.service
sudo systemctl status rsyncbackup.service

# Check monitoring
sudo -u backup /usr/local/bin/backup-monitor.sh