Automate backup and restore for Ollama models with systemd timers and shell scripts

Update system packages

Start by updating your package manager to ensure you have the latest security patches and tools.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install backup prerequisites

Install compression tools, rsync for efficient transfers, and monitoring utilities for backup verification.

sudo apt install -y rsync gzip tar curl jq pigz pv

sudo dnf install -y rsync gzip tar curl jq pigz pv

Create backup directory structure

Set up organized directories for storing backups with proper permissions for the backup system.

sudo mkdir -p /opt/ollama-backup/{scripts,backups,logs,restore}
sudo mkdir -p /opt/ollama-backup/backups/{daily,weekly,monthly}
sudo useradd --system --home /opt/ollama-backup --shell /bin/bash ollama-backup
sudo chown -R ollama-backup:ollama-backup /opt/ollama-backup
sudo chmod 750 /opt/ollama-backup
sudo chmod 755 /opt/ollama-backup/backups

Identify Ollama data locations

Find where Ollama stores models and configuration data on your system.

sudo systemctl status ollama
sudo find /home /root /usr/share -name "*.gguf" -o -name "modelfile" 2>/dev/null | head -10
ls -la ~/.ollama/models/ 2>/dev/null || echo "Default location not found"
sudo ls -la /usr/share/ollama/.ollama/models/ 2>/dev/null || echo "System location not found"

Create the main backup script

Build a comprehensive backup script that handles model discovery, compression, and verification.

#!/bin/bash

# Ollama Model Backup Script
# Automatically discovers and backs up Ollama models with compression

set -euo pipefail

# Configuration
BACKUP_BASE="/opt/ollama-backup/backups"
LOG_FILE="/opt/ollama-backup/logs/backup-$(date +%Y%m%d-%H%M%S).log"
RETENTION_DAYS=30
COMPRESSION_LEVEL=6
MAX_PARALLEL_JOBS=2

# Ollama data locations to check
OLLAMA_LOCATIONS=(
    "$HOME/.ollama"
    "/usr/share/ollama/.ollama"
    "/var/lib/ollama"
    "/opt/ollama"
)

# Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

# Error handling
error_exit() {
    log "ERROR: $1"
    exit 1
}

# Check if running as backup user
if [[ $(id -u) -eq 0 ]]; then
    error_exit "Do not run this script as root. Use the ollama-backup user."
fi

# Create backup type directory
BACKUP_TYPE=${1:-daily}
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
BACKUP_DIR="$BACKUP_BASE/$BACKUP_TYPE/$TIMESTAMP"
mkdir -p "$BACKUP_DIR"

log "Starting Ollama backup: $BACKUP_TYPE"
log "Backup directory: $BACKUP_DIR"

# Function to backup a location
backup_location() {
    local source_dir="$1"
    local backup_name="$2"
    
    if [[ ! -d "$source_dir" ]]; then
        log "Skipping $source_dir - directory not found"
        return 0
    fi
    
    local size=$(du -sh "$source_dir" | cut -f1)
    log "Backing up $source_dir ($size) as $backup_name"
    
    # Create compressed archive with progress
    tar -C "$(dirname "$source_dir")" -cf - "$(basename "$source_dir")" | \
        pv -s $(du -sb "$source_dir" | cut -f1) | \
        pigz -$COMPRESSION_LEVEL > "$BACKUP_DIR/$backup_name.tar.gz"
    
    # Verify backup integrity
    if pigz -t "$BACKUP_DIR/$backup_name.tar.gz" >/dev/null 2>&1; then
        log "Successfully backed up $source_dir"
        # Store metadata
        echo "source=$source_dir" > "$BACKUP_DIR/$backup_name.meta"
        echo "size=$size" >> "$BACKUP_DIR/$backup_name.meta"
        echo "timestamp=$TIMESTAMP" >> "$BACKUP_DIR/$backup_name.meta"
        echo "checksum=$(sha256sum "$BACKUP_DIR/$backup_name.tar.gz" | cut -d' ' -f1)" >> "$BACKUP_DIR/$backup_name.meta"
    else
        error_exit "Backup verification failed for $source_dir"
    fi
}

# Backup Ollama service configuration
if systemctl is-enabled ollama >/dev/null 2>&1; then
    log "Backing up Ollama systemd configuration"
    mkdir -p "$BACKUP_DIR/systemd"
    sudo cp /etc/systemd/system/ollama.service "$BACKUP_DIR/systemd/" 2>/dev/null || true
    systemctl show ollama --no-pager > "$BACKUP_DIR/systemd/ollama-service-status.txt"
fi

# Backup Ollama models and data
for location in "${OLLAMA_LOCATIONS[@]}"; do
    # Expand tilde to actual home directory
    expanded_location=$(eval echo "$location")
    
    if [[ -d "$expanded_location" ]]; then
        backup_name=$(echo "$expanded_location" | sed 's|/|_|g' | sed 's/^_//')
        backup_location "$expanded_location" "ollama-$backup_name"
    fi
done

# Get list of installed models
if command -v ollama >/dev/null 2>&1; then
    log "Saving list of installed models"
    ollama list > "$BACKUP_DIR/model-list.txt" 2>/dev/null || echo "No models found" > "$BACKUP_DIR/model-list.txt"
fi

# Cleanup old backups
log "Cleaning up backups older than $RETENTION_DAYS days"
find "$BACKUP_BASE/$BACKUP_TYPE" -type d -mtime +$RETENTION_DAYS -exec rm -rf {} + 2>/dev/null || true

# Generate backup summary
BACKUP_SIZE=$(du -sh "$BACKUP_DIR" | cut -f1)
log "Backup completed successfully"
log "Total backup size: $BACKUP_SIZE"
log "Backup location: $BACKUP_DIR"

# Send metrics if Prometheus node_exporter textfile collector is available
if [[ -d "/var/lib/node_exporter/textfile_collector" ]]; then
    cat > "/var/lib/node_exporter/textfile_collector/ollama_backup.prom" << EOF
# HELP ollama_backup_last_success_timestamp_seconds Last successful backup timestamp
# TYPE ollama_backup_last_success_timestamp_seconds gauge
ollama_backup_last_success_timestamp_seconds{type="$BACKUP_TYPE"} $(date +%s)
# HELP ollama_backup_size_bytes Size of the backup in bytes
# TYPE ollama_backup_size_bytes gauge
ollama_backup_size_bytes{type="$BACKUP_TYPE"} $(du -sb "$BACKUP_DIR" | cut -f1)
EOF
fi

log "Ollama backup completed: $BACKUP_DIR"

Create the restore script

Build a restore script that can recover models from backups with verification and rollback capabilities.

#!/bin/bash

# Ollama Model Restore Script
# Restores Ollama models from compressed backups

set -euo pipefail

# Configuration
BACKUP_BASE="/opt/ollama-backup/backups"
RESTORE_DIR="/opt/ollama-backup/restore"
LOG_FILE="/opt/ollama-backup/logs/restore-$(date +%Y%m%d-%H%M%S).log"

# Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

# Error handling
error_exit() {
    log "ERROR: $1"
    exit 1
}

# Usage information
usage() {
    echo "Usage: $0 

Make scripts executable and set permissions

Set proper permissions on the backup scripts to ensure they can execute but remain secure.

sudo chmod 755 /opt/ollama-backup/scripts/ollama-backup.sh
sudo chmod 755 /opt/ollama-backup/scripts/ollama-restore.sh
sudo chown ollama-backup:ollama-backup /opt/ollama-backup/scripts/*.sh

Create systemd service for backup

Create a systemd service that handles the backup execution with proper environment and error handling.

[Unit]
Description=Ollama Model Backup (%i)
After=network.target
Wants=network.target

[Service]
Type=oneshot
User=ollama-backup
Group=ollama-backup
WorkingDirectory=/opt/ollama-backup
Environment=HOME=/opt/ollama-backup
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ExecStart=/opt/ollama-backup/scripts/ollama-backup.sh %i
StandardOutput=journal
StandardError=journal
TimeoutStartSec=3600

# Security hardening
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=read-only
ProtectSystem=strict
ReadWritePaths=/opt/ollama-backup /var/lib/node_exporter
SupplementaryGroups=ollama

[Install]
WantedBy=multi-user.target

Create systemd timers for scheduled backups

Set up multiple timer schedules for daily, weekly, and monthly backups with different retention policies.

[Unit]
Description=Daily Ollama Model Backup
Requires=ollama-backup@daily.service

[Timer]
OnCalendar=daily
RandomizedDelaySec=1800
Persistent=true
AccuracySec=1m

[Install]
WantedBy=timers.target

[Unit]
Description=Weekly Ollama Model Backup
Requires=ollama-backup@weekly.service

[Timer]
OnCalendar=weekly
RandomizedDelaySec=3600
Persistent=true
AccuracySec=1m

[Install]
WantedBy=timers.target

[Unit]
Description=Monthly Ollama Model Backup
Requires=ollama-backup@monthly.service

[Timer]
OnCalendar=monthly
RandomizedDelaySec=7200
Persistent=true
AccuracySec=1m

[Install]
WantedBy=timers.target

Create backup monitoring script

Build a monitoring script that checks backup health and sends alerts when backups fail or are missing.

#!/bin/bash

# Ollama Backup Monitoring Script
# Checks backup health and sends alerts

set -euo pipefail

BACKUP_BASE="/opt/ollama-backup/backups"
LOG_FILE="/opt/ollama-backup/logs/monitor-$(date +%Y%m%d).log"
MAX_AGE_HOURS=25  # Daily backups should not be older than 25 hours
MIN_BACKUP_SIZE="100M"  # Minimum expected backup size

# Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

# Alert function (customize for your notification system)
alert() {
    local severity="$1"
    local message="$2"
    
    log "ALERT [$severity]: $message"
    
    # Send to syslog
    logger -t "ollama-backup" -p "daemon.$severity" "$message"
    
    # Optional: Send email (requires mailutils)
    # echo "$message" | mail -s "Ollama Backup Alert" admin@example.com
    
    # Optional: Send to Slack/Discord webhook
    # curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" YOUR_WEBHOOK_URL
}

# Check if backup exists and is recent
check_backup_freshness() {
    local backup_type="$1"
    local backup_dir="$BACKUP_BASE/$backup_type"
    
    if [[ ! -d "$backup_dir" ]]; then
        alert "error" "Backup directory missing: $backup_dir"
        return 1
    fi
    
    # Find most recent backup
    local latest_backup=$(find "$backup_dir" -type d -name "[0-9]*" | sort | tail -1)
    
    if [[ -z "$latest_backup" ]]; then
        alert "error" "No backups found in $backup_dir"
        return 1
    fi
    
    # Check backup age
    local backup_age=$(( ($(date +%s) - $(stat -c %Y "$latest_backup")) / 3600 ))
    
    if [[ $backup_age -gt $MAX_AGE_HOURS ]]; then
        alert "warning" "Latest $backup_type backup is $backup_age hours old: $latest_backup"
        return 1
    fi
    
    log "$backup_type backup is current ($backup_age hours old)"
    return 0
}

# Check backup integrity
check_backup_integrity() {
    local backup_dir="$1"
    local failed=false
    
    for backup_file in "$backup_dir"/*.tar.gz; do
        if [[ -f "$backup_file" ]]; then
            if ! pigz -t "$backup_file" >/dev/null 2>&1; then
                alert "error" "Corrupted backup file: $backup_file"
                failed=true
            fi
            
            # Check minimum size
            local size=$(stat -c%s "$backup_file")
            local min_size=$(numfmt --from=iec "$MIN_BACKUP_SIZE")
            
            if [[ $size -lt $min_size ]]; then
                alert "warning" "Backup file smaller than expected: $backup_file ($(numfmt --to=iec $size))"
            fi
        fi
    done
    
    if [[ "$failed" == "false" ]]; then
        log "All backup files in $backup_dir passed integrity checks"
        return 0
    else
        return 1
    fi
}

# Check systemd service status
check_service_status() {
    for service in ollama-backup-daily.timer ollama-backup-weekly.timer ollama-backup-monthly.timer; do
        if ! systemctl is-active "$service" >/dev/null 2>&1; then
            alert "warning" "Backup timer not active: $service"
        elif ! systemctl is-enabled "$service" >/dev/null 2>&1; then
            alert "warning" "Backup timer not enabled: $service"
        else
            log "Backup timer $service is active and enabled"
        fi
    done
}

# Check disk space
check_disk_space() {
    local backup_partition=$(df "$BACKUP_BASE" | tail -1)
    local usage_percent=$(echo "$backup_partition" | awk '{print $5}' | sed 's/%//')
    
    if [[ $usage_percent -gt 90 ]]; then
        alert "error" "Backup disk usage critical: ${usage_percent}%"
    elif [[ $usage_percent -gt 80 ]]; then
        alert "warning" "Backup disk usage high: ${usage_percent}%"
    else
        log "Backup disk usage normal: ${usage_percent}%"
    fi
}

log "Starting backup monitoring check"

# Run all checks
overall_status=0

check_service_status
check_disk_space

# Check daily backups
if ! check_backup_freshness "daily"; then
    overall_status=1
else
    latest_daily=$(find "$BACKUP_BASE/daily" -type d -name "[0-9]*" | sort | tail -1)
    if [[ -n "$latest_daily" ]]; then
        check_backup_integrity "$latest_daily" || overall_status=1
    fi
fi

# Generate summary metrics
if [[ -d "/var/lib/node_exporter/textfile_collector" ]]; then
    cat > "/var/lib/node_exporter/textfile_collector/ollama_backup_health.prom" << EOF
# HELP ollama_backup_health_status Overall backup system health (1=healthy, 0=issues)
# TYPE ollama_backup_health_status gauge
ollama_backup_health_status $((overall