Implement encryption for Consul backups using GPG with automated snapshots and secure storage

Export and secure GPG keys

Export the public key for sharing and securely store the private key with proper permissions.

sudo -u consul-backup bash -c '
GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format LONG | grep sec | cut -d"/" -f2 | cut -d" " -f1)
echo $GPG_KEY_ID > /opt/consul-backup/keys/key-id.txt
gpg --armor --export $GPG_KEY_ID > /opt/consul-backup/keys/public-key.asc
gpg --armor --export-secret-keys $GPG_KEY_ID > /opt/consul-backup/keys/private-key.asc
'
sudo chmod 600 /opt/consul-backup/keys/private-key.asc
sudo chmod 644 /opt/consul-backup/keys/public-key.asc
sudo chmod 644 /opt/consul-backup/keys/key-id.txt

Create GPG configuration

Configure GPG settings for automated operations without interactive prompts.

use-agent
pinentry-mode loopback
no-tty
batch
yes
quiet
no-greeting
no-permission-warning
no-secmem-warning

sudo chown consul-backup:consul-backup /opt/consul-backup/.gnupg/gpg.conf
sudo chmod 600 /opt/consul-backup/.gnupg/gpg.conf

Create main backup script

Develop a comprehensive script that creates Consul snapshots, compresses them, and encrypts with GPG.

#!/bin/bash

Consul Encrypted Backup Script
set -euo pipefail

Configuration
BACKUP_DIR="/opt/consul-backup/backups"
LOG_DIR="/opt/consul-backup/logs"
KEY_DIR="/opt/consul-backup/keys"
DATE=$(date +"%Y%m%d-%H%M%S")
HOSTNAME=$(hostname -s)
BACKUP_NAME="consul-snapshot-${HOSTNAME}-${DATE}"
LOG_FILE="${LOG_DIR}/backup-${DATE}.log"
RETENTION_DAYS=30
COMPRESSION_LEVEL=9

Consul configuration
CONSUL_HTTP_ADDR=${CONSUL_HTTP_ADDR:-"127.0.0.1:8500"}
CONSUL_TOKEN_FILE=${CONSUL_TOKEN_FILE:-"/etc/consul/tokens/backup-token"}
CONSUL_CACERT=${CONSUL_CACERT:-"/etc/consul/tls/ca-cert.pem"}
CONSUL_CLIENT_CERT=${CONSUL_CLIENT_CERT:-"/etc/consul/tls/client-cert.pem"}
CONSUL_CLIENT_KEY=${CONSUL_CLIENT_KEY:-"/etc/consul/tls/client-key.pem"}

GPG key ID
GPG_KEY_ID=$(cat ${KEY_DIR}/key-id.txt)

Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "${LOG_FILE}"
}

Error handling
error_exit() {
    log "ERROR: $1"
    exit 1
}

Start backup process
log "Starting Consul backup process"

Check Consul connectivity
log "Checking Consul connectivity"
if ! consul members >/dev/null 2>&1; then
    error_exit "Cannot connect to Consul cluster"
fi

Create temporary directory for processing
TEMP_DIR=$(mktemp -d)
trap "rm -rf ${TEMP_DIR}" EXIT

Create snapshot
log "Creating Consul snapshot"
if [[ -f "${CONSUL_TOKEN_FILE}" ]]; then
    CONSUL_HTTP_TOKEN=$(cat "${CONSUL_TOKEN_FILE}")
    export CONSUL_HTTP_TOKEN
fi

consul snapshot save "${TEMP_DIR}/${BACKUP_NAME}.snap" 2>>"${LOG_FILE}" || error_exit "Failed to create Consul snapshot"

Verify snapshot
log "Verifying snapshot integrity"
consul snapshot inspect "${TEMP_DIR}/${BACKUP_NAME}.snap" >/dev/null 2>>"${LOG_FILE}" || error_exit "Snapshot verification failed"

Compress snapshot
log "Compressing snapshot with gzip level ${COMPRESSION_LEVEL}"
gzip -${COMPRESSION_LEVEL} "${TEMP_DIR}/${BACKUP_NAME}.snap" || error_exit "Compression failed"

Encrypt compressed snapshot
log "Encrypting backup with GPG key ${GPG_KEY_ID}"
gpg --trust-model always --batch --yes --compress-algo 0 \
    --cipher-algo AES256 --digest-algo SHA512 \
    --s2k-digest-algo SHA512 --s2k-cipher-algo AES256 \
    --recipient "${GPG_KEY_ID}" \
    --encrypt "${TEMP_DIR}/${BACKUP_NAME}.snap.gz" \
    2>>"${LOG_FILE}" || error_exit "GPG encryption failed"

Move encrypted backup to final location
log "Moving encrypted backup to storage directory"
mv "${TEMP_DIR}/${BACKUP_NAME}.snap.gz.gpg" "${BACKUP_DIR}/" || error_exit "Failed to move backup file"

Create metadata file
log "Creating backup metadata"
cat > "${BACKUP_DIR}/${BACKUP_NAME}.meta" </dev/null || stat -c%s "${TEMP_DIR}/${BACKUP_NAME}.snap")",
  "compressed_size": "$(stat -f%z "${BACKUP_DIR}/${BACKUP_NAME}.snap.gz.gpg" 2>/dev/null || stat -c%s "${BACKUP_DIR}/${BACKUP_NAME}.snap.gz.gpg")",
  "backup_file": "${BACKUP_NAME}.snap.gz.gpg"
}
EOF

Set proper permissions
chmod 600 "${BACKUP_DIR}/${BACKUP_NAME}.snap.gz.gpg"
chmod 644 "${BACKUP_DIR}/${BACKUP_NAME}.meta"

Clean old backups
log "Cleaning backups older than ${RETENTION_DAYS} days"
find "${BACKUP_DIR}" -name "consul-snapshot-*.gpg" -mtime +${RETENTION_DAYS} -delete 2>>"${LOG_FILE}" || true
find "${BACKUP_DIR}" -name "consul-snapshot-*.meta" -mtime +${RETENTION_DAYS} -delete 2>>"${LOG_FILE}" || true

Clean old logs
find "${LOG_DIR}" -name "backup-*.log" -mtime +7 -delete 2>/dev/null || true

log "Backup completed successfully: ${BACKUP_NAME}.snap.gz.gpg"
log "Backup size: $(du -h "${BACKUP_DIR}/${BACKUP_NAME}.snap.gz.gpg" | cut -f1)"
log "Available backups: $(ls -1 "${BACKUP_DIR}"/*.gpg 2>/dev/null | wc -l)"

exit 0

sudo chown consul-backup:consul-backup /opt/consul-backup/scripts/consul-backup.sh
sudo chmod 750 /opt/consul-backup/scripts/consul-backup.sh

Create backup restoration script

Build a script to decrypt and restore Consul snapshots from encrypted backups.

#!/bin/bash

Consul Backup Restoration Script
set -euo pipefail

Configuration
BACKUP_DIR="/opt/consul-backup/backups"
LOG_DIR="/opt/consul-backup/logs"
DATE=$(date +"%Y%m%d-%H%M%S")
LOG_FILE="${LOG_DIR}/restore-${DATE}.log"

Usage function
usage() {
    echo "Usage: $0 "
    echo "Example: $0 consul-snapshot-web01-20241201-143022.snap.gz.gpg"
    exit 1
}

Check arguments
if [[ $# -ne 1 ]]; then
    usage
fi

BACKUP_FILE="$1"

Validate backup file exists
if [[ ! -f "${BACKUP_DIR}/${BACKUP_FILE}" ]]; then
    echo "Error: Backup file ${BACKUP_DIR}/${BACKUP_FILE} not found"
    exit 1
fi

Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "${LOG_FILE}"
}

Error handling
error_exit() {
    log "ERROR: $1"
    exit 1
}

log "Starting Consul backup restoration from ${BACKUP_FILE}"

Create temporary directory
TEMP_DIR=$(mktemp -d)
trap "rm -rf ${TEMP_DIR}" EXIT

Decrypt backup
log "Decrypting backup file"
gpg --batch --yes --decrypt "${BACKUP_DIR}/${BACKUP_FILE}" > "${TEMP_DIR}/snapshot.snap.gz" 2>>"${LOG_FILE}" || error_exit "GPG decryption failed"

Decompress
log "Decompressing snapshot"
gunzip "${TEMP_DIR}/snapshot.snap.gz" || error_exit "Decompression failed"

Verify snapshot before restoration
log "Verifying snapshot integrity"
consul snapshot inspect "${TEMP_DIR}/snapshot.snap" >/dev/null 2>>"${LOG_FILE}" || error_exit "Snapshot verification failed"

Restore snapshot
log "Restoring Consul snapshot"
read -p "Are you sure you want to restore this snapshot? This will overwrite current data (y/N): " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
    consul snapshot restore "${TEMP_DIR}/snapshot.snap" 2>>"${LOG_FILE}" || error_exit "Snapshot restoration failed"
    log "Restoration completed successfully"
else
    log "Restoration cancelled by user"
fi

sudo chown consul-backup:consul-backup /opt/consul-backup/scripts/consul-restore.sh
sudo chmod 750 /opt/consul-backup/scripts/consul-restore.sh

Create backup verification script

Implement automated verification to ensure backup integrity and decryption capabilities.

#!/bin/bash

Consul Backup Verification Script
set -euo pipefail

BACKUP_DIR="/opt/consul-backup/backups"
LOG_DIR="/opt/consul-backup/logs"
DATE=$(date +"%Y%m%d-%H%M%S")
LOG_FILE="${LOG_DIR}/verify-${DATE}.log"

Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "${LOG_FILE}"
}

log "Starting backup verification process"

Verify recent backups
BACKUPS_VERIFIED=0
BACKUPS_FAILED=0

for backup_file in $(ls -t "${BACKUP_DIR}"/*.gpg 2>/dev/null | head -5); do
    backup_name=$(basename "$backup_file")
    log "Verifying ${backup_name}"
    
    # Create temporary directory
    TEMP_DIR=$(mktemp -d)
    
    # Try to decrypt and verify
    if gpg --batch --yes --decrypt "$backup_file" > "${TEMP_DIR}/test.snap.gz" 2>/dev/null; then
        if gunzip "${TEMP_DIR}/test.snap.gz" 2>/dev/null; then
            if consul snapshot inspect "${TEMP_DIR}/test.snap" >/dev/null 2>&1; then
                log "✓ ${backup_name} verification successful"
                ((BACKUPS_VERIFIED++))
            else
                log "✗ ${backup_name} snapshot integrity check failed"
                ((BACKUPS_FAILED++))
            fi
        else
            log "✗ ${backup_name} decompression failed"
            ((BACKUPS_FAILED++))
        fi
    else
        log "✗ ${backup_name} decryption failed"
        ((BACKUPS_FAILED++))
    fi
    
    rm -rf "${TEMP_DIR}"
done

log "Verification complete: ${BACKUPS_VERIFIED} successful, ${BACKUPS_FAILED} failed"

if [[ $BACKUPS_FAILED -gt 0 ]]; then
    exit 1
fi

exit 0

sudo chown consul-backup:consul-backup /opt/consul-backup/scripts/verify-backup.sh
sudo chmod 750 /opt/consul-backup/scripts/verify-backup.sh

Create S3 sync script for remote storage

Set up secure remote storage synchronization with S3-compatible storage for offsite backup retention.

#!/bin/bash

S3 Backup Sync Script
set -euo pipefail

Configuration - adjust these variables
S3_BUCKET="consul-backups-$(hostname -s)"
S3_PREFIX="consul/$(date +%Y/%m)"
BACKUP_DIR="/opt/consul-backup/backups"
LOG_DIR="/opt/consul-backup/logs"
DATE=$(date +"%Y%m%d-%H%M%S")
LOG_FILE="${LOG_DIR}/s3sync-${DATE}.log"
RETENTION_DAYS=90

AWS CLI configuration check
if ! aws sts get-caller-identity >/dev/null 2>&1; then
    echo "Error: AWS CLI not configured or credentials invalid"
    exit 1
fi

Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "${LOG_FILE}"
}

log "Starting S3 backup synchronization"

Sync recent backups to S3
log "Syncing backups to S3: s3://${S3_BUCKET}/${S3_PREFIX}/"
aws s3 sync "${BACKUP_DIR}" "s3://${S3_BUCKET}/${S3_PREFIX}/" \
    --exclude "*" \
    --include "*.gpg" \
    --include "*.meta" \
    --storage-class STANDARD_IA \
    --server-side-encryption AES256 \
    2>>"${LOG_FILE}" || {
    log "S3 sync failed"
    exit 1
}

Clean old S3 backups
log "Cleaning S3 backups older than ${RETENTION_DAYS} days"
CUTOFF_DATE=$(date -d "${RETENTION_DAYS} days ago" +%Y-%m-%d)
aws s3 ls "s3://${S3_BUCKET}/consul/" --recursive | \
    awk '$1 <= "'${CUTOFF_DATE}'" {print $4}' | \
    while read -r key; do
        if [[ -n "$key" ]]; then
            log "Deleting old backup: s3://${S3_BUCKET}/${key}"
            aws s3 rm "s3://${S3_BUCKET}/${key}" 2>>"${LOG_FILE}" || true
        fi
    done

log "S3 synchronization completed"

exit 0

sudo chown consul-backup:consul-backup /opt/consul-backup/scripts/sync-to-s3.sh
sudo chmod 750 /opt/consul-backup/scripts/sync-to-s3.sh

Configure AWS credentials for backup user

Set up AWS credentials for the backup user with minimal required permissions for S3 operations.

sudo -u consul-backup mkdir -p /opt/consul-backup/.aws
sudo -u consul-backup tee /opt/consul-backup/.aws/credentials >/dev/null <