Configure ScyllaDB backup and restore with automation

Update system packages

Start by updating your package manager to ensure you have the latest security patches and package versions.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install backup dependencies

Install the AWS CLI for S3 integration, Python for backup scripts, and email utilities for notifications.

sudo apt install -y awscli python3-pip python3-venv mailutils jq curl

sudo dnf install -y awscli python3-pip mailx jq curl

Create backup directories and user

Create dedicated directories for backup scripts and temporary storage with proper ownership.

sudo mkdir -p /opt/scylladb-backup/{scripts,logs,temp}
sudo useradd -r -s /bin/bash -d /opt/scylladb-backup scyllabackup
sudo chown -R scyllabackup:scyllabackup /opt/scylladb-backup
sudo chmod 755 /opt/scylladb-backup
sudo chmod 750 /opt/scylladb-backup/{scripts,logs,temp}

Configure AWS credentials

Set up AWS credentials for S3 backup storage. Create a configuration file with your S3 access details.

sudo -u scyllabackup mkdir -p /opt/scylladb-backup/.aws

[default]
aws_access_key_id = YOUR_ACCESS_KEY
aws_secret_access_key = YOUR_SECRET_KEY

[default]
region = us-east-1
output = json

sudo chown -R scyllabackup:scyllabackup /opt/scylladb-backup/.aws
sudo chmod 600 /opt/scylladb-backup/.aws/{credentials,config}

Create backup configuration file

Define backup settings including retention policies, S3 bucket details, and notification preferences.

# ScyllaDB Backup Configuration
SCYLLA_HOST="127.0.0.1"
SCYLLA_PORT="10000"
KEYSPACES="mykeyspace1 mykeyspace2"

# S3 Configuration
S3_BUCKET="my-scylla-backups"
S3_PREFIX="scylladb/backups"
S3_STORAGE_CLASS="STANDARD_IA"

# Retention Settings
LOCAL_RETENTION_DAYS=3
S3_RETENTION_DAYS=30

# Email Notifications
EMAIL_ENABLED=true
EMAIL_TO="admin@example.com"
EMAIL_FROM="scylla-backup@example.com"
SMTP_SERVER="localhost"

# Backup Settings
COMPRESSION=true
PARALLEL_UPLOADS=4
MAX_BACKUP_SIZE_GB=100

sudo chown scyllabackup:scyllabackup /opt/scylladb-backup/scripts/backup-config.conf
sudo chmod 640 /opt/scylladb-backup/scripts/backup-config.conf

Create backup script

Develop the main backup script that handles snapshots, compression, and S3 upload with error handling.

#!/usr/bin/env python3
import os
import sys
import subprocess
import datetime
import json
import logging
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
import configparser
import shutil
import glob

# Load configuration
config = configparser.ConfigParser()
config.read('/opt/scylladb-backup/scripts/backup-config.conf')

# Setup logging
log_file = f"/opt/scylladb-backup/logs/backup-{datetime.datetime.now().strftime('%Y%m%d')}.log"
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler(log_file),
        logging.StreamHandler(sys.stdout)
    ]
)
logger = logging.getLogger(__name__)

class ScyllaBackup:
    def __init__(self):
        self.scylla_host = config.get('DEFAULT', 'SCYLLA_HOST')
        self.scylla_port = config.get('DEFAULT', 'SCYLLA_PORT')
        self.keyspaces = config.get('DEFAULT', 'KEYSPACES').split()
        self.s3_bucket = config.get('DEFAULT', 'S3_BUCKET')
        self.s3_prefix = config.get('DEFAULT', 'S3_PREFIX')
        self.backup_dir = '/opt/scylladb-backup/temp'
        self.timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
        
    def send_notification(self, subject, message, is_error=False):
        if not config.getboolean('DEFAULT', 'EMAIL_ENABLED'):
            return
            
        try:
            msg = MIMEMultipart()
            msg['From'] = config.get('DEFAULT', 'EMAIL_FROM')
            msg['To'] = config.get('DEFAULT', 'EMAIL_TO')
            msg['Subject'] = f"ScyllaDB Backup: {subject}"
            
            body = f"Backup timestamp: {self.timestamp}\n\n{message}"
            msg.attach(MIMEText(body, 'plain'))
            
            server = smtplib.SMTP(config.get('DEFAULT', 'SMTP_SERVER'))
            server.sendmail(msg['From'], msg['To'], msg.as_string())
            server.quit()
            
        except Exception as e:
            logger.error(f"Failed to send email notification: {e}")
    
    def create_snapshot(self, keyspace):
        snapshot_name = f"backup_{self.timestamp}"
        try:
            cmd = f"nodetool -h {self.scylla_host} -p {self.scylla_port} snapshot -t {snapshot_name} {keyspace}"
            result = subprocess.run(cmd.split(), capture_output=True, text=True, check=True)
            logger.info(f"Created snapshot {snapshot_name} for keyspace {keyspace}")
            return snapshot_name
        except subprocess.CalledProcessError as e:
            logger.error(f"Failed to create snapshot for {keyspace}: {e.stderr}")
            raise
    
    def find_snapshot_files(self, keyspace, snapshot_name):
        data_dir = "/var/lib/scylla/data"
        snapshot_files = []
        
        keyspace_dir = os.path.join(data_dir, keyspace)
        if not os.path.exists(keyspace_dir):
            logger.warning(f"Keyspace directory not found: {keyspace_dir}")
            return snapshot_files
            
        for table_dir in os.listdir(keyspace_dir):
            snapshot_path = os.path.join(keyspace_dir, table_dir, "snapshots", snapshot_name)
            if os.path.exists(snapshot_path):
                for file in os.listdir(snapshot_path):
                    snapshot_files.append(os.path.join(snapshot_path, file))
                    
        return snapshot_files
    
    def compress_and_upload(self, keyspace, snapshot_name, files):
        if not files:
            logger.warning(f"No files found for keyspace {keyspace}")
            return False
            
        archive_name = f"{keyspace}_{snapshot_name}.tar.gz"
        archive_path = os.path.join(self.backup_dir, archive_name)
        
        try:
            # Create compressed archive
            cmd = ["tar", "-czf", archive_path] + files
            subprocess.run(cmd, check=True)
            logger.info(f"Created archive: {archive_name}")
            
            # Upload to S3
            s3_key = f"{self.s3_prefix}/{datetime.datetime.now().strftime('%Y/%m/%d')}/{archive_name}"
            upload_cmd = [
                "aws", "s3", "cp", archive_path, f"s3://{self.s3_bucket}/{s3_key}",
                "--storage-class", config.get('DEFAULT', 'S3_STORAGE_CLASS')
            ]
            subprocess.run(upload_cmd, check=True, env={**os.environ, 'HOME': '/opt/scylladb-backup'})
            logger.info(f"Uploaded to S3: s3://{self.s3_bucket}/{s3_key}")
            
            # Clean up local archive
            os.remove(archive_path)
            return True
            
        except subprocess.CalledProcessError as e:
            logger.error(f"Failed to compress/upload {keyspace}: {e}")
            return False
    
    def cleanup_snapshots(self):
        try:
            cmd = f"nodetool -h {self.scylla_host} -p {self.scylla_port} clearsnapshot"
            subprocess.run(cmd.split(), check=True)
            logger.info("Cleaned up old snapshots")
        except subprocess.CalledProcessError as e:
            logger.error(f"Failed to cleanup snapshots: {e}")
    
    def cleanup_old_backups(self):
        retention_days = int(config.get('DEFAULT', 'S3_RETENTION_DAYS'))
        cutoff_date = datetime.datetime.now() - datetime.timedelta(days=retention_days)
        
        try:
            # List old S3 objects
            list_cmd = [
                "aws", "s3api", "list-objects-v2",
                "--bucket", self.s3_bucket,
                "--prefix", self.s3_prefix,
                "--query", f"Contents[?LastModified<'{cutoff_date.isoformat()}']"
            ]
            
            result = subprocess.run(list_cmd, capture_output=True, text=True, check=True,
                                  env={**os.environ, 'HOME': '/opt/scylladb-backup'})
            old_objects = json.loads(result.stdout)
            
            if old_objects:
                for obj in old_objects:
                    delete_cmd = ["aws", "s3", "rm", f"s3://{self.s3_bucket}/{obj['Key']}"]
                    subprocess.run(delete_cmd, check=True,
                                 env={**os.environ, 'HOME': '/opt/scylladb-backup'})
                    logger.info(f"Deleted old backup: {obj['Key']}")
            
        except subprocess.CalledProcessError as e:
            logger.error(f"Failed to cleanup old backups: {e}")
    
    def run_backup(self):
        start_time = datetime.datetime.now()
        success_count = 0
        total_keyspaces = len(self.keyspaces)
        
        logger.info(f"Starting backup for keyspaces: {', '.join(self.keyspaces)}")
        
        try:
            for keyspace in self.keyspaces:
                try:
                    snapshot_name = self.create_snapshot(keyspace)
                    files = self.find_snapshot_files(keyspace, snapshot_name)
                    
                    if self.compress_and_upload(keyspace, snapshot_name, files):
                        success_count += 1
                        
                except Exception as e:
                    logger.error(f"Failed to backup keyspace {keyspace}: {e}")
            
            # Cleanup
            self.cleanup_snapshots()
            self.cleanup_old_backups()
            
            # Calculate duration
            duration = datetime.datetime.now() - start_time
            
            # Send notification
            if success_count == total_keyspaces:
                subject = "Backup Completed Successfully"
                message = f"All {total_keyspaces} keyspaces backed up successfully.\nDuration: {duration}"
                logger.info(message)
                self.send_notification(subject, message)
            else:
                subject = "Backup Completed with Errors"
                message = f"Backed up {success_count}/{total_keyspaces} keyspaces.\nDuration: {duration}\nCheck logs for details."
                logger.warning(message)
                self.send_notification(subject, message, is_error=True)
                
        except Exception as e:
            subject = "Backup Failed"
            message = f"Backup process failed: {str(e)}"
            logger.error(message)
            self.send_notification(subject, message, is_error=True)
            sys.exit(1)

if __name__ == "__main__":
    backup = ScyllaBackup()
    backup.run_backup()

sudo chown scyllabackup:scyllabackup /opt/scylladb-backup/scripts/scylla-backup.py
sudo chmod 750 /opt/scylladb-backup/scripts/scylla-backup.py

Create restore script

Develop a restore script for disaster recovery scenarios with keyspace and timestamp selection.

#!/usr/bin/env python3
import os
import sys
import subprocess
import datetime
import json
import logging
import argparse
import configparser
import tempfile

# Load configuration
config = configparser.ConfigParser()
config.read('/opt/scylladb-backup/scripts/backup-config.conf')

# Setup logging
log_file = f"/opt/scylladb-backup/logs/restore-{datetime.datetime.now().strftime('%Y%m%d')}.log"
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler(log_file),
        logging.StreamHandler(sys.stdout)
    ]
)
logger = logging.getLogger(__name__)

class ScyllaRestore:
    def __init__(self):
        self.scylla_host = config.get('DEFAULT', 'SCYLLA_HOST')
        self.scylla_port = config.get('DEFAULT', 'SCYLLA_PORT')
        self.s3_bucket = config.get('DEFAULT', 'S3_BUCKET')
        self.s3_prefix = config.get('DEFAULT', 'S3_PREFIX')
        self.temp_dir = tempfile.mkdtemp(prefix='scylla-restore-')
        
    def list_available_backups(self, keyspace=None):
        try:
            prefix = self.s3_prefix
            if keyspace:
                prefix = f"{self.s3_prefix}/*/{keyspace}_*"
                
            list_cmd = [
                "aws", "s3api", "list-objects-v2",
                "--bucket", self.s3_bucket,
                "--prefix", self.s3_prefix,
                "--query", "Contents[?contains(Key, '.tar.gz')]"
            ]
            
            result = subprocess.run(list_cmd, capture_output=True, text=True, check=True,
                                  env={**os.environ, 'HOME': '/opt/scylladb-backup'})
            backups = json.loads(result.stdout)
            
            if backups:
                print("Available backups:")
                for backup in sorted(backups, key=lambda x: x['LastModified'], reverse=True):
                    key = backup['Key']
                    size = backup['Size'] / (1024*1024)  # MB
                    date = backup['LastModified'][:19]
                    print(f"  {key} ({size:.1f}MB, {date})")
            else:
                print("No backups found")
                
        except subprocess.CalledProcessError as e:
            logger.error(f"Failed to list backups: {e}")
            return []
    
    def download_backup(self, s3_key):
        local_path = os.path.join(self.temp_dir, os.path.basename(s3_key))
        
        try:
            download_cmd = [
                "aws", "s3", "cp", f"s3://{self.s3_bucket}/{s3_key}", local_path
            ]
            subprocess.run(download_cmd, check=True,
                         env={**os.environ, 'HOME': '/opt/scylladb-backup'})
            logger.info(f"Downloaded backup: {s3_key}")
            return local_path
            
        except subprocess.CalledProcessError as e:
            logger.error(f"Failed to download backup {s3_key}: {e}")
            return None
    
    def extract_backup(self, archive_path, keyspace):
        extract_dir = os.path.join(self.temp_dir, f"extract_{keyspace}")
        os.makedirs(extract_dir, exist_ok=True)
        
        try:
            extract_cmd = ["tar", "-xzf", archive_path, "-C", extract_dir]
            subprocess.run(extract_cmd, check=True)
            logger.info(f"Extracted backup for keyspace {keyspace}")
            return extract_dir
            
        except subprocess.CalledProcessError as e:
            logger.error(f"Failed to extract backup: {e}")
            return None
    
    def restore_keyspace(self, keyspace, backup_date, dry_run=False):
        # Find matching backup
        s3_key = f"{self.s3_prefix}/{backup_date[:4]}/{backup_date[4:6]}/{backup_date[6:8]}/{keyspace}_backup_{backup_date}.tar.gz"
        
        if dry_run:
            print(f"[DRY RUN] Would restore {keyspace} from {s3_key}")
            return True
            
        try:
            # Download backup
            archive_path = self.download_backup(s3_key)
            if not archive_path:
                return False
                
            # Extract backup
            extract_dir = self.extract_backup(archive_path, keyspace)
            if not extract_dir:
                return False
            
            # Stop ScyllaDB for restoration
            logger.info("Stopping ScyllaDB service")
            subprocess.run(["sudo", "systemctl", "stop", "scylla-server"], check=True)
            
            # Copy restored files to data directory
            data_dir = f"/var/lib/scylla/data/{keyspace}"
            if os.path.exists(data_dir):
                subprocess.run(["sudo", "rm", "-rf", data_dir], check=True)
                
            subprocess.run(["sudo", "cp", "-r", extract_dir, data_dir], check=True)
            subprocess.run(["sudo", "chown", "-R", "scylla:scylla", data_dir], check=True)
            
            # Start ScyllaDB
            logger.info("Starting ScyllaDB service")
            subprocess.run(["sudo", "systemctl", "start", "scylla-server"], check=True)
            
            # Wait for service to be ready
            import time
            time.sleep(30)
            
            # Verify restoration
            verify_cmd = f"cqlsh -e 'DESCRIBE KEYSPACE {keyspace};'"
            result = subprocess.run(verify_cmd.split(), capture_outpu