Configure ScyllaDB cluster monitoring with Prometheus and Grafana dashboards

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Configuration
PROMETHEUS_VERSION="2.48.0"
PROMETHEUS_USER="prometheus"
PROMETHEUS_HOME="/opt/prometheus"
PROMETHEUS_CONFIG="/etc/prometheus"
PROMETHEUS_DATA="/var/lib/prometheus"
SCYLLA_NODES="${SCYLLA_NODES:-}"

# Function to print colored output
print_status() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Usage function
usage() {
    echo "Usage: $0 [SCYLLA_NODE1,SCYLLA_NODE2,...]"
    echo "Example: $0 203.0.113.10,203.0.113.11,203.0.113.12"
    echo "If no nodes specified, you'll need to edit the config manually"
    exit 1
}

# Cleanup function
cleanup() {
    if [[ $? -ne 0 ]]; then
        print_error "Script failed. Cleaning up..."
        systemctl stop prometheus 2>/dev/null || true
        systemctl disable prometheus 2>/dev/null || true
        rm -f /etc/systemd/system/prometheus.service
        rm -rf "$PROMETHEUS_HOME" "$PROMETHEUS_CONFIG" "$PROMETHEUS_DATA"
        userdel "$PROMETHEUS_USER" 2>/dev/null || true
    fi
}

trap cleanup ERR

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    print_error "This script must be run as root or with sudo"
    exit 1
fi

# Parse arguments
if [[ $# -gt 1 ]]; then
    usage
elif [[ $# -eq 1 ]]; then
    SCYLLA_NODES="$1"
fi

# Auto-detect distribution
if [[ -f /etc/os-release ]]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            ;;
        *)
            print_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    print_error "Cannot detect distribution. /etc/os-release not found"
    exit 1
fi

print_status "[1/8] Updating package manager..."
$PKG_UPDATE

print_status "[2/8] Installing required packages..."
$PKG_INSTALL wget tar

print_status "[3/8] Creating prometheus user..."
if ! id "$PROMETHEUS_USER" &>/dev/null; then
    useradd --system --shell /bin/false --home-dir /var/lib/prometheus --create-home "$PROMETHEUS_USER"
fi

print_status "[4/8] Downloading and installing Prometheus..."
cd /tmp
wget -q "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"
tar xzf "prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"
rm -rf "$PROMETHEUS_HOME"
mv "prometheus-${PROMETHEUS_VERSION}.linux-amd64" "$PROMETHEUS_HOME"
mkdir -p "$PROMETHEUS_CONFIG" "$PROMETHEUS_DATA"
chown -R "$PROMETHEUS_USER":"$PROMETHEUS_USER" "$PROMETHEUS_HOME" "$PROMETHEUS_CONFIG" "$PROMETHEUS_DATA"
chmod 755 "$PROMETHEUS_HOME" "$PROMETHEUS_CONFIG" "$PROMETHEUS_DATA"
ln -sf "$PROMETHEUS_HOME/prometheus" /usr/local/bin/prometheus
ln -sf "$PROMETHEUS_HOME/promtool" /usr/local/bin/promtool

print_status "[5/8] Creating Prometheus configuration..."
cat > "$PROMETHEUS_CONFIG/prometheus.yml" << EOF
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "scylla_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - localhost:9093

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'scylla'
    static_configs:
      - targets:
EOF

if [[ -n "$SCYLLA_NODES" ]]; then
    IFS=',' read -ra NODES <<< "$SCYLLA_NODES"
    for node in "${NODES[@]}"; do
        echo "        - '$node:9180'" >> "$PROMETHEUS_CONFIG/prometheus.yml"
    done
else
    cat >> "$PROMETHEUS_CONFIG/prometheus.yml" << EOF
        - 'localhost:9180'
EOF
fi

cat >> "$PROMETHEUS_CONFIG/prometheus.yml" << EOF
    scrape_interval: 10s
    metrics_path: /metrics
    params:
      format: [prometheus]

  - job_name: 'scylla-manager'
    static_configs:
      - targets:
        - 'localhost:56090'
    scrape_interval: 30s
EOF

print_status "[6/8] Creating ScyllaDB alerting rules..."
cat > "$PROMETHEUS_CONFIG/scylla_rules.yml" << 'EOF'
groups:
- name: scylla.rules
  rules:
  - alert: ScyllaNodeDown
    expr: up{job="scylla"} == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "ScyllaDB node is down"
      description: "ScyllaDB node {{ $labels.instance }} has been down for more than 1 minute."

  - alert: ScyllaHighCPU
    expr: scylla_reactor_utilization > 0.8
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High CPU utilization on ScyllaDB node"
      description: "CPU utilization is {{ $value }} on {{ $labels.instance }}"

  - alert: ScyllaHighLatency
    expr: scylla_storage_proxy_coordinator_read_latency{quantile="0.99"} > 100000
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "High read latency detected"
      description: "99th percentile read latency is {{ $value }}us on {{ $labels.instance }}"

  - alert: ScyllaLowDiskSpace
    expr: scylla_node_filesystem_avail_bytes / scylla_node_filesystem_size_bytes < 0.1
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Low disk space on ScyllaDB node"
      description: "Available disk space is below 10% on {{ $labels.instance }}"

  - alert: ScyllaCompactionBacklog
    expr: scylla_compaction_manager_pending_tasks > 100
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "High compaction backlog"
      description: "Compaction backlog has {{ $value }} pending tasks on {{ $labels.instance }}"
EOF

chown -R "$PROMETHEUS_USER":"$PROMETHEUS_USER" "$PROMETHEUS_CONFIG"
chmod 644 "$PROMETHEUS_CONFIG"/*.yml

print_status "[7/8] Creating systemd service..."
cat > /etc/systemd/system/prometheus.service << EOF
[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=$PROMETHEUS_USER
Group=$PROMETHEUS_USER
Type=simple
ExecStart=$PROMETHEUS_HOME/prometheus \\
    --config.file=$PROMETHEUS_CONFIG/prometheus.yml \\
    --storage.tsdb.path=$PROMETHEUS_DATA \\
    --web.console.templates=$PROMETHEUS_HOME/consoles \\
    --web.console.libraries=$PROMETHEUS_HOME/console_libraries \\
    --web.listen-address=0.0.0.0:9090 \\
    --web.enable-lifecycle
ExecReload=/bin/kill -HUP \$MAINPID
Restart=always

[Install]
WantedBy=multi-user.target
EOF

print_status "[8/8] Starting and enabling Prometheus service..."
systemctl daemon-reload
systemctl enable prometheus
systemctl start prometheus

# Configure firewall
if command -v firewall-cmd &> /dev/null && systemctl is-active firewalld &> /dev/null; then
    firewall-cmd --permanent --add-port=9090/tcp
    firewall-cmd --reload
elif command -v ufw &> /dev/null && ufw status | grep -q "Status: active"; then
    ufw allow 9090/tcp
fi

# Cleanup temporary files
rm -f "/tmp/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"

print_status "Verifying installation..."
if systemctl is-active prometheus &> /dev/null; then
    print_status "✓ Prometheus service is running"
else
    print_error "✗ Prometheus service failed to start"
    exit 1
fi

if promtool check config "$PROMETHEUS_CONFIG/prometheus.yml" &> /dev/null; then
    print_status "✓ Prometheus configuration is valid"
else
    print_error "✗ Prometheus configuration is invalid"
    exit 1
fi

print_status "ScyllaDB monitoring setup completed successfully!"
echo
echo "Prometheus is now running on: http://$(hostname -I | awk '{print $1}'):9090"
echo "Configuration files:"
echo "  - Main config: $PROMETHEUS_CONFIG/prometheus.yml"
echo "  - Alert rules: $PROMETHEUS_CONFIG/scylla_rules.yml"
echo
if [[ -z "$SCYLLA_NODES" ]]; then
    print_warning "No ScyllaDB nodes specified. Edit $PROMETHEUS_CONFIG/prometheus.yml to add your ScyllaDB nodes."
fi
echo "Next steps:"
echo "1. Install and configure Grafana for visualization"
echo "2. Install Alertmanager for alert notifications"
echo "3. Verify ScyllaDB nodes are exposing metrics on port 9180"