Monitor Apache Cassandra cluster with Prometheus and Grafana dashboards

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Default configuration
CASSANDRA_USER="${CASSANDRA_USER:-cassandra}"
JMX_EXPORTER_PORT="${JMX_EXPORTER_PORT:-7070}"
PROMETHEUS_PORT="${PROMETHEUS_PORT:-9090}"

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Install and configure Cassandra monitoring with Prometheus and Grafana"
    echo ""
    echo "Options:"
    echo "  -h, --help              Show this help message"
    echo "  --cassandra-user USER   Cassandra user (default: cassandra)"
    echo "  --jmx-port PORT         JMX exporter port (default: 7070)"
    echo "  --prometheus-port PORT  Prometheus port (default: 9090)"
    echo ""
    exit 1
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            usage
            ;;
        --cassandra-user)
            CASSANDRA_USER="$2"
            shift 2
            ;;
        --jmx-port)
            JMX_EXPORTER_PORT="$2"
            shift 2
            ;;
        --prometheus-port)
            PROMETHEUS_PORT="$2"
            shift 2
            ;;
        *)
            echo -e "${RED}Error: Unknown option $1${NC}"
            usage
            ;;
    esac
done

# Cleanup function for trap
cleanup() {
    echo -e "${RED}Installation failed! Cleaning up...${NC}"
    systemctl stop prometheus 2>/dev/null || true
    systemctl stop cassandra 2>/dev/null || true
}

trap cleanup ERR

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}Error: This script must be run as root or with sudo${NC}"
    exit 1
fi

# Detect distribution
echo -e "${YELLOW}[1/10] Detecting Linux distribution...${NC}"
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            PROMETHEUS_PKG="prometheus"
            PROMETHEUS_CONFIG="/etc/prometheus/prometheus.yml"
            PROMETHEUS_SERVICE="prometheus"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            PROMETHEUS_PKG="prometheus2"
            PROMETHEUS_CONFIG="/etc/prometheus/prometheus.yml"
            PROMETHEUS_SERVICE="prometheus"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            PROMETHEUS_PKG="prometheus2"
            PROMETHEUS_CONFIG="/etc/prometheus/prometheus.yml"
            PROMETHEUS_SERVICE="prometheus"
            ;;
        *) 
            echo -e "${RED}Error: Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
    echo -e "${GREEN}Detected: $PRETTY_NAME${NC}"
else
    echo -e "${RED}Error: Cannot detect Linux distribution${NC}"
    exit 1
fi

# Check prerequisites
echo -e "${YELLOW}[2/10] Checking prerequisites...${NC}"
command -v wget >/dev/null 2>&1 || $PKG_INSTALL wget
command -v curl >/dev/null 2>&1 || $PKG_INSTALL curl
command -v ss >/dev/null 2>&1 || $PKG_INSTALL iproute2

# Check if Cassandra user exists
if ! id "$CASSANDRA_USER" &>/dev/null; then
    echo -e "${RED}Error: Cassandra user '$CASSANDRA_USER' does not exist${NC}"
    echo "Please install Cassandra first or specify correct user with --cassandra-user"
    exit 1
fi

# Download JMX Prometheus exporter
echo -e "${YELLOW}[3/10] Downloading JMX Prometheus exporter...${NC}"
cd /opt
if [ ! -f jmx_prometheus_javaagent-0.20.0.jar ]; then
    wget -q https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.20.0/jmx_prometheus_javaagent-0.20.0.jar
    chown $CASSANDRA_USER:$CASSANDRA_USER jmx_prometheus_javaagent-0.20.0.jar
    chmod 644 jmx_prometheus_javaagent-0.20.0.jar
    echo -e "${GREEN}JMX exporter downloaded successfully${NC}"
else
    echo -e "${GREEN}JMX exporter already exists${NC}"
fi

# Create JMX exporter configuration
echo -e "${YELLOW}[4/10] Creating JMX exporter configuration...${NC}"
cat > /opt/cassandra-jmx-config.yaml << 'EOF'
rules:
  # Node health metrics
  - pattern: "org.apache.cassandra.metrics<type=Storage, name=Load><>Value"
    name: cassandra_storage_load_bytes
    help: "Total disk space used by node in bytes"
  
  # Read/Write latency
  - pattern: "org.apache.cassandra.metrics<type=ClientRequest, scope=([^,]+), name=Latency><>Count"
    name: cassandra_client_request_latency_total
    labels:
      request_type: "$1"
    help: "Total client request latency count"
  
  - pattern: "org.apache.cassandra.metrics<type=ClientRequest, scope=([^,]+), name=Latency><>(Mean|95thPercentile|99thPercentile)"
    name: cassandra_client_request_latency_seconds
    type: GAUGE
    labels:
      request_type: "$1"
      quantile: "$2"
    help: "Client request latency in seconds"
    valueFactor: 0.000001
  
  # Connection metrics
  - pattern: "org.apache.cassandra.metrics<type=Connection, scope=([^,]+), name=([^,]+)><>Value"
    name: cassandra_connection_$2
    labels:
      connection_type: "$1"
    help: "Cassandra connection metrics"
  
  # Thread pool metrics
  - pattern: "org.apache.cassandra.metrics<type=ThreadPools, path=([^,]+), scope=([^,]+), name=([^,]+)><>Value"
    name: cassandra_threadpool_$3
    labels:
      pool_type: "$1"
      pool_name: "$2"
    help: "Cassandra thread pool metrics"
  
  # Compaction metrics
  - pattern: "org.apache.cassandra.metrics<type=Compaction, name=([^,]+)><>(Count|Value)"
    name: cassandra_compaction_$1
    help: "Cassandra compaction metrics"
  
  # Cache metrics
  - pattern: "org.apache.cassandra.metrics<type=Cache, scope=([^,]+), name=([^,]+)><>(Count|Value)"
    name: cassandra_cache_$2
    labels:
      cache_name: "$1"
    help: "Cassandra cache metrics"
EOF

chown $CASSANDRA_USER:$CASSANDRA_USER /opt/cassandra-jmx-config.yaml
chmod 644 /opt/cassandra-jmx-config.yaml

# Configure Cassandra with JMX exporter
echo -e "${YELLOW}[5/10] Configuring Cassandra with JMX exporter...${NC}"
JVM_OPTS_FILE=""
if [ -f /etc/cassandra/cassandra-env.sh ]; then
    JVM_OPTS_FILE="/etc/cassandra/cassandra-env.sh"
elif [ -f /etc/cassandra/conf/cassandra-env.sh ]; then
    JVM_OPTS_FILE="/etc/cassandra/conf/cassandra-env.sh"
elif [ -f /opt/cassandra/conf/cassandra-env.sh ]; then
    JVM_OPTS_FILE="/opt/cassandra/conf/cassandra-env.sh"
else
    echo -e "${RED}Error: Cannot find cassandra-env.sh file${NC}"
    exit 1
fi

if ! grep -q "jmx_prometheus_javaagent" "$JVM_OPTS_FILE"; then
    cp "$JVM_OPTS_FILE" "$JVM_OPTS_FILE.backup"
    echo "" >> "$JVM_OPTS_FILE"
    echo "# JMX Prometheus exporter" >> "$JVM_OPTS_FILE"
    echo "JVM_OPTS=\"\$JVM_OPTS -javaagent:/opt/jmx_prometheus_javaagent-0.20.0.jar=$JMX_EXPORTER_PORT:/opt/cassandra-jmx-config.yaml\"" >> "$JVM_OPTS_FILE"
    echo -e "${GREEN}JMX exporter added to Cassandra configuration${NC}"
else
    echo -e "${GREEN}JMX exporter already configured${NC}"
fi

# Install Prometheus
echo -e "${YELLOW}[6/10] Installing Prometheus...${NC}"
if [ "$PKG_MGR" = "apt" ]; then
    $PKG_UPDATE
fi
$PKG_INSTALL $PROMETHEUS_PKG

# Create Prometheus configuration
echo -e "${YELLOW}[7/10] Configuring Prometheus...${NC}"
mkdir -p $(dirname $PROMETHEUS_CONFIG)
cat > $PROMETHEUS_CONFIG << EOF
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:$PROMETHEUS_PORT']
  
  - job_name: 'cassandra-cluster'
    static_configs:
      - targets: ['localhost:$JMX_EXPORTER_PORT']
    scrape_interval: 30s
    metrics_path: /metrics
EOF

chown prometheus:prometheus $PROMETHEUS_CONFIG 2>/dev/null || chown root:root $PROMETHEUS_CONFIG
chmod 644 $PROMETHEUS_CONFIG

# Configure firewall
echo -e "${YELLOW}[8/10] Configuring firewall...${NC}"
if command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active --quiet firewalld; then
    firewall-cmd --permanent --add-port=$JMX_EXPORTER_PORT/tcp
    firewall-cmd --permanent --add-port=$PROMETHEUS_PORT/tcp
    firewall-cmd --reload
    echo -e "${GREEN}Firewalld configured${NC}"
elif command -v ufw >/dev/null 2>&1 && ufw status | grep -q "Status: active"; then
    ufw allow $JMX_EXPORTER_PORT/tcp
    ufw allow $PROMETHEUS_PORT/tcp
    echo -e "${GREEN}UFW configured${NC}"
else
    echo -e "${YELLOW}No active firewall detected, skipping firewall configuration${NC}"
fi

# Start services
echo -e "${YELLOW}[9/10] Starting services...${NC}"
systemctl restart cassandra
systemctl enable prometheus
systemctl start prometheus

# Wait for services to start
sleep 10

# Verify installation
echo -e "${YELLOW}[10/10] Verifying installation...${NC}"

# Check Cassandra is running
if systemctl is-active --quiet cassandra; then
    echo -e "${GREEN}✓ Cassandra is running${NC}"
else
    echo -e "${RED}✗ Cassandra is not running${NC}"
    exit 1
fi

# Check JMX exporter port
if ss -tlnp | grep -q ":$JMX_EXPORTER_PORT "; then
    echo -e "${GREEN}✓ JMX exporter is listening on port $JMX_EXPORTER_PORT${NC}"
else
    echo -e "${RED}✗ JMX exporter is not listening on port $JMX_EXPORTER_PORT${NC}"
    exit 1
fi

# Check Prometheus is running
if systemctl is-active --quiet prometheus; then
    echo -e "${GREEN}✓ Prometheus is running${NC}"
else
    echo -e "${RED}✗ Prometheus is not running${NC}"
    exit 1
fi

# Test metrics endpoint
if curl -s http://localhost:$JMX_EXPORTER_PORT/metrics | grep -q "cassandra_"; then
    echo -e "${GREEN}✓ Cassandra metrics are being exposed${NC}"
else
    echo -e "${YELLOW}⚠ Cassandra metrics not yet available (may need more time to start)${NC}"
fi

echo -e "${GREEN}"
echo "=================================="
echo "Installation completed successfully!"
echo "=================================="
echo -e "${NC}"
echo "Access points:"
echo "- Cassandra JMX metrics: http://localhost:$JMX_EXPORTER_PORT/metrics"
echo "- Prometheus: http://localhost:$PROMETHEUS_PORT"
echo ""
echo "Configuration files:"
echo "- JMX exporter config: /opt/cassandra-jmx-config.yaml"
echo "- Prometheus config: $PROMETHEUS_CONFIG"
echo ""
echo "Next steps:"
echo "1. Install Grafana for visualization"
echo "2. Import Cassandra dashboard templates"
echo "3. Configure alerting rules"