Set up Grafana Enterprise high availability clustering with PostgreSQL backend and load balancing

#!/usr/bin/env bash
set -euo pipefail

# Grafana Enterprise HA Cluster Installation Script
# Supports: Ubuntu, Debian, AlmaLinux, Rocky Linux, CentOS, RHEL

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Default values
ROLE=""
PRIMARY_DB_IP=""
GRAFANA_DB_PASS=""
REPLICATION_PASS=""

usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  --role [primary-db|standby-db|grafana|haproxy]  Required: Server role"
    echo "  --primary-db-ip IP                              Primary DB IP (required for standby-db/grafana)"
    echo "  --grafana-db-pass PASSWORD                      Grafana database password"
    echo "  --replication-pass PASSWORD                     Replication user password"
    echo ""
    echo "Examples:"
    echo "  $0 --role primary-db --grafana-db-pass mypass --replication-pass replpass"
    echo "  $0 --role standby-db --primary-db-ip 10.0.1.10 --replication-pass replpass"
    echo "  $0 --role grafana --primary-db-ip 10.0.1.10 --grafana-db-pass mypass"
    exit 1
}

log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

cleanup() {
    log_error "Installation failed. Check logs above for details."
    exit 1
}
trap cleanup ERR

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --role) ROLE="$2"; shift 2 ;;
        --primary-db-ip) PRIMARY_DB_IP="$2"; shift 2 ;;
        --grafana-db-pass) GRAFANA_DB_PASS="$2"; shift 2 ;;
        --replication-pass) REPLICATION_PASS="$2"; shift 2 ;;
        -h|--help) usage ;;
        *) log_error "Unknown option: $1"; usage ;;
    esac
done

[[ -z "$ROLE" ]] && { log_error "Role is required"; usage; }

# Detect distro and package manager
if [[ ! -f /etc/os-release ]]; then
    log_error "Cannot detect OS distribution"
    exit 1
fi

. /etc/os-release
case "$ID" in
    ubuntu|debian)
        PKG_MGR="apt"
        PKG_UPDATE="apt update"
        PKG_INSTALL="apt install -y"
        PG_VERSION="14"
        PG_DATA_DIR="/var/lib/postgresql/$PG_VERSION/main"
        PG_CONFIG_DIR="/etc/postgresql/$PG_VERSION/main"
        PG_USER="postgres"
        ;;
    almalinux|rocky|centos|rhel|ol|fedora)
        PKG_MGR="dnf"
        PKG_UPDATE="dnf update -y"
        PKG_INSTALL="dnf install -y"
        PG_VERSION=""
        PG_DATA_DIR="/var/lib/pgsql/data"
        PG_CONFIG_DIR="/var/lib/pgsql/data"
        PG_USER="postgres"
        ;;
    amzn)
        PKG_MGR="yum"
        PKG_UPDATE="yum update -y"
        PKG_INSTALL="yum install -y"
        PG_DATA_DIR="/var/lib/pgsql/data"
        PG_CONFIG_DIR="/var/lib/pgsql/data"
        PG_USER="postgres"
        ;;
    *)
        log_error "Unsupported distribution: $ID"
        exit 1
        ;;
esac

# Check prerequisites
if [[ $EUID -ne 0 ]]; then
    log_error "This script must be run as root"
    exit 1
fi

install_postgresql() {
    echo "[1/6] Installing PostgreSQL..."
    $PKG_UPDATE
    
    if [[ "$ID" =~ ^(ubuntu|debian)$ ]]; then
        $PKG_INSTALL postgresql postgresql-contrib postgresql-client
    else
        $PKG_INSTALL postgresql-server postgresql-contrib postgresql
        postgresql-setup --initdb || true
    fi
    
    systemctl enable --now postgresql
    log_info "PostgreSQL installed and started"
}

configure_primary_db() {
    echo "[2/6] Configuring primary PostgreSQL..."
    
    # Backup original config
    cp "$PG_CONFIG_DIR/postgresql.conf" "$PG_CONFIG_DIR/postgresql.conf.backup"
    
    # Configure PostgreSQL for replication
    cat >> "$PG_CONFIG_DIR/postgresql.conf" << EOF
# Grafana HA Configuration
listen_addresses = '*'
wal_level = replica
max_wal_senders = 3
max_replication_slots = 3
archive_mode = on
archive_command = 'cp %p $PG_DATA_DIR/archive/%f'
log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '
log_checkpoints = on
log_connections = on
log_disconnections = on
EOF

    # Create archive directory
    mkdir -p "$PG_DATA_DIR/archive"
    chown "$PG_USER:$PG_USER" "$PG_DATA_DIR/archive"
    chmod 750 "$PG_DATA_DIR/archive"
    
    systemctl restart postgresql
    log_info "Primary PostgreSQL configured"
}

configure_pg_hba() {
    echo "[3/6] Configuring PostgreSQL authentication..."
    
    # Backup original pg_hba.conf
    cp "$PG_CONFIG_DIR/pg_hba.conf" "$PG_CONFIG_DIR/pg_hba.conf.backup"
    
    # Add Grafana and replication access
    cat >> "$PG_CONFIG_DIR/pg_hba.conf" << EOF
# Grafana connections
host    grafana         grafana_user    0.0.0.0/0         md5
# Replication connections
host    replication     replicator      0.0.0.0/0         md5
EOF
    
    systemctl reload postgresql
    log_info "PostgreSQL authentication configured"
}

create_grafana_database() {
    echo "[4/6] Creating Grafana database and user..."
    
    [[ -z "$GRAFANA_DB_PASS" ]] && { log_error "Grafana database password required"; exit 1; }
    
    sudo -u "$PG_USER" psql << EOF
CREATE DATABASE grafana;
CREATE USER grafana_user WITH ENCRYPTED PASSWORD '$GRAFANA_DB_PASS';
GRANT ALL PRIVILEGES ON DATABASE grafana TO grafana_user;
ALTER USER grafana_user CREATEDB;
EOF
    
    log_info "Grafana database created"
}

create_replication_user() {
    echo "[5/6] Creating replication user..."
    
    [[ -z "$REPLICATION_PASS" ]] && { log_error "Replication password required"; exit 1; }
    
    sudo -u "$PG_USER" psql << EOF
CREATE USER replicator WITH REPLICATION ENCRYPTED PASSWORD '$REPLICATION_PASS';
EOF
    
    log_info "Replication user created"
}

setup_standby_db() {
    echo "[1/4] Setting up PostgreSQL standby..."
    
    [[ -z "$PRIMARY_DB_IP" ]] && { log_error "Primary DB IP required for standby"; exit 1; }
    [[ -z "$REPLICATION_PASS" ]] && { log_error "Replication password required"; exit 1; }
    
    install_postgresql
    
    systemctl stop postgresql
    
    # Remove existing data
    rm -rf "$PG_DATA_DIR"/*
    
    # Create base backup
    sudo -u "$PG_USER" PGPASSWORD="$REPLICATION_PASS" pg_basebackup -h "$PRIMARY_DB_IP" -D "$PG_DATA_DIR" -U replicator -v -P -W
    
    # Create standby signal
    sudo -u "$PG_USER" touch "$PG_DATA_DIR/standby.signal"
    
    echo "[2/4] Configuring standby recovery..."
    cat > "$PG_DATA_DIR/postgresql.auto.conf" << EOF
primary_conninfo = 'host=$PRIMARY_DB_IP port=5432 user=replicator password=$REPLICATION_PASS'
restore_command = 'cp $PG_DATA_DIR/archive/%f %p'
recovery_target_timeline = 'latest'
EOF
    
    chown "$PG_USER:$PG_USER" "$PG_DATA_DIR/postgresql.auto.conf"
    chmod 600 "$PG_DATA_DIR/postgresql.auto.conf"
    
    systemctl start postgresql
    log_info "PostgreSQL standby configured"
}

install_grafana() {
    echo "[1/3] Installing Grafana Enterprise..."
    
    [[ -z "$PRIMARY_DB_IP" ]] && { log_error "Primary DB IP required for Grafana"; exit 1; }
    [[ -z "$GRAFANA_DB_PASS" ]] && { log_error "Grafana database password required"; exit 1; }
    
    if [[ "$ID" =~ ^(ubuntu|debian)$ ]]; then
        wget -q -O /usr/share/keyrings/grafana.key https://apt.grafana.com/gpg.key
        echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list
        $PKG_UPDATE
        $PKG_INSTALL grafana-enterprise
    else
        cat > /etc/yum.repos.d/grafana.repo << EOF
[grafana]
name=grafana
baseurl=https://rpm.grafana.com
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://rpm.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
        $PKG_INSTALL grafana-enterprise
    fi
    
    echo "[2/3] Configuring Grafana for PostgreSQL..."
    cat > /etc/grafana/grafana.ini << EOF
[database]
type = postgres
host = $PRIMARY_DB_IP:5432
name = grafana
user = grafana_user
password = $GRAFANA_DB_PASS

[security]
admin_user = admin
admin_password = admin

[server]
http_port = 3000

[clustering]
enabled = true
EOF
    
    chown grafana:grafana /etc/grafana/grafana.ini
    chmod 640 /etc/grafana/grafana.ini
    
    systemctl enable --now grafana-server
    log_info "Grafana Enterprise installed and configured"
}

case "$ROLE" in
    primary-db)
        install_postgresql
        configure_primary_db
        configure_pg_hba
        create_grafana_database
        create_replication_user
        echo "[6/6] Opening firewall for PostgreSQL..."
        if command -v firewall-cmd &> /dev/null; then
            firewall-cmd --permanent --add-port=5432/tcp
            firewall-cmd --reload
        elif command -v ufw &> /dev/null; then
            ufw allow 5432/tcp
        fi
        log_info "Primary PostgreSQL database setup complete"
        ;;
    standby-db)
        setup_standby_db
        echo "[3/4] Opening firewall for PostgreSQL..."
        if command -v firewall-cmd &> /dev/null; then
            firewall-cmd --permanent --add-port=5432/tcp
            firewall-cmd --reload
        elif command -v ufw &> /dev/null; then
            ufw allow 5432/tcp
        fi
        echo "[4/4] Verifying replication status..."
        sudo -u "$PG_USER" psql -c "SELECT pg_is_in_recovery();" || log_warn "Could not verify replication status"
        log_info "Standby PostgreSQL database setup complete"
        ;;
    grafana)
        install_grafana
        echo "[3/3] Opening firewall for Grafana..."
        if command -v firewall-cmd &> /dev/null; then
            firewall-cmd --permanent --add-port=3000/tcp
            firewall-cmd --reload
        elif command -v ufw &> /dev/null; then
            ufw allow 3000/tcp
        fi
        log_info "Grafana Enterprise setup complete"
        log_info "Access Grafana at http://$(hostname -I | awk '{print $1}'):3000"
        ;;
    *)
        log_error "Invalid role: $ROLE"
        usage
        ;;
esac

log_info "Installation completed successfully!"