Configure Keycloak high availability clustering for production with load balancing and failover

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Global variables
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOG_FILE="/var/log/keycloak-cluster-install.log"

# Default configuration
CLUSTER_SUBNET="${CLUSTER_SUBNET:-203.0.113.0/24}"
NODE_TYPE="${1:-}"
NODE_IP="${2:-}"
KC_DB_PASSWORD="${KC_DB_PASSWORD:-$(openssl rand -base64 32)}"

usage() {
    echo "Usage: $0 <node_type> <node_ip> [cluster_subnet]"
    echo "Node types: haproxy, keycloak, postgres-primary, postgres-replica"
    echo "Example: $0 keycloak 203.0.113.20 203.0.113.0/24"
    exit 1
}

log() {
    echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$LOG_FILE"
}

warn() {
    echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE"
    exit 1
}

cleanup() {
    if [[ $? -ne 0 ]]; then
        error "Installation failed. Check $LOG_FILE for details."
    fi
}

trap cleanup ERR

detect_distro() {
    if [[ ! -f /etc/os-release ]]; then
        error "/etc/os-release not found. Cannot detect distribution."
    fi
    
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            FIREWALL_CMD="ufw"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            FIREWALL_CMD="firewalld"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            FIREWALL_CMD="firewalld"
            ;;
        *)
            error "Unsupported distribution: $ID"
            ;;
    esac
}

check_prerequisites() {
    log "[1/12] Checking prerequisites..."
    
    if [[ $EUID -ne 0 ]]; then
        error "This script must be run as root"
    fi
    
    if [[ -z "$NODE_TYPE" || -z "$NODE_IP" ]]; then
        usage
    fi
    
    if ! command -v openssl &> /dev/null; then
        $PKG_INSTALL openssl
    fi
}

configure_hosts() {
    log "[2/12] Configuring hosts file..."
    
    cat >> /etc/hosts << EOF
# Keycloak Cluster Configuration
203.0.113.10 haproxy1.example.com haproxy1
203.0.113.11 haproxy2.example.com haproxy2
203.0.113.20 keycloak1.example.com keycloak1
203.0.113.21 keycloak2.example.com keycloak2
203.0.113.22 keycloak3.example.com keycloak3
203.0.113.30 postgres1.example.com postgres1
203.0.113.31 postgres2.example.com postgres2
EOF
}

configure_firewall() {
    log "[3/12] Configuring firewall..."
    
    case "$FIREWALL_CMD" in
        ufw)
            systemctl enable --now ufw
            ufw --force enable
            ufw allow from $CLUSTER_SUBNET to any port 8080 comment "Keycloak HTTP"
            ufw allow from $CLUSTER_SUBNET to any port 7800 comment "Keycloak clustering"
            ufw allow from $CLUSTER_SUBNET to any port 5432 comment "PostgreSQL"
            ufw allow from $CLUSTER_SUBNET to any port 8404 comment "HAProxy stats"
            ufw reload
            ;;
        firewalld)
            systemctl enable --now firewalld
            firewall-cmd --permanent --add-rich-rule="rule family=ipv4 source address=$CLUSTER_SUBNET port protocol=tcp port=8080 accept"
            firewall-cmd --permanent --add-rich-rule="rule family=ipv4 source address=$CLUSTER_SUBNET port protocol=tcp port=7800 accept"
            firewall-cmd --permanent --add-rich-rule="rule family=ipv4 source address=$CLUSTER_SUBNET port protocol=tcp port=5432 accept"
            firewall-cmd --permanent --add-rich-rule="rule family=ipv4 source address=$CLUSTER_SUBNET port protocol=tcp port=8404 accept"
            firewall-cmd --reload
            ;;
    esac
}

install_java() {
    log "[4/12] Installing Java..."
    
    case "$PKG_MGR" in
        apt)
            $PKG_INSTALL openjdk-17-jdk
            ;;
        dnf|yum)
            $PKG_INSTALL java-17-openjdk-devel
            ;;
    esac
}

install_keycloak() {
    log "[5/12] Installing Keycloak..."
    
    KC_VERSION="23.0.3"
    cd /opt
    wget -q "https://github.com/keycloak/keycloak/releases/download/$KC_VERSION/keycloak-$KC_VERSION.tar.gz"
    tar -xzf "keycloak-$KC_VERSION.tar.gz"
    mv "keycloak-$KC_VERSION" keycloak
    chown -R root:root /opt/keycloak
    chmod -R 755 /opt/keycloak
    
    useradd -r -s /bin/false keycloak || true
    chown -R keycloak:keycloak /opt/keycloak
}

configure_keycloak() {
    log "[6/12] Configuring Keycloak cluster..."
    
    mkdir -p /opt/keycloak/conf
    cat > /opt/keycloak/conf/keycloak.conf << EOF
# Database configuration
db=postgres
db-url=jdbc:postgresql://postgres1.example.com:5432/keycloak
db-username=keycloak_user
db-password=$KC_DB_PASSWORD

# Cluster configuration
cache=ispn
cache-config-file=cluster-ispn.xml

# Network configuration
hostname=$NODE_IP
http-enabled=true
http-port=8080

# Clustering
jgroups-bind-addr=$NODE_IP
jgroups-bind-port=7800
EOF

    cat > /opt/keycloak/conf/cluster-ispn.xml << 'EOF'
<infinispan xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
           xsi:schemaLocation="urn:infinispan:config:13.0 http://www.infinispan.org/schemas/infinispan-config-13.0.xsd"
           xmlns="urn:infinispan:config:13.0">
    <jgroups>
        <stack name="tcp" extends="tcp">
            <TCP bind_addr="${jgroups.bind.address:127.0.0.1}" bind_port="${jgroups.bind.port:7800}"/>
            <TCPPING initial_hosts="keycloak1[7800],keycloak2[7800],keycloak3[7800]" port_range="0"/>
        </stack>
    </jgroups>
    <cache-container name="keycloak">
        <transport cluster="${keycloak.cache.cluster.name:keycloak}" stack="tcp"/>
        <local-cache name="realms"/>
        <local-cache name="users"/>
        <distributed-cache name="sessions"/>
        <distributed-cache name="authenticationSessions"/>
        <distributed-cache name="offlineSessions"/>
        <distributed-cache name="clientSessions"/>
        <distributed-cache name="offlineClientSessions"/>
        <distributed-cache name="loginFailures"/>
        <distributed-cache name="actionTokens"/>
    </cache-container>
</infinispan>
EOF

    chown keycloak:keycloak /opt/keycloak/conf/*
    chmod 640 /opt/keycloak/conf/*
}

install_postgres() {
    log "[7/12] Installing PostgreSQL..."
    
    case "$PKG_MGR" in
        apt)
            $PKG_INSTALL postgresql postgresql-contrib
            ;;
        dnf|yum)
            $PKG_INSTALL postgresql-server postgresql-contrib
            postgresql-setup --initdb
            ;;
    esac
    
    systemctl enable --now postgresql
}

configure_postgres_primary() {
    log "[8/12] Configuring PostgreSQL primary..."
    
    sudo -u postgres psql << EOF
CREATE DATABASE keycloak;
CREATE USER keycloak_user WITH ENCRYPTED PASSWORD '$KC_DB_PASSWORD';
GRANT ALL PRIVILEGES ON DATABASE keycloak TO keycloak_user;
CREATE USER replicator WITH REPLICATION ENCRYPTED PASSWORD 'repl_pass_2024!';
EOF

    case "$PKG_MGR" in
        apt)
            PG_CONF="/etc/postgresql/*/main/postgresql.conf"
            PG_HBA="/etc/postgresql/*/main/pg_hba.conf"
            ;;
        dnf|yum)
            PG_CONF="/var/lib/pgsql/data/postgresql.conf"
            PG_HBA="/var/lib/pgsql/data/pg_hba.conf"
            ;;
    esac

    cp $PG_CONF $PG_CONF.backup
    cat >> $PG_CONF << EOF

# Cluster configuration
max_connections = 200
shared_buffers = 256MB
effective_cache_size = 1GB
wal_level = replica
max_wal_senders = 3
max_replication_slots = 3
listen_addresses = '*'
EOF

    cp $PG_HBA $PG_HBA.backup
    cat >> $PG_HBA << EOF
host    keycloak        keycloak_user   $CLUSTER_SUBNET         scram-sha-256
host    replication     replicator      203.0.113.31/32         scram-sha-256
EOF

    systemctl restart postgresql
}

install_haproxy() {
    log "[9/12] Installing HAProxy..."
    
    $PKG_INSTALL haproxy
    
    cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.backup
    cat > /etc/haproxy/haproxy.cfg << EOF
global
    daemon
    maxconn 4096
    
defaults
    mode http
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms
    
frontend keycloak_frontend
    bind *:80
    default_backend keycloak_backend
    
backend keycloak_backend
    balance roundrobin
    option httpchk GET /auth/realms/master
    server keycloak1 keycloak1:8080 check
    server keycloak2 keycloak2:8080 check
    server keycloak3 keycloak3:8080 check
    
listen stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s
EOF

    systemctl enable --now haproxy
}

create_systemd_service() {
    log "[10/12] Creating systemd service..."
    
    if [[ "$NODE_TYPE" == "keycloak" ]]; then
        cat > /etc/systemd/system/keycloak.service << EOF
[Unit]
Description=Keycloak Identity Provider
After=network.target postgresql.service
Wants=network.target

[Service]
Type=notify
User=keycloak
Group=keycloak
ExecStart=/opt/keycloak/bin/kc.sh start --optimized
Restart=always
RestartSec=5
StandardOutput=journal
StandardError=journal
SyslogIdentifier=keycloak

[Install]
WantedBy=multi-user.target
EOF

        systemctl daemon-reload
        systemctl enable keycloak
    fi
}

start_services() {
    log "[11/12] Starting services..."
    
    case "$NODE_TYPE" in
        keycloak)
            /opt/keycloak/bin/kc.sh build
            systemctl start keycloak
            ;;
        postgres-primary)
            systemctl restart postgresql
            ;;
        haproxy)
            systemctl restart haproxy
            ;;
    esac
}

verify_installation() {
    log "[12/12] Verifying installation..."
    
    case "$NODE_TYPE" in
        keycloak)
            if systemctl is-active --quiet keycloak; then
                log "Keycloak service is running"
            else
                error "Keycloak service failed to start"
            fi
            ;;
        postgres-primary)
            if systemctl is-active --quiet postgresql; then
                log "PostgreSQL service is running"
            else
                error "PostgreSQL service failed to start"
            fi
            ;;
        haproxy)
            if systemctl is-active --quiet haproxy; then
                log "HAProxy service is running"
            else
                error "HAProxy service failed to start"
            fi
            ;;
    esac
}

main() {
    detect_distro
    check_prerequisites
    configure_hosts
    configure_firewall
    
    $PKG_UPDATE
    
    case "$NODE_TYPE" in
        keycloak)
            install_java
            install_keycloak
            configure_keycloak
            create_systemd_service
            start_services
            ;;
        postgres-primary)
            install_postgres
            configure_postgres_primary
            start_services
            ;;
        haproxy)
            install_haproxy
            start_services
            ;;
        *)
            error "Invalid node type: $NODE_TYPE"
            ;;
    esac
    
    verify_installation
    
    log "Installation completed successfully!"
    log "Database password: $KC_DB_PASSWORD"
    log "Log file: $LOG_FILE"
}

main "$@"