Set up HAProxy high availability with keepalived clustering for automatic failover

#!/usr/bin/env bash

set -euo pipefail

# HAProxy High Availability with Keepalived Setup Script
# Usage: ./setup_haproxy_ha.sh <role> <primary_ip> <backup_ip> <vip> [backend1_ip] [backend2_ip]
# Role: primary or backup

# Color codes
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

log() { echo -e "${GREEN}[INFO]${NC} $1"; }
warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }

usage() {
    cat << EOF
Usage: $0 <role> <primary_ip> <backup_ip> <vip> [backend1_ip] [backend2_ip]

Parameters:
  role         - Node role: 'primary' or 'backup'
  primary_ip   - IP address of primary HAProxy node
  backup_ip    - IP address of backup HAProxy node
  vip          - Virtual IP address for failover
  backend1_ip  - First backend server IP (default: 192.168.1.20)
  backend2_ip  - Second backend server IP (default: 192.168.1.21)

Example:
  $0 primary 203.0.113.10 203.0.113.11 203.0.113.100 203.0.113.20 203.0.113.21
EOF
    exit 1
}

cleanup() {
    error "Installation failed. Performing cleanup..."
    systemctl stop haproxy keepalived 2>/dev/null || true
    exit 1
}

trap cleanup ERR

# Parameter validation
[[ $# -lt 4 || $# -gt 6 ]] && usage
[[ ! "$1" =~ ^(primary|backup)$ ]] && { error "Role must be 'primary' or 'backup'"; usage; }

ROLE="$1"
PRIMARY_IP="$2"
BACKUP_IP="$3"
VIP="$4"
BACKEND1_IP="${5:-192.168.1.20}"
BACKEND2_IP="${6:-192.168.1.21}"

# Check if running as root
[[ $EUID -ne 0 ]] && { error "This script must be run as root"; exit 1; }

# Detect distribution
if [[ -f /etc/os-release ]]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            PKG_UPGRADE="apt upgrade -y"
            HAPROXY_CONFIG="/etc/haproxy/haproxy.cfg"
            KEEPALIVED_CONFIG="/etc/keepalived/keepalived.conf"
            NETWORK_INTERFACE=$(ip route | grep default | awk '{print $5}' | head -1)
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf check-update || true"
            PKG_INSTALL="dnf install -y"
            PKG_UPGRADE="dnf upgrade -y"
            HAPROXY_CONFIG="/etc/haproxy/haproxy.cfg"
            KEEPALIVED_CONFIG="/etc/keepalived/keepalived.conf"
            NETWORK_INTERFACE=$(ip route | grep default | awk '{print $5}' | head -1)
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum check-update || true"
            PKG_INSTALL="yum install -y"
            PKG_UPGRADE="yum upgrade -y"
            HAPROXY_CONFIG="/etc/haproxy/haproxy.cfg"
            KEEPALIVED_CONFIG="/etc/keepalived/keepalived.conf"
            NETWORK_INTERFACE=$(ip route | grep default | awk '{print $5}' | head -1)
            ;;
        *)
            error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    error "Cannot detect distribution"
    exit 1
fi

log "[1/8] Updating system packages..."
$PKG_UPDATE
$PKG_UPGRADE

log "[2/8] Installing HAProxy and keepalived..."
$PKG_INSTALL haproxy keepalived

log "[3/8] Configuring kernel parameters..."
cat > /etc/sysctl.d/99-haproxy-keepalived.conf << 'EOF'
net.ipv4.ip_forward = 1
net.ipv4.ip_nonlocal_bind = 1
EOF
sysctl -p /etc/sysctl.d/99-haproxy-keepalived.conf

log "[4/8] Configuring HAProxy..."
cp "$HAPROXY_CONFIG" "${HAPROXY_CONFIG}.backup"
cat > "$HAPROXY_CONFIG" << EOF
global
    log stdout local0
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    mode http
    log global
    option httplog
    option dontlognull
    option redispatch
    retries 3
    timeout connect 5000
    timeout client 50000
    timeout server 50000

frontend web_frontend
    bind *:80
    bind *:443
    default_backend web_servers

backend web_servers
    balance roundrobin
    option httpchk GET /
    http-check expect status 200
    server web1 $BACKEND1_IP:80 check inter 3000 rise 2 fall 3
    server web2 $BACKEND2_IP:80 check inter 3000 rise 2 fall 3

listen stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s
    stats admin if TRUE
EOF
chown root:root "$HAPROXY_CONFIG"
chmod 644 "$HAPROXY_CONFIG"

log "[5/8] Creating keepalived notification scripts..."
mkdir -p /etc/keepalived/scripts
cat > /etc/keepalived/scripts/notify_master.sh << 'EOF'
#!/bin/bash
echo "$(date): Becoming MASTER" >> /var/log/keepalived-notifications.log
EOF

cat > /etc/keepalived/scripts/notify_backup.sh << 'EOF'
#!/bin/bash
echo "$(date): Becoming BACKUP" >> /var/log/keepalived-notifications.log
EOF

cat > /etc/keepalived/scripts/notify_fault.sh << 'EOF'
#!/bin/bash
echo "$(date): Entering FAULT state" >> /var/log/keepalived-notifications.log
EOF

chmod 755 /etc/keepalived/scripts/*.sh

log "[6/8] Configuring keepalived for $ROLE node..."
if [[ "$ROLE" == "primary" ]]; then
    PRIORITY=110
    STATE="MASTER"
else
    PRIORITY=100
    STATE="BACKUP"
fi

cat > "$KEEPALIVED_CONFIG" << EOF
global_defs {
    router_id LB_DEVEL_${ROLE^^}
    vrrp_skip_check_adv_addr
    vrrp_strict
    vrrp_garp_interval 0
    vrrp_gna_interval 0
}

vrrp_script chk_haproxy {
    script "/usr/bin/killall -0 haproxy"
    interval 2
    weight 2
    fall 3
    rise 2
}

vrrp_instance VI_1 {
    state $STATE
    interface $NETWORK_INTERFACE
    virtual_router_id 51
    priority $PRIORITY
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass $(openssl rand -base64 12)
    }
    virtual_ipaddress {
        $VIP
    }
    track_script {
        chk_haproxy
    }
    notify_master "/etc/keepalived/scripts/notify_master.sh"
    notify_backup "/etc/keepalived/scripts/notify_backup.sh"
    notify_fault "/etc/keepalived/scripts/notify_fault.sh"
}
EOF
chown root:root "$KEEPALIVED_CONFIG"
chmod 644 "$KEEPALIVED_CONFIG"

log "[7/8] Enabling and starting services..."
systemctl enable haproxy keepalived
systemctl restart haproxy keepalived

log "[8/8] Verifying installation..."
sleep 5

# Check if services are running
if systemctl is-active --quiet haproxy; then
    log "✓ HAProxy is running"
else
    error "✗ HAProxy failed to start"
    exit 1
fi

if systemctl is-active --quiet keepalived; then
    log "✓ Keepalived is running"
else
    error "✗ Keepalived failed to start"
    exit 1
fi

# Check if ports are listening
if ss -tlnp | grep -q ":80 "; then
    log "✓ HAProxy listening on port 80"
else
    warn "⚠ HAProxy not listening on port 80"
fi

if ss -tlnp | grep -q ":8404 "; then
    log "✓ HAProxy stats available on port 8404"
else
    warn "⚠ HAProxy stats not available on port 8404"
fi

log "HAProxy High Availability setup completed successfully!"
log "Role: $ROLE"
log "Virtual IP: $VIP"
log "Backend servers: $BACKEND1_IP, $BACKEND2_IP"
log "Stats URL: http://$VIP:8404/stats"
warn "Make sure to configure the same keepalived authentication password on both nodes"
warn "Verify backend servers are accessible before putting into production"