Configure keepalived with HAProxy backend health monitoring for high availability load balancing

#!/usr/bin/env bash
set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Global variables
SCRIPT_NAME=$(basename "$0")
PRIMARY_IP=""
BACKUP_IP=""
VIRTUAL_IP=""
INTERFACE="eth0"
DOMAIN="example.com"
NODE_TYPE=""

# Usage function
usage() {
    echo "Usage: $SCRIPT_NAME [OPTIONS]"
    echo "Options:"
    echo "  --primary-ip IP        Primary load balancer IP"
    echo "  --backup-ip IP         Backup load balancer IP" 
    echo "  --virtual-ip IP        Virtual IP address"
    echo "  --interface IF         Network interface (default: eth0)"
    echo "  --domain DOMAIN        Domain name (default: example.com)"
    echo "  --node-type TYPE       Node type: primary or backup"
    echo "  --help                 Show this help"
    exit 1
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --primary-ip) PRIMARY_IP="$2"; shift 2 ;;
        --backup-ip) BACKUP_IP="$2"; shift 2 ;;
        --virtual-ip) VIRTUAL_IP="$2"; shift 2 ;;
        --interface) INTERFACE="$2"; shift 2 ;;
        --domain) DOMAIN="$2"; shift 2 ;;
        --node-type) NODE_TYPE="$2"; shift 2 ;;
        --help) usage ;;
        *) echo -e "${RED}Unknown option: $1${NC}"; usage ;;
    esac
done

# Validate required arguments
if [[ -z "$PRIMARY_IP" || -z "$BACKUP_IP" || -z "$VIRTUAL_IP" || -z "$NODE_TYPE" ]]; then
    echo -e "${RED}Error: Missing required arguments${NC}"
    usage
fi

if [[ "$NODE_TYPE" != "primary" && "$NODE_TYPE" != "backup" ]]; then
    echo -e "${RED}Error: node-type must be 'primary' or 'backup'${NC}"
    exit 1
fi

# Cleanup function for rollback
cleanup() {
    echo -e "${RED}Installation failed. Cleaning up...${NC}"
    systemctl stop haproxy keepalived 2>/dev/null || true
}

trap cleanup ERR

# Check prerequisites
echo -e "${YELLOW}[1/10] Checking prerequisites...${NC}"
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root${NC}"
    exit 1
fi

# Detect distribution
if [ ! -f /etc/os-release ]; then
    echo -e "${RED}Cannot detect distribution${NC}"
    exit 1
fi

. /etc/os-release
case "$ID" in
    ubuntu|debian) 
        PKG_MGR="apt"
        PKG_INSTALL="apt install -y"
        PKG_UPDATE="apt update && apt upgrade -y"
        FIREWALL_CMD="ufw"
        ;;
    almalinux|rocky|centos|rhel|ol|fedora)
        PKG_MGR="dnf"
        PKG_INSTALL="dnf install -y"
        PKG_UPDATE="dnf update -y"
        FIREWALL_CMD="firewall-cmd"
        ;;
    amzn)
        PKG_MGR="yum" 
        PKG_INSTALL="yum install -y"
        PKG_UPDATE="yum update -y"
        FIREWALL_CMD="firewall-cmd"
        ;;
    *)
        echo -e "${RED}Unsupported distribution: $ID${NC}"
        exit 1
        ;;
esac

echo -e "${GREEN}Detected distribution: $PRETTY_NAME${NC}"

# Update system packages
echo -e "${YELLOW}[2/10] Updating system packages...${NC}"
$PKG_UPDATE

# Install required packages
echo -e "${YELLOW}[3/10] Installing HAProxy and keepalived...${NC}"
$PKG_INSTALL haproxy keepalived curl openssl

# Create SSL certificate directory and certificate
echo -e "${YELLOW}[4/10] Creating SSL certificate...${NC}"
mkdir -p /etc/ssl/certs
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
    -keyout /tmp/${DOMAIN}.key \
    -out /tmp/${DOMAIN}.crt \
    -subj "/C=US/ST=State/L=City/O=Organization/CN=${DOMAIN}" 2>/dev/null

cat /tmp/${DOMAIN}.crt /tmp/${DOMAIN}.key > /etc/ssl/certs/${DOMAIN}.pem
chmod 600 /etc/ssl/certs/${DOMAIN}.pem
chown root:root /etc/ssl/certs/${DOMAIN}.pem
rm -f /tmp/${DOMAIN}.key /tmp/${DOMAIN}.crt

# Configure HAProxy
echo -e "${YELLOW}[5/10] Configuring HAProxy...${NC}"
cat > /etc/haproxy/haproxy.cfg << EOF
global
    daemon
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    log stdout local0

defaults
    mode http
    timeout connect 5000
    timeout client  50000
    timeout server  50000
    option httplog
    option dontlognull
    option redispatch
    retries 3
    maxconn 2000

frontend web_frontend
    bind *:80
    bind *:443 ssl crt /etc/ssl/certs/${DOMAIN}.pem
    redirect scheme https if !{ ssl_fc }
    default_backend web_servers

backend web_servers
    balance roundrobin
    option httpchk GET /health
    http-check expect status 200
    server web1 ${PRIMARY_IP}:8080 check inter 10s fall 3 rise 2
    server web2 ${BACKUP_IP}:8080 check inter 10s fall 3 rise 2

listen stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s
    stats admin if TRUE
EOF

# Configure keepalived based on node type
echo -e "${YELLOW}[6/10] Configuring keepalived for ${NODE_TYPE} node...${NC}"

if [[ "$NODE_TYPE" == "primary" ]]; then
    ROUTER_ID="LB_PRIMARY"
    STATE="MASTER"
    PRIORITY="110"
    EMAIL_FROM="lb-primary@${DOMAIN}"
else
    ROUTER_ID="LB_BACKUP"
    STATE="BACKUP"
    PRIORITY="100"
    EMAIL_FROM="lb-backup@${DOMAIN}"
fi

cat > /etc/keepalived/keepalived.conf << EOF
global_defs {
    notification_email {
        admin@${DOMAIN}
    }
    notification_email_from ${EMAIL_FROM}
    smtp_server localhost
    smtp_connect_timeout 30
    router_id ${ROUTER_ID}
    script_user root
    enable_script_security
}

vrrp_script chk_haproxy {
    script "/usr/bin/curl -f http://localhost:8404/stats || exit 1"
    interval 2
    weight -2
    fall 3
    rise 2
}

vrrp_instance VI_1 {
    state ${STATE}
    interface ${INTERFACE}
    virtual_router_id 51
    priority ${PRIORITY}
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass mySecurePass123
    }
    virtual_ipaddress {
        ${VIRTUAL_IP}/24
    }
    track_script {
        chk_haproxy
    }
    notify_master "/etc/keepalived/master.sh"
    notify_backup "/etc/keepalived/backup.sh" 
    notify_fault "/etc/keepalived/fault.sh"
}
EOF

# Create keepalived notification scripts
echo -e "${YELLOW}[7/10] Creating keepalived notification scripts...${NC}"
cat > /etc/keepalived/master.sh << 'EOF'
#!/bin/bash
echo "$(date): Became MASTER" >> /var/log/keepalived-state.log
systemctl restart haproxy
EOF

cat > /etc/keepalived/backup.sh << 'EOF'
#!/bin/bash
echo "$(date): Became BACKUP" >> /var/log/keepalived-state.log
EOF

cat > /etc/keepalived/fault.sh << 'EOF'
#!/bin/bash
echo "$(date): Entered FAULT state" >> /var/log/keepalived-state.log
EOF

chmod 755 /etc/keepalived/*.sh
chown root:root /etc/keepalived/*.sh

# Configure firewall
echo -e "${YELLOW}[8/10] Configuring firewall...${NC}"
case "$PKG_MGR" in
    apt)
        ufw allow 80/tcp
        ufw allow 443/tcp
        ufw allow 8404/tcp
        ufw allow 112/tcp
        echo "y" | ufw enable 2>/dev/null || true
        ;;
    dnf|yum)
        systemctl enable firewalld
        systemctl start firewalld
        firewall-cmd --permanent --add-port=80/tcp
        firewall-cmd --permanent --add-port=443/tcp
        firewall-cmd --permanent --add-port=8404/tcp
        firewall-cmd --permanent --add-protocol=vrrp
        firewall-cmd --reload
        ;;
esac

# Enable and start services
echo -e "${YELLOW}[9/10] Starting services...${NC}"
systemctl enable haproxy keepalived
systemctl start haproxy
systemctl start keepalived

# Verify installation
echo -e "${YELLOW}[10/10] Verifying installation...${NC}"
sleep 5

if ! systemctl is-active --quiet haproxy; then
    echo -e "${RED}HAProxy service is not running${NC}"
    exit 1
fi

if ! systemctl is-active --quiet keepalived; then
    echo -e "${RED}Keepalived service is not running${NC}"
    exit 1
fi

if ! curl -f http://localhost:8404/stats >/dev/null 2>&1; then
    echo -e "${RED}HAProxy stats page is not accessible${NC}"
    exit 1
fi

echo -e "${GREEN}Installation completed successfully!${NC}"
echo -e "${GREEN}HAProxy stats available at: http://$(hostname -I | awk '{print $1}'):8404/stats${NC}"
echo -e "${GREEN}Virtual IP: ${VIRTUAL_IP}${NC}"
echo -e "${GREEN}Node type: ${NODE_TYPE}${NC}"