Build a highly available Linux cluster with Pacemaker and Corosync

#!/usr/bin/env bash
set -euo pipefail

# ------------------------------------------------------------------
# Pacemaker + Corosync HA cluster bootstrap script
# Run on EACH node, then only the final cluster creation step
# needs to run once (use --init-cluster on one node after both
# have run this script with --node-setup).
# ------------------------------------------------------------------

# ---- Colors ----
RED=$'\033[0;31m'
GREEN=$'\033[0;32m'
YELLOW=$'\033[1;33m'
NC=$'\033[0m'

log()  { echo -e "${GREEN}$*${NC}"; }
warn() { echo -e "${YELLOW}$*${NC}"; }
err()  { echo -e "${RED}$*${NC}" >&2; }

usage() {
    cat <<EOF
Usage: $0 --local-name <name> --local-ip <ip> --peer-name <name> --peer-ip <ip> [--init-cluster]

  --local-name     Hostname of this node (e.g. node1)
  --local-ip       IP of this node (e.g. 203.0.113.10)
  --peer-name      Hostname of the peer node (e.g. node2)
  --peer-ip        IP of the peer node (e.g. 203.0.113.11)
  --cluster-name   Name of the cluster (default: mycluster)
  --init-cluster   After setup, also authenticate + create the cluster
                    (run this flag on only ONE of the two nodes)

Example:
  $0 --local-name node1 --local-ip 203.0.113.10 \\
     --peer-name node2 --peer-ip 203.0.113.11 --init-cluster
EOF
    exit 1
}

CLUSTER_NAME="mycluster"
INIT_CLUSTER=0
LOCAL_NAME=""
LOCAL_IP=""
PEER_NAME=""
PEER_IP=""

while [ $# -gt 0 ]; do
    case "$1" in
        --local-name) LOCAL_NAME="$2"; shift 2 ;;
        --local-ip) LOCAL_IP="$2"; shift 2 ;;
        --peer-name) PEER_NAME="$2"; shift 2 ;;
        --peer-ip) PEER_IP="$2"; shift 2 ;;
        --cluster-name) CLUSTER_NAME="$2"; shift 2 ;;
        --init-cluster) INIT_CLUSTER=1; shift ;;
        -h|--help) usage ;;
        *) err "Unknown argument: $1"; usage ;;
    esac
done

[ -z "$LOCAL_NAME" ] || [ -z "$LOCAL_IP" ] || [ -z "$PEER_NAME" ] || [ -z "$PEER_IP" ] && {
    if [ -z "$LOCAL_NAME" ] || [ -z "$LOCAL_IP" ] || [ -z "$PEER_NAME" ] || [ -z "$PEER_IP" ]; then
        err "Missing required arguments."
        usage
    fi
}

# ---- Prerequisite checks ----
if [ "$(id -u)" -ne 0 ]; then
    err "This script must be run as root (or via sudo)."
    exit 1
fi

# ---- Rollback on failure ----
cleanup_on_error() {
    err "[ERROR] Installation failed. Rolling back partial changes..."
    systemctl stop pcsd 2>/dev/null || true
    systemctl stop corosync 2>/dev/null || true
    systemctl stop pacemaker 2>/dev/null || true
    warn "Rollback complete. Review /var/log for details before retrying."
}
trap cleanup_on_error ERR

TOTAL_STEPS=9

# ---- [1/9] Detect distro ----
echo "[1/9] Detecting distribution..."
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            FW_CMD="ufw"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            FW_CMD="firewalld"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            FW_CMD="firewalld"
            ;;
        *)
            err "Unsupported distro: $ID"
            exit 1
            ;;
    esac
else
    err "/etc/os-release not found, cannot detect distro."
    exit 1
fi
log "Detected distro: $ID ($PKG_MGR)"

# ---- [2/9] Set hostname and /etc/hosts ----
echo "[2/9] Setting hostname and /etc/hosts entries..."
hostnamectl set-hostname "$LOCAL_NAME"

# Add host entries idempotently
for entry in "$LOCAL_IP $LOCAL_NAME ${LOCAL_NAME}.example.com" "$PEER_IP $PEER_NAME ${PEER_NAME}.example.com"; do
    ip_part=$(echo "$entry" | awk '{print $1}')
    if ! grep -q "^${ip_part}[[:space:]]" /etc/hosts 2>/dev/null; then
        echo "$entry" >> /etc/hosts
    fi
done
log "Hostname and /etc/hosts configured."

# ---- [3/9] Update package cache ----
echo "[3/9] Updating package cache..."
if [ "$PKG_MGR" = "apt" ]; then
    apt update
elif [ "$PKG_MGR" = "dnf" ]; then
    dnf install -y epel-release || true
    dnf config-manager --set-enabled highavailability || true
fi

# ---- [4/9] Install cluster stack ----
echo "[4/9] Installing Pacemaker, Corosync, pcs, fencing agents..."
if [ "$PKG_MGR" = "apt" ]; then
    $PKG_INSTALL pacemaker corosync pcs fence-agents resource-agents chrony
elif [ "$PKG_MGR" = "dnf" ] || [ "$PKG_MGR" = "yum" ]; then
    $PKG_INSTALL pacemaker corosync pcs fence-agents-all resource-agents chrony
fi
log "Cluster packages installed."

# ---- [5/9] Ensure time sync is running ----
echo "[5/9] Ensuring chrony (NTP) is enabled..."
systemctl enable --now chronyd 2>/dev/null || systemctl enable --now chrony 2>/dev/null || \
    warn "Could not enable chrony automatically; verify time sync manually."

# ---- [6/9] Set hacluster password and enable pcsd ----
echo "[6/9] Setting hacluster password and enabling pcsd..."
if [ -z "${HACLUSTER_PASSWORD:-}" ]; then
    warn "HACLUSTER_PASSWORD env var not set. Generating a random password."
    HACLUSTER_PASSWORD=$(tr -dc 'A-Za-z0-9!@#%' </dev/urandom | head -c 20)
    warn "Generated hacluster password: $HACLUSTER_PASSWORD"
    warn "Store this securely — you need the SAME password on both nodes."
fi
echo "hacluster:${HACLUSTER_PASSWORD}" | chpasswd
systemctl enable --now pcsd
log "pcsd enabled and hacluster password set."

# ---- [7/9] Open firewall ports ----
echo "[7/9] Configuring firewall rules..."
if [ "$FW_CMD" = "ufw" ]; then
    if command -v ufw >/dev/null 2>&1; then
        ufw allow 2224/tcp
        ufw allow 5405/udp
        ufw allow 3121/tcp
        ufw allow 21064/tcp
        log "ufw rules added (2224/tcp, 5405/udp, 3121/tcp, 21064/tcp)."
    else
        warn "ufw not found; skipping firewall configuration."
    fi
elif [ "$FW_CMD" = "firewalld" ]; then
    if command -v firewall-cmd >/dev/null 2>&1; then
        systemctl enable --now firewalld
        firewall-cmd --permanent --add-service=high-availability
        firewall-cmd --reload
        log "firewalld high-availability service enabled."
    else
        warn "firewall-cmd not found; skipping firewall configuration."
    fi
fi

# ---- [8/9] Enable corosync/pacemaker services ----
echo "[8/9] Enabling corosync and pacemaker services..."
systemctl enable corosync 2>/dev/null || true
systemctl enable pacemaker 2>/dev/null || true
log "Base node setup complete for $LOCAL_NAME."

# ---- [9/9] Optionally initialize the cluster ----
echo "[9/9] Cluster initialization step..."
if [ "$INIT_CLUSTER" -eq 1 ]; then
    log "Running cluster initialization (this host: $LOCAL_NAME)..."

    # Authenticate both nodes using hacluster credentials
    pcs host auth "$LOCAL_NAME" "$PEER_NAME" -u hacluster -p "$HACLUSTER_PASSWORD"

    # Generate corosync config and create the cluster
    pcs cluster setup "$CLUSTER_NAME" "$LOCAL_NAME" "$PEER_NAME" --force

    # Start and enable cluster on both nodes
    pcs cluster start --all
    pcs cluster enable --all

    # For a strict two-node cluster, allow quorum with one node reachable.
    # In production with 3+ nodes, remove this and use a qdevice instead.
    pcs property set no-quorum-policy=stop || true
    if [ -f /etc/corosync/corosync.conf ] && ! grep -q "two_node" /etc/corosync/corosync.conf; then
        warn "two_node setting not found in corosync.conf; pcs cluster setup should have added it automatically for a 2-node cluster."
    fi

    log "Cluster '$CLUSTER_NAME' created and started."
else
    warn "Skipping cluster creation (--init-cluster not passed)."
    warn "Run this script with --init-cluster on exactly ONE node once both nodes are set up."
fi

# ---- Verification ----
echo "Running verification checks..."
FAILED=0

if ! systemctl is-active --quiet pcsd; then
    err "pcsd is not running."
    FAILED=1
else
    log "pcsd is running."
fi

if [ "$INIT_CLUSTER" -eq 1 ]; then
    if command -v pcs >/dev/null 2>&1; then
        if pcs cluster status >/dev/null 2>&1; then
            log "Cluster status check passed."
            pcs status || true
        else
            err "Cluster status check failed."
            FAILED=1
        fi
    fi
fi

if [ "$FAILED" -eq 1 ]; then
    err "One or more verification checks failed. Review output above."
    exit 1
fi

trap - ERR
log "Setup complete for node: $LOCAL_NAME"
warn "Reminder: do NOT skip STONITH fencing configuration before production use."