Implement Alertmanager high availability clustering with automatic failover and load balancing

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Global variables
ALERTMANAGER_VERSION="0.26.0"
CLUSTER_PEERS=""
NODE_ID="node1"
LISTEN_PORT="9093"
CLUSTER_PORT="9094"

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  --node-id ID         Node identifier (default: node1)"
    echo "  --cluster-peers IP   Comma-separated list of cluster peer IPs"
    echo "  --listen-port PORT   Alertmanager listen port (default: 9093)"
    echo "  --cluster-port PORT  Cluster communication port (default: 9094)"
    echo "  -h, --help          Show this help message"
    exit 1
}

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --node-id)
            NODE_ID="$2"
            shift 2
            ;;
        --cluster-peers)
            CLUSTER_PEERS="$2"
            shift 2
            ;;
        --listen-port)
            LISTEN_PORT="$2"
            shift 2
            ;;
        --cluster-port)
            CLUSTER_PORT="$2"
            shift 2
            ;;
        -h|--help)
            usage
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Cleanup function for rollback
cleanup() {
    echo -e "${YELLOW}Installation failed. Cleaning up...${NC}"
    systemctl stop alertmanager 2>/dev/null || true
    systemctl disable alertmanager 2>/dev/null || true
    rm -f /etc/systemd/system/alertmanager.service
    rm -f /usr/local/bin/alertmanager /usr/local/bin/amtool
    rm -rf /etc/alertmanager /var/lib/alertmanager
    userdel alertmanager 2>/dev/null || true
    systemctl daemon-reload
    echo -e "${YELLOW}Cleanup completed${NC}"
}

trap cleanup ERR

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root or with sudo${NC}"
    exit 1
fi

# Detect distribution
echo -e "${YELLOW}[1/8] Detecting distribution...${NC}"
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update && apt upgrade -y"
            PKG_INSTALL="apt install -y"
            ;;
        almalinux|rocky|centos|rhel|ol)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            ;;
        fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            ;;
        *)
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
    echo -e "${GREEN}Detected: $PRETTY_NAME${NC}"
else
    echo -e "${RED}Cannot detect distribution${NC}"
    exit 1
fi

# Update system packages
echo -e "${YELLOW}[2/8] Updating system packages...${NC}"
$PKG_UPDATE
$PKG_INSTALL wget curl systemd tar

# Create alertmanager user and directories
echo -e "${YELLOW}[3/8] Creating alertmanager user and directories...${NC}"
useradd --no-create-home --shell /bin/false alertmanager 2>/dev/null || true
mkdir -p /etc/alertmanager
mkdir -p /var/lib/alertmanager
chown alertmanager:alertmanager /etc/alertmanager
chown alertmanager:alertmanager /var/lib/alertmanager
chmod 755 /etc/alertmanager
chmod 755 /var/lib/alertmanager

# Download and install Alertmanager
echo -e "${YELLOW}[4/8] Downloading and installing Alertmanager...${NC}"
cd /tmp
wget -q "https://github.com/prometheus/alertmanager/releases/download/v${ALERTMANAGER_VERSION}/alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz"
tar xf "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz"
cp "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/alertmanager" /usr/local/bin/
cp "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/amtool" /usr/local/bin/
chown alertmanager:alertmanager /usr/local/bin/alertmanager
chown alertmanager:alertmanager /usr/local/bin/amtool
chmod 755 /usr/local/bin/alertmanager
chmod 755 /usr/local/bin/amtool
rm -rf "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64"*

# Create Alertmanager configuration
echo -e "${YELLOW}[5/8] Creating Alertmanager configuration...${NC}"
cat > /etc/alertmanager/alertmanager.yml << 'EOF'
global:
  smtp_smarthost: 'localhost:587'
  smtp_from: 'alertmanager@example.com'

route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  receiver: 'default'
  routes:
  - match:
      severity: critical
    receiver: 'critical-alerts'
  - match:
      severity: warning
    receiver: 'warning-alerts'

receivers:
- name: 'default'
  email_configs:
  - to: 'admin@example.com'
    subject: 'Alertmanager notification'
    body: |
      {{ range .Alerts }}
      Alert: {{ .Annotations.summary }}
      Description: {{ .Annotations.description }}
      {{ end }}

- name: 'critical-alerts'
  email_configs:
  - to: 'critical@example.com'
    subject: 'CRITICAL: {{ .GroupLabels.alertname }}'
    body: |
      {{ range .Alerts }}
      Alert: {{ .Annotations.summary }}
      Severity: {{ .Labels.severity }}
      Instance: {{ .Labels.instance }}
      Description: {{ .Annotations.description }}
      {{ end }}

- name: 'warning-alerts'
  email_configs:
  - to: 'warnings@example.com'
    subject: 'WARNING: {{ .GroupLabels.alertname }}'
    body: |
      {{ range .Alerts }}
      Alert: {{ .Annotations.summary }}
      Severity: {{ .Labels.severity }}
      Instance: {{ .Labels.instance }}
      Description: {{ .Annotations.description }}
      {{ end }}

inhibit_rules:
- source_match:
    severity: 'critical'
  target_match:
    severity: 'warning'
  equal: ['alertname', 'instance']
EOF

chown alertmanager:alertmanager /etc/alertmanager/alertmanager.yml
chmod 644 /etc/alertmanager/alertmanager.yml

# Create systemd service
echo -e "${YELLOW}[6/8] Creating systemd service...${NC}"
CLUSTER_LISTEN_ADDRESS="0.0.0.0:${CLUSTER_PORT}"
CLUSTER_ADVERTISE_ADDRESS="$(hostname -I | awk '{print $1}'):${CLUSTER_PORT}"

# Build cluster peer arguments
PEER_ARGS=""
if [[ -n "$CLUSTER_PEERS" ]]; then
    IFS=',' read -ra PEERS <<< "$CLUSTER_PEERS"
    for peer in "${PEERS[@]}"; do
        PEER_ARGS="$PEER_ARGS --cluster.peer=${peer}:${CLUSTER_PORT}"
    done
fi

cat > /etc/systemd/system/alertmanager.service << EOF
[Unit]
Description=Alertmanager
Wants=network-online.target
After=network-online.target

[Service]
User=alertmanager
Group=alertmanager
Type=simple
WorkingDirectory=/var/lib/alertmanager
ExecStart=/usr/local/bin/alertmanager \\
  --config.file=/etc/alertmanager/alertmanager.yml \\
  --storage.path=/var/lib/alertmanager \\
  --web.listen-address=0.0.0.0:${LISTEN_PORT} \\
  --cluster.listen-address=${CLUSTER_LISTEN_ADDRESS} \\
  --cluster.advertise-address=${CLUSTER_ADVERTISE_ADDRESS}${PEER_ARGS}
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

# Configure firewall
echo -e "${YELLOW}[7/8] Configuring firewall...${NC}"
case "$ID" in
    ubuntu|debian)
        if command -v ufw >/dev/null 2>&1; then
            ufw allow ${LISTEN_PORT}/tcp
            ufw allow ${CLUSTER_PORT}/tcp
        fi
        ;;
    almalinux|rocky|centos|rhel|ol|fedora)
        if command -v firewall-cmd >/dev/null 2>&1; then
            firewall-cmd --permanent --add-port=${LISTEN_PORT}/tcp
            firewall-cmd --permanent --add-port=${CLUSTER_PORT}/tcp
            firewall-cmd --reload
        fi
        ;;
esac

# Start and enable service
echo -e "${YELLOW}[8/8] Starting Alertmanager service...${NC}"
systemctl daemon-reload
systemctl enable alertmanager
systemctl start alertmanager

# Verify installation
echo -e "${YELLOW}Verifying installation...${NC}"
sleep 5

if systemctl is-active --quiet alertmanager; then
    echo -e "${GREEN}✓ Alertmanager service is running${NC}"
else
    echo -e "${RED}✗ Alertmanager service failed to start${NC}"
    exit 1
fi

if curl -s http://localhost:${LISTEN_PORT}/-/healthy >/dev/null 2>&1; then
    echo -e "${GREEN}✓ Alertmanager is responding on port ${LISTEN_PORT}${NC}"
else
    echo -e "${RED}✗ Alertmanager is not responding${NC}"
    exit 1
fi

echo -e "${GREEN}Alertmanager cluster installation completed successfully!${NC}"
echo -e "Node ID: ${NODE_ID}"
echo -e "Web UI: http://$(hostname -I | awk '{print $1}'):${LISTEN_PORT}"
echo -e "Configuration: /etc/alertmanager/alertmanager.yml"
echo -e "Logs: journalctl -u alertmanager -f"

trap - ERR