Configure Prometheus Alertmanager with custom webhook integrations for Slack, Microsoft Teams, and PagerDuty notifications

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Default configuration
ALERTMANAGER_VERSION="0.27.0"
ALERTMANAGER_USER="alertmanager"
SLACK_WEBHOOK=""
TEAMS_WEBHOOK=""
PAGERDUTY_KEY=""

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  --slack-webhook URL    Slack webhook URL"
    echo "  --teams-webhook URL    Microsoft Teams webhook URL" 
    echo "  --pagerduty-key KEY    PagerDuty integration key"
    echo "  --version VERSION      Alertmanager version (default: $ALERTMANAGER_VERSION)"
    echo "  -h, --help            Show this help message"
    exit 1
}

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --slack-webhook)
            SLACK_WEBHOOK="$2"
            shift 2
            ;;
        --teams-webhook)
            TEAMS_WEBHOOK="$2"
            shift 2
            ;;
        --pagerduty-key)
            PAGERDUTY_KEY="$2"
            shift 2
            ;;
        --version)
            ALERTMANAGER_VERSION="$2"
            shift 2
            ;;
        -h|--help)
            usage
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Cleanup function for rollback
cleanup() {
    echo -e "${RED}Installation failed. Cleaning up...${NC}"
    systemctl stop alertmanager 2>/dev/null || true
    systemctl disable alertmanager 2>/dev/null || true
    rm -f /etc/systemd/system/alertmanager.service
    rm -rf /etc/alertmanager /var/lib/alertmanager
    userdel alertmanager 2>/dev/null || true
    rm -f /usr/local/bin/alertmanager /usr/local/bin/amtool
    rm -f alertmanager-*.tar.gz
    rm -rf alertmanager-*
}

trap cleanup ERR

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root or with sudo${NC}"
    exit 1
fi

echo -e "${GREEN}Starting Prometheus Alertmanager installation...${NC}"

# Detect distribution
echo -e "${YELLOW}[1/8] Detecting operating system...${NC}"
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            ;;
        *) 
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}Cannot detect operating system${NC}"
    exit 1
fi

echo -e "${GREEN}Detected: $PRETTY_NAME${NC}"

# Update package manager and install dependencies
echo -e "${YELLOW}[2/8] Installing dependencies...${NC}"
$PKG_UPDATE
$PKG_INSTALL wget tar curl

# Download and install Alertmanager
echo -e "${YELLOW}[3/8] Downloading Alertmanager v${ALERTMANAGER_VERSION}...${NC}"
DOWNLOAD_URL="https://github.com/prometheus/alertmanager/releases/download/v${ALERTMANAGER_VERSION}/alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz"

wget "$DOWNLOAD_URL"
tar xvfz "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz"

# Install binaries
echo -e "${YELLOW}[4/8] Installing Alertmanager binaries...${NC}"
mv "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/alertmanager" /usr/local/bin/
mv "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/amtool" /usr/local/bin/
chmod 755 /usr/local/bin/alertmanager /usr/local/bin/amtool

# Create user and directories
echo -e "${YELLOW}[5/8] Creating user and directories...${NC}"
useradd --no-create-home --shell /bin/false $ALERTMANAGER_USER || true
mkdir -p /etc/alertmanager /var/lib/alertmanager
chown $ALERTMANAGER_USER:$ALERTMANAGER_USER /etc/alertmanager /var/lib/alertmanager
chmod 755 /etc/alertmanager /var/lib/alertmanager

# Create systemd service file
echo -e "${YELLOW}[6/8] Creating systemd service...${NC}"
cat > /etc/systemd/system/alertmanager.service << 'EOF'
[Unit]
Description=Alertmanager
Wants=network-online.target
After=network-online.target

[Service]
User=alertmanager
Group=alertmanager
Type=simple
ExecStart=/usr/local/bin/alertmanager \
    --config.file /etc/alertmanager/alertmanager.yml \
    --storage.path /var/lib/alertmanager/ \
    --web.external-url=http://localhost:9093 \
    --cluster.listen-address=""
Restart=always
RestartSec=5
StandardOutput=journal
StandardError=journal
SyslogIdentifier=alertmanager

[Install]
WantedBy=multi-user.target
EOF

chmod 644 /etc/systemd/system/alertmanager.service

# Create Alertmanager configuration
echo -e "${YELLOW}[7/8] Creating configuration file...${NC}"
cat > /etc/alertmanager/alertmanager.yml << EOF
global:
  smtp_smarthost: 'localhost:587'
  smtp_from: 'alertmanager@example.com'

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  receiver: 'default'
  routes:
  - match:
      service: critical
    receiver: pagerduty-critical
    routes:
    - match:
        severity: critical
      receiver: pagerduty-critical
      continue: true
    - match:
        severity: critical
      receiver: slack-critical
  - match:
      service: database
    receiver: slack-database
  - match:
      service: frontend
    receiver: teams-frontend

receivers:
- name: 'default'
  webhook_configs:
  - url: 'http://localhost:5001/'

- name: 'slack-critical'
  slack_configs:
  - api_url: '${SLACK_WEBHOOK:-https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK}'
    channel: '#critical-alerts'
    title: 'CRITICAL: {{ .GroupLabels.alertname }}'
    text: |
      {{ range .Alerts }}
      Alert: {{ .Annotations.summary }}
      Description: {{ .Annotations.description }}
      Severity: {{ .Labels.severity }}
      Instance: {{ .Labels.instance }}
      {{ end }}
    color: 'danger'
    send_resolved: true

- name: 'slack-database'
  slack_configs:
  - api_url: '${SLACK_WEBHOOK:-https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK}'
    channel: '#database-alerts'
    title: 'Database Alert: {{ .GroupLabels.alertname }}'
    text: |
      {{ range .Alerts }}
      Database Issue: {{ .Annotations.summary }}
      Details: {{ .Annotations.description }}
      Server: {{ .Labels.instance }}
      {{ end }}
    send_resolved: true

- name: 'teams-frontend'
  webhook_configs:
  - url: '${TEAMS_WEBHOOK:-https://outlook.office.com/webhook/YOUR/TEAMS/WEBHOOK}'
    title: 'Frontend Alert'
    text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'

- name: 'pagerduty-critical'
  pagerduty_configs:
  - routing_key: '${PAGERDUTY_KEY:-YOUR_PAGERDUTY_INTEGRATION_KEY}'
    description: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
    severity: 'critical'

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']
EOF

chown $ALERTMANAGER_USER:$ALERTMANAGER_USER /etc/alertmanager/alertmanager.yml
chmod 644 /etc/alertmanager/alertmanager.yml

# Configure firewall based on distribution
case "$ID" in
    ubuntu|debian)
        if command -v ufw >/dev/null 2>&1; then
            ufw allow 9093/tcp
        fi
        ;;
    almalinux|rocky|centos|rhel|ol|fedora)
        if command -v firewall-cmd >/dev/null 2>&1; then
            firewall-cmd --permanent --add-port=9093/tcp
            firewall-cmd --reload
        fi
        ;;
esac

# Start and enable service
echo -e "${YELLOW}[8/8] Starting Alertmanager service...${NC}"
systemctl daemon-reload
systemctl enable alertmanager
systemctl start alertmanager

# Cleanup temporary files
rm -f "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz"
rm -rf "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64"

# Verification
echo -e "${YELLOW}Verifying installation...${NC}"
sleep 5

if systemctl is-active --quiet alertmanager; then
    echo -e "${GREEN}✓ Alertmanager service is running${NC}"
else
    echo -e "${RED}✗ Alertmanager service failed to start${NC}"
    systemctl status alertmanager
    exit 1
fi

if curl -s http://localhost:9093 >/dev/null; then
    echo -e "${GREEN}✓ Alertmanager web interface is accessible${NC}"
else
    echo -e "${YELLOW}! Alertmanager web interface may not be ready yet${NC}"
fi

echo -e "${GREEN}Installation completed successfully!${NC}"
echo ""
echo "Next steps:"
echo "1. Update webhook URLs in /etc/alertmanager/alertmanager.yml"
echo "2. Configure Prometheus to send alerts to http://localhost:9093"
echo "3. Access Alertmanager web UI at http://localhost:9093"
echo "4. Test configuration with: amtool config check /etc/alertmanager/alertmanager.yml"