Set up InfluxDB alerting with Kapacitor and notifications

#!/usr/bin/env bash
set -euo pipefail

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Configuration
SMTP_SERVER="${SMTP_SERVER:-localhost:587}"
SLACK_WEBHOOK="${SLACK_WEBHOOK:-}"
ADMIN_EMAIL="${ADMIN_EMAIL:-admin@localhost}"

# Error handling
cleanup() {
    if [[ ${#CREATED_FILES[@]} -gt 0 ]]; then
        echo -e "${YELLOW}[CLEANUP] Removing created files...${NC}"
        rm -f "${CREATED_FILES[@]}" 2>/dev/null || true
    fi
}
trap cleanup ERR EXIT

declare -a CREATED_FILES=()

usage() {
    cat << EOF
Usage: $0 [OPTIONS]
Install InfluxDB and Kapacitor with alerting configuration

OPTIONS:
    -e EMAIL    Admin email address (default: admin@localhost)
    -s SERVER   SMTP server (default: localhost:587)
    -w WEBHOOK  Slack webhook URL
    -h          Show this help

EXAMPLES:
    $0 -e admin@company.com -s smtp.company.com:587
    $0 -e alerts@domain.com -w https://hooks.slack.com/services/...

ENVIRONMENT:
    ADMIN_EMAIL     Admin email address
    SMTP_SERVER     SMTP server and port
    SLACK_WEBHOOK   Slack webhook URL
EOF
    exit 1
}

log() {
    echo -e "${GREEN}$*${NC}"
}

warn() {
    echo -e "${YELLOW}$*${NC}"
}

error() {
    echo -e "${RED}$*${NC}" >&2
}

check_prerequisites() {
    echo "[1/12] Checking prerequisites..."
    
    if [[ $EUID -ne 0 ]]; then
        error "This script must be run as root or with sudo"
        exit 1
    fi

    if ! command -v curl &> /dev/null; then
        error "curl is required but not installed"
        exit 1
    fi
    
    if ! command -v gpg &> /dev/null; then
        error "gpg is required but not installed"
        exit 1
    fi

    log "Prerequisites check passed"
}

detect_distro() {
    echo "[2/12] Detecting distribution..."
    
    if [[ ! -f /etc/os-release ]]; then
        error "Cannot detect distribution - /etc/os-release not found"
        exit 1
    fi

    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            PKG_UPGRADE="apt upgrade -y"
            REPO_SETUP="debian"
            FIREWALL_CMD="ufw"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            PKG_UPGRADE=""
            REPO_SETUP="rhel"
            FIREWALL_CMD="firewall-cmd"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            PKG_UPGRADE=""
            REPO_SETUP="rhel"
            FIREWALL_CMD="firewall-cmd"
            ;;
        *)
            error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac

    log "Detected: $PRETTY_NAME ($PKG_MGR)"
}

parse_args() {
    while getopts "e:s:w:h" opt; do
        case $opt in
            e) ADMIN_EMAIL="$OPTARG" ;;
            s) SMTP_SERVER="$OPTARG" ;;
            w) SLACK_WEBHOOK="$OPTARG" ;;
            h) usage ;;
            *) usage ;;
        esac
    done
}

update_packages() {
    echo "[3/12] Updating system packages..."
    $PKG_UPDATE
    [[ -n "$PKG_UPGRADE" ]] && $PKG_UPGRADE
    log "System packages updated"
}

setup_repository() {
    echo "[4/12] Setting up InfluxDB repository..."
    
    if [[ "$REPO_SETUP" == "debian" ]]; then
        curl -s https://repos.influxdata.com/influxdata-archive_compat.key | gpg --dearmor > /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg
        echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' > /etc/apt/sources.list.d/influxdata.list
        CREATED_FILES+=("/etc/apt/sources.list.d/influxdata.list")
        $PKG_UPDATE
    else
        cat > /etc/yum.repos.d/influxdata.repo << 'EOF'
[influxdata]
name = InfluxDB Repository - Stable
baseurl = https://repos.influxdata.com/stable/$basearch/main
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdata-archive_compat.key
EOF
        CREATED_FILES+=("/etc/yum.repos.d/influxdata.repo")
    fi
    
    log "Repository configured"
}

install_influxdb() {
    echo "[5/12] Installing InfluxDB and Kapacitor..."
    $PKG_INSTALL influxdb kapacitor
    log "InfluxDB and Kapacitor installed"
}

configure_influxdb() {
    echo "[6/12] Configuring InfluxDB..."
    
    # Backup original config
    if [[ -f /etc/influxdb/influxdb.conf ]]; then
        cp /etc/influxdb/influxdb.conf /etc/influxdb/influxdb.conf.backup
    fi

    # Enable InfluxDB service
    systemctl enable influxdb
    systemctl start influxdb
    
    log "InfluxDB configured and started"
}

configure_kapacitor() {
    echo "[7/12] Configuring Kapacitor..."
    
    # Configure Kapacitor
    cat > /etc/kapacitor/kapacitor.conf << EOF
[logging]
  file = "/var/log/kapacitor/kapacitor.log"
  level = "INFO"

[[influxdb]]
  enabled = true
  name = "localhost"
  default = true
  urls = ["http://localhost:8086"]
  
[[smtp]]
  enabled = true
  host = "${SMTP_SERVER%%:*}"
  port = ${SMTP_SERVER##*:}
  username = ""
  password = ""
  from = "kapacitor@localhost"
  to = ["${ADMIN_EMAIL}"]
  subject = "Kapacitor Alert"
  
EOF

    if [[ -n "$SLACK_WEBHOOK" ]]; then
        cat >> /etc/kapacitor/kapacitor.conf << EOF
[slack]
  enabled = true
  default = true
  url = "${SLACK_WEBHOOK}"
  channel = "#alerts"
  icon-emoji = ":exclamation:"

EOF
    fi

    chown kapacitor:kapacitor /etc/kapacitor/kapacitor.conf
    chmod 640 /etc/kapacitor/kapacitor.conf
    
    # Enable and start Kapacitor
    systemctl enable kapacitor
    systemctl start kapacitor
    
    log "Kapacitor configured and started"
}

create_alert_tasks() {
    echo "[8/12] Creating alert tasks..."
    
    # Create CPU alert
    cat > /tmp/cpu_alert.tick << 'EOF'
stream
    |from()
        .measurement('cpu_usage')
    |alert()
        .crit(lambda: "value" > 90.0)
        .warn(lambda: "value" > 70.0)
        .message('{{ .Time }}: {{ index .Tags "host" }} CPU usage is {{ index .Fields "value" }}%')
        .id('{{ index .Tags "host" }}/cpu_usage')
        .email()
        .slack()
            .channel('#alerts')
EOF
    
    # Create memory alert
    cat > /tmp/memory_alert.tick << 'EOF'
stream
    |from()
        .measurement('memory_usage')
    |alert()
        .crit(lambda: "value" > 90.0)
        .warn(lambda: "value" > 75.0)
        .message('{{ .Time }}: {{ index .Tags "host" }} Memory usage is {{ index .Fields "value" }}%')
        .id('{{ index .Tags "host" }}/memory_usage')
        .email()
        .slack()
            .channel('#alerts')
EOF
    
    # Create deadman alert
    cat > /tmp/deadman_alert.tick << 'EOF'
stream
    |from()
        .measurement('cpu_usage')
    |deadman(2.0m, 1m)
        .id('{{ index .Tags "host" }}/deadman')
        .message('{{ .Time }}: No data from {{ index .Tags "host" }} for 2 minutes')
        .email()
        .slack()
            .channel('#critical')
EOF

    CREATED_FILES+=("/tmp/cpu_alert.tick" "/tmp/memory_alert.tick" "/tmp/deadman_alert.tick")
    
    # Wait for Kapacitor to be ready
    sleep 5
    
    # Define and enable tasks
    kapacitor define cpu_alert -tick /tmp/cpu_alert.tick
    kapacitor define memory_alert -tick /tmp/memory_alert.tick  
    kapacitor define deadman_alert -tick /tmp/deadman_alert.tick
    
    kapacitor enable cpu_alert
    kapacitor enable memory_alert
    kapacitor enable deadman_alert
    
    log "Alert tasks created and enabled"
}

configure_firewall() {
    echo "[9/12] Configuring firewall..."
    
    if [[ "$FIREWALL_CMD" == "ufw" ]] && command -v ufw &> /dev/null; then
        ufw allow 8086/tcp comment "InfluxDB"
        ufw allow 9092/tcp comment "Kapacitor"
    elif [[ "$FIREWALL_CMD" == "firewall-cmd" ]] && command -v firewall-cmd &> /dev/null; then
        if systemctl is-active firewalld &> /dev/null; then
            firewall-cmd --permanent --add-port=8086/tcp
            firewall-cmd --permanent --add-port=9092/tcp
            firewall-cmd --reload
        fi
    fi
    
    log "Firewall configured"
}

create_sample_data() {
    echo "[10/12] Creating sample database..."
    
    # Wait for InfluxDB to be fully ready
    sleep 10
    
    # Create sample database and insert test data
    influx -execute "CREATE DATABASE telegraf"
    
    log "Sample database created"
}

verify_installation() {
    echo "[11/12] Verifying installation..."
    
    # Check services
    if ! systemctl is-active influxdb &> /dev/null; then
        error "InfluxDB service is not running"
        return 1
    fi
    
    if ! systemctl is-active kapacitor &> /dev/null; then
        error "Kapacitor service is not running"
        return 1
    fi
    
    # Check ports
    if ! netstat -tuln | grep -q ":8086"; then
        error "InfluxDB port 8086 is not listening"
        return 1
    fi
    
    if ! netstat -tuln | grep -q ":9092"; then
        error "Kapacitor port 9092 is not listening"
        return 1
    fi
    
    # Check tasks
    if ! kapacitor list tasks | grep -q "cpu_alert"; then
        error "CPU alert task not found"
        return 1
    fi
    
    log "Installation verified successfully"
}

show_summary() {
    echo "[12/12] Installation complete!"
    
    cat << EOF

${GREEN}InfluxDB and Kapacitor have been successfully installed!${NC}

Services:
  - InfluxDB: http://localhost:8086
  - Kapacitor: http://localhost:9092

Configuration:
  - Admin email: ${ADMIN_EMAIL}
  - SMTP server: ${SMTP_SERVER}
  - Slack webhook: ${SLACK_WEBHOOK:-"Not configured"}

Alert tasks enabled:
  - cpu_alert: CPU usage monitoring
  - memory_alert: Memory usage monitoring  
  - deadman_alert: Data flow monitoring

Next steps:
  1. Install Telegraf to send metrics to InfluxDB
  2. Customize alert thresholds in /tmp/*.tick files
  3. Test alerts: kapacitor show cpu_alert
  4. Monitor logs: journalctl -u kapacitor -f

Configuration files:
  - InfluxDB: /etc/influxdb/influxdb.conf
  - Kapacitor: /etc/kapacitor/kapacitor.conf

${GREEN}Installation completed successfully!${NC}
EOF
}

main() {
    parse_args "$@"
    check_prerequisites
    detect_distro
    update_packages
    setup_repository
    install_influxdb
    configure_influxdb
    configure_kapacitor
    create_alert_tasks
    configure_firewall
    create_sample_data
    verify_installation
    show_summary
}

main "$@"