Configure Jaeger alerting with Prometheus and Grafana for distributed tracing observability

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Configuration
PROMETHEUS_VERSION="2.47.2"
PROMETHEUS_USER="prometheus"
PROMETHEUS_HOME="/opt/prometheus"
PROMETHEUS_CONFIG="/etc/prometheus"
PROMETHEUS_DATA="/var/lib/prometheus"

# Print colored output
print_status() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Usage message
usage() {
    echo "Usage: $0"
    echo "Installs and configures Prometheus with Jaeger monitoring"
    exit 1
}

# Cleanup function for rollback
cleanup() {
    print_error "Installation failed. Cleaning up..."
    systemctl stop prometheus 2>/dev/null || true
    systemctl disable prometheus 2>/dev/null || true
    rm -f /etc/systemd/system/prometheus.service
    rm -rf "$PROMETHEUS_HOME" "$PROMETHEUS_CONFIG" "$PROMETHEUS_DATA"
    userdel "$PROMETHEUS_USER" 2>/dev/null || true
    rm -f prometheus-*.tar.gz
    rm -rf prometheus-*
}

# Set up error trap
trap cleanup ERR

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    print_error "This script must be run as root or with sudo"
    exit 1
fi

# Detect distribution
echo "[1/8] Detecting Linux distribution..."
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            ;;
        *)
            print_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    print_error "Cannot detect Linux distribution"
    exit 1
fi

print_status "Detected distribution: $ID using $PKG_MGR"

# Update system packages
echo "[2/8] Updating system packages..."
$PKG_UPDATE

# Install prerequisites
echo "[3/8] Installing prerequisites..."
$PKG_INSTALL wget tar curl

# Download and install Prometheus
echo "[4/8] Downloading and installing Prometheus..."
cd /tmp
wget "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"
tar xzf "prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"

# Create directories and user
echo "[5/8] Creating Prometheus user and directories..."
useradd --no-create-home --shell /bin/false "$PROMETHEUS_USER" || true
mkdir -p "$PROMETHEUS_HOME" "$PROMETHEUS_CONFIG" "$PROMETHEUS_DATA"

# Install Prometheus binaries
cp "prometheus-${PROMETHEUS_VERSION}.linux-amd64/prometheus" "$PROMETHEUS_HOME/"
cp "prometheus-${PROMETHEUS_VERSION}.linux-amd64/promtool" "$PROMETHEUS_HOME/"
cp -r "prometheus-${PROMETHEUS_VERSION}.linux-amd64/consoles" "$PROMETHEUS_HOME/"
cp -r "prometheus-${PROMETHEUS_VERSION}.linux-amd64/console_libraries" "$PROMETHEUS_HOME/"

# Set permissions
chown -R "$PROMETHEUS_USER:$PROMETHEUS_USER" "$PROMETHEUS_HOME"
chown -R "$PROMETHEUS_USER:$PROMETHEUS_USER" "$PROMETHEUS_CONFIG"
chown -R "$PROMETHEUS_USER:$PROMETHEUS_USER" "$PROMETHEUS_DATA"
chmod 755 "$PROMETHEUS_HOME/prometheus" "$PROMETHEUS_HOME/promtool"

# Create Prometheus configuration
echo "[6/8] Creating Prometheus configuration..."
cat > "$PROMETHEUS_CONFIG/prometheus.yml" << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "jaeger_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - localhost:9093

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'jaeger-query'
    static_configs:
      - targets: ['localhost:16687']
    metrics_path: /metrics
    scrape_interval: 15s

  - job_name: 'jaeger-collector'
    static_configs:
      - targets: ['localhost:14269']
    metrics_path: /metrics
    scrape_interval: 15s

  - job_name: 'jaeger-agent'
    static_configs:
      - targets: ['localhost:14271']
    metrics_path: /metrics
    scrape_interval: 15s

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['localhost:9100']
EOF

# Create Jaeger alerting rules
cat > "$PROMETHEUS_CONFIG/jaeger_rules.yml" << 'EOF'
groups:
  - name: jaeger_alerts
    rules:
      - alert: JaegerCollectorDown
        expr: up{job="jaeger-collector"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Jaeger collector is down"
          description: "Jaeger collector has been down for more than 1 minute"

      - alert: JaegerQueryDown
        expr: up{job="jaeger-query"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Jaeger query service is down"
          description: "Jaeger query service has been down for more than 1 minute"

      - alert: JaegerHighSpanDropRate
        expr: rate(jaeger_collector_spans_dropped_total[5m]) > 100
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High span drop rate in Jaeger collector"
          description: "Jaeger collector is dropping {{ $value }} spans per second"

      - alert: JaegerHighTraceLatency
        expr: histogram_quantile(0.95, rate(jaeger_query_latency_bucket[5m])) > 1000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High trace query latency"
          description: "95th percentile trace query latency is {{ $value }}ms"
EOF

# Set configuration file permissions
chown "$PROMETHEUS_USER:$PROMETHEUS_USER" "$PROMETHEUS_CONFIG/prometheus.yml" "$PROMETHEUS_CONFIG/jaeger_rules.yml"
chmod 644 "$PROMETHEUS_CONFIG/prometheus.yml" "$PROMETHEUS_CONFIG/jaeger_rules.yml"

# Create systemd service
echo "[7/8] Creating systemd service..."
cat > /etc/systemd/system/prometheus.service << EOF
[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=$PROMETHEUS_USER
Group=$PROMETHEUS_USER
Type=simple
ExecStart=$PROMETHEUS_HOME/prometheus \\
    --config.file=$PROMETHEUS_CONFIG/prometheus.yml \\
    --storage.tsdb.path=$PROMETHEUS_DATA/ \\
    --web.console.templates=$PROMETHEUS_HOME/consoles \\
    --web.console.libraries=$PROMETHEUS_HOME/console_libraries \\
    --web.listen-address=0.0.0.0:9090 \\
    --web.enable-lifecycle

[Install]
WantedBy=multi-user.target
EOF

# Enable and start Prometheus
systemctl daemon-reload
systemctl enable prometheus
systemctl start prometheus

# Configure firewall if available
if command -v ufw >/dev/null 2>&1; then
    ufw allow 9090/tcp 2>/dev/null || true
elif command -v firewall-cmd >/dev/null 2>&1; then
    firewall-cmd --permanent --add-port=9090/tcp 2>/dev/null || true
    firewall-cmd --reload 2>/dev/null || true
fi

# Cleanup temporary files
rm -rf "/tmp/prometheus-${PROMETHEUS_VERSION}.linux-amd64"*

# Verify installation
echo "[8/8] Verifying installation..."
sleep 5

if systemctl is-active --quiet prometheus; then
    print_status "Prometheus service is running"
else
    print_error "Prometheus service failed to start"
    exit 1
fi

if curl -s http://localhost:9090/-/healthy >/dev/null; then
    print_status "Prometheus health check passed"
else
    print_warning "Prometheus health check failed - service may still be starting"
fi

# Clear trap
trap - ERR

print_status "Installation completed successfully!"
echo ""
echo "Prometheus is now running and configured for Jaeger monitoring:"
echo "  - Web UI: http://$(hostname -I | awk '{print $1}'):9090"
echo "  - Configuration: $PROMETHEUS_CONFIG/prometheus.yml"
echo "  - Alert rules: $PROMETHEUS_CONFIG/jaeger_rules.yml"
echo "  - Data directory: $PROMETHEUS_DATA"
echo ""
echo "Next steps:"
echo "  1. Install and configure Jaeger with metrics enabled"
echo "  2. Install Alertmanager for alert handling"
echo "  3. Install Grafana for visualization"
echo "  4. Configure dashboards for Jaeger metrics"