Install and configure OpenTelemetry Collector for distributed tracing and observability

#!/usr/bin/env bash
set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Default variables
OTEL_VERSION="0.91.0"
JAEGER_PORT="16686"

# Logging functions
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Cleanup function for rollback
cleanup() {
    log_error "Installation failed. Cleaning up..."
    sudo systemctl stop otelcol 2>/dev/null || true
    sudo systemctl disable otelcol 2>/dev/null || true
    sudo rm -f /etc/systemd/system/otelcol.service
    sudo rm -rf /etc/otelcol /var/lib/otelcol /var/log/otelcol
    sudo rm -f /usr/local/bin/otelcol
    sudo userdel otelcol 2>/dev/null || true
    sudo docker stop jaeger 2>/dev/null || true
    sudo docker rm jaeger 2>/dev/null || true
    exit 1
}

trap cleanup ERR

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -v, --version VERSION    OpenTelemetry Collector version (default: $OTEL_VERSION)"
    echo "  -p, --port PORT         Jaeger UI port (default: $JAEGER_PORT)"
    echo "  -h, --help              Show this help message"
    exit 1
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -v|--version) OTEL_VERSION="$2"; shift 2 ;;
        -p|--port) JAEGER_PORT="$2"; shift 2 ;;
        -h|--help) usage ;;
        *) log_error "Unknown option: $1"; usage ;;
    esac
done

# Check prerequisites
if [[ $EUID -ne 0 ]] && ! sudo -n true 2>/dev/null; then
    log_error "This script requires root privileges or passwordless sudo access"
    exit 1
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_UPDATE="apt update && apt upgrade -y"
            PKG_INSTALL="apt install -y"
            DOCKER_PKG="docker.io"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            DOCKER_PKG="docker"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            DOCKER_PKG="docker"
            ;;
        *)
            log_error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    log_error "Cannot detect distribution (missing /etc/os-release)"
    exit 1
fi

log_info "Detected distribution: $ID"
log_info "Installing OpenTelemetry Collector version $OTEL_VERSION"

# Step 1: Update system packages
echo "[1/8] Updating system packages..."
sudo $PKG_UPDATE
sudo $PKG_INSTALL curl wget gnupg2

# Step 2: Download OpenTelemetry Collector
echo "[2/8] Downloading OpenTelemetry Collector..."
cd /tmp
wget "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTEL_VERSION}/otelcol_${OTEL_VERSION}_linux_amd64.tar.gz"
tar -xzf "otelcol_${OTEL_VERSION}_linux_amd64.tar.gz"
sudo mv otelcol /usr/local/bin/
sudo chmod 755 /usr/local/bin/otelcol
sudo chown root:root /usr/local/bin/otelcol

# Step 3: Create OpenTelemetry user and directories
echo "[3/8] Creating OpenTelemetry user and directories..."
sudo useradd --system --shell /bin/false --home /var/lib/otelcol --create-home otelcol || true
sudo mkdir -p /etc/otelcol /var/lib/otelcol /var/log/otelcol
sudo chown otelcol:otelcol /var/lib/otelcol /var/log/otelcol
sudo chmod 755 /etc/otelcol
sudo chmod 750 /var/lib/otelcol /var/log/otelcol

# Step 4: Install Docker and Jaeger
echo "[4/8] Installing Docker and Jaeger..."
if [[ "$ID" == "ubuntu" || "$ID" == "debian" ]]; then
    sudo $PKG_INSTALL software-properties-common
fi

sudo $PKG_INSTALL $DOCKER_PKG
sudo systemctl enable docker
sudo systemctl start docker

# Wait for Docker to be ready
sleep 5

sudo docker run -d --name jaeger \
    --restart unless-stopped \
    -p ${JAEGER_PORT}:16686 \
    -p 14250:14250 \
    -p 14268:14268 \
    jaegertracing/all-in-one:latest

# Step 5: Create OpenTelemetry Collector configuration
echo "[5/8] Creating OpenTelemetry Collector configuration..."
sudo tee /etc/otelcol/config.yaml > /dev/null << 'EOF'
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
  jaeger:
    protocols:
      grpc:
        endpoint: 0.0.0.0:14250
      thrift_http:
        endpoint: 0.0.0.0:14268
  zipkin:
    endpoint: 0.0.0.0:9411
  prometheus:
    config:
      scrape_configs:
        - job_name: 'otelcol'
          scrape_interval: 10s
          static_configs:
            - targets: ['0.0.0.0:8888']

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024
  memory_limiter:
    limit_mib: 512
  resource:
    attributes:
      - key: service.instance.id
        from_attribute: host.name
        action: insert

exporters:
  jaeger:
    endpoint: localhost:14250
    tls:
      insecure: true
  prometheus:
    endpoint: "0.0.0.0:8889"
  logging:
    loglevel: info

extensions:
  health_check:
    endpoint: 0.0.0.0:13133
  pprof:
    endpoint: 0.0.0.0:1777

service:
  pipelines:
    traces:
      receivers: [otlp, jaeger, zipkin]
      processors: [memory_limiter, resource, batch]
      exporters: [jaeger, logging]
    metrics:
      receivers: [otlp, prometheus]
      processors: [memory_limiter, resource, batch]
      exporters: [prometheus, logging]
  extensions: [health_check, pprof]
  telemetry:
    logs:
      level: "info"
    metrics:
      address: 0.0.0.0:8888
EOF

# Step 6: Set configuration file permissions
echo "[6/8] Setting configuration file permissions..."
sudo chown root:otelcol /etc/otelcol/config.yaml
sudo chmod 640 /etc/otelcol/config.yaml

# Step 7: Create systemd service file
echo "[7/8] Creating systemd service..."
sudo tee /etc/systemd/system/otelcol.service > /dev/null << 'EOF'
[Unit]
Description=OpenTelemetry Collector
Documentation=https://opentelemetry.io/
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
User=otelcol
Group=otelcol
ExecStart=/usr/local/bin/otelcol --config=/etc/otelcol/config.yaml
Restart=on-failure
RestartSec=5
StandardOutput=journal
StandardError=journal
SyslogIdentifier=otelcol
KillMode=mixed
KillSignal=SIGTERM

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable otelcol
sudo systemctl start otelcol

# Step 8: Configure firewall (if active)
echo "[8/8] Configuring firewall..."
if command -v ufw &> /dev/null && sudo ufw status | grep -q "Status: active"; then
    sudo ufw allow 4317/tcp comment "OpenTelemetry OTLP gRPC"
    sudo ufw allow 4318/tcp comment "OpenTelemetry OTLP HTTP"
    sudo ufw allow ${JAEGER_PORT}/tcp comment "Jaeger UI"
elif command -v firewall-cmd &> /dev/null && sudo firewall-cmd --state &> /dev/null; then
    sudo firewall-cmd --permanent --add-port=4317/tcp --add-port=4318/tcp --add-port=${JAEGER_PORT}/tcp
    sudo firewall-cmd --reload
fi

# Verification checks
echo
log_info "Performing verification checks..."

# Check if OpenTelemetry Collector is running
if sudo systemctl is-active --quiet otelcol; then
    log_info "✓ OpenTelemetry Collector service is running"
else
    log_error "✗ OpenTelemetry Collector service is not running"
    sudo journalctl -u otelcol -n 10 --no-pager
fi

# Check if Jaeger is running
if sudo docker ps | grep -q jaeger; then
    log_info "✓ Jaeger container is running"
else
    log_error "✗ Jaeger container is not running"
fi

# Check if ports are listening
for port in 4317 4318 13133; do
    if netstat -tln 2>/dev/null | grep -q ":${port} " || ss -tln 2>/dev/null | grep -q ":${port} "; then
        log_info "✓ Port ${port} is listening"
    else
        log_warn "✗ Port ${port} is not listening"
    fi
done

echo
log_info "Installation completed successfully!"
log_info "Jaeger UI: http://localhost:${JAEGER_PORT}"
log_info "Health check: http://localhost:13133"
log_info "Metrics: http://localhost:8888/metrics"
log_info "OTLP gRPC endpoint: localhost:4317"
log_info "OTLP HTTP endpoint: localhost:4318"