Configure Kubernetes OpenTelemetry auto-instrumentation for microservices observability

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Script configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TEMP_DIR=$(mktemp -d)
NAMESPACE="${1:-default}"

# Usage message
usage() {
    cat << EOF
Usage: $0 [NAMESPACE]
Install and configure Kubernetes OpenTelemetry auto-instrumentation

Arguments:
    NAMESPACE    Kubernetes namespace for collector (default: default)

Examples:
    $0
    $0 monitoring
EOF
}

# Cleanup function
cleanup() {
    local exit_code=$?
    echo -e "${YELLOW}[INFO]${NC} Cleaning up temporary files..."
    rm -rf "$TEMP_DIR"
    if [ $exit_code -ne 0 ]; then
        echo -e "${RED}[ERROR]${NC} Installation failed. Check logs above for details."
        echo -e "${YELLOW}[INFO]${NC} To rollback, run: kubectl delete -f /tmp/otel-manifests/ 2>/dev/null || true"
    fi
    exit $exit_code
}

trap cleanup EXIT ERR

# Logging functions
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Detect OS distribution
detect_os() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian)
                PKG_MGR="apt"
                PKG_INSTALL="apt install -y"
                PKG_UPDATE="apt update"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora)
                PKG_MGR="dnf"
                PKG_INSTALL="dnf install -y"
                PKG_UPDATE="dnf check-update || true"
                ;;
            amzn)
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                PKG_UPDATE="yum check-update || true"
                ;;
            *)
                log_error "Unsupported distribution: $ID"
                exit 1
                ;;
        esac
    else
        log_error "Cannot detect OS distribution"
        exit 1
    fi
    log_info "Detected OS: $PRETTY_NAME ($PKG_MGR)"
}

# Check prerequisites
check_prerequisites() {
    log_info "[1/8] Checking prerequisites..."
    
    if [ "$EUID" -eq 0 ]; then
        log_error "Do not run this script as root. Use a user with sudo privileges."
        exit 1
    fi
    
    if ! command -v sudo >/dev/null 2>&1; then
        log_error "sudo is required but not installed"
        exit 1
    fi
    
    if ! command -v kubectl >/dev/null 2>&1; then
        log_error "kubectl is required but not installed"
        exit 1
    fi
    
    if ! kubectl cluster-info >/dev/null 2>&1; then
        log_error "kubectl cannot connect to Kubernetes cluster"
        exit 1
    fi
    
    log_success "Prerequisites check completed"
}

# Install system dependencies
install_dependencies() {
    log_info "[2/8] Installing system dependencies..."
    
    sudo $PKG_UPDATE
    
    case "$PKG_MGR" in
        apt)
            sudo $PKG_INSTALL curl wget jq
            ;;
        dnf|yum)
            sudo $PKG_INSTALL curl wget jq
            ;;
    esac
    
    log_success "System dependencies installed"
}

# Install cert-manager
install_cert_manager() {
    log_info "[3/8] Installing cert-manager..."
    
    if kubectl get namespace cert-manager >/dev/null 2>&1; then
        log_warning "cert-manager namespace already exists, skipping installation"
        return 0
    fi
    
    kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.0/cert-manager.yaml
    
    log_info "Waiting for cert-manager pods to be ready..."
    kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s
    
    log_success "cert-manager installed successfully"
}

# Install OpenTelemetry Operator
install_otel_operator() {
    log_info "[4/8] Installing OpenTelemetry Operator..."
    
    if kubectl get namespace opentelemetry-operator-system >/dev/null 2>&1; then
        log_warning "OpenTelemetry Operator already exists, skipping installation"
        return 0
    fi
    
    kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml
    
    log_info "Waiting for operator pods to be ready..."
    sleep 10
    kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=opentelemetry-operator -n opentelemetry-operator-system --timeout=300s
    
    log_success "OpenTelemetry Operator installed successfully"
}

# Create OpenTelemetry Collector configuration
create_otel_collector() {
    log_info "[5/8] Creating OpenTelemetry Collector configuration..."
    
    mkdir -p /tmp/otel-manifests
    
    cat > /tmp/otel-manifests/otel-collector.yaml << 'EOF'
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
  name: otel-collector
spec:
  mode: deployment
  config: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
      jaeger:
        protocols:
          grpc:
            endpoint: 0.0.0.0:14250
          thrift_http:
            endpoint: 0.0.0.0:14268

    processors:
      batch:
        timeout: 1s
        send_batch_size: 1024
      memory_limiter:
        limit_mib: 512

    exporters:
      logging:
        loglevel: info
      jaeger:
        endpoint: jaeger-collector.jaeger:14250
        tls:
          insecure: true
      prometheus:
        endpoint: "0.0.0.0:8889"

    extensions:
      health_check:
        endpoint: 0.0.0.0:13133
      pprof:
        endpoint: 0.0.0.0:1777
      zpages:
        endpoint: 0.0.0.0:55679

    service:
      extensions: [health_check, pprof, zpages]
      pipelines:
        traces:
          receivers: [otlp, jaeger]
          processors: [memory_limiter, batch]
          exporters: [logging, jaeger]
        metrics:
          receivers: [otlp]
          processors: [memory_limiter, batch]
          exporters: [logging, prometheus]
EOF
    
    sed -i "s/^metadata:/metadata:\n  namespace: $NAMESPACE/" /tmp/otel-manifests/otel-collector.yaml
    kubectl apply -f /tmp/otel-manifests/otel-collector.yaml -n "$NAMESPACE"
    
    log_success "OpenTelemetry Collector configuration created"
}

# Install Jaeger
install_jaeger() {
    log_info "[6/8] Installing Jaeger for trace visualization..."
    
    cat > /tmp/otel-manifests/jaeger.yaml << 'EOF'
apiVersion: v1
kind: Namespace
metadata:
  name: jaeger
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: jaeger
  namespace: jaeger
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jaeger
  template:
    metadata:
      labels:
        app: jaeger
    spec:
      containers:
      - name: jaeger
        image: jaegertracing/all-in-one:1.50
        ports:
        - containerPort: 16686
          name: ui
        - containerPort: 14250
          name: grpc
        - containerPort: 14268
          name: http
        env:
        - name: COLLECTOR_OTLP_ENABLED
          value: "true"
        resources:
          requests:
            memory: "256Mi"
            cpu: "100m"
          limits:
            memory: "512Mi"
            cpu: "500m"
---
apiVersion: v1
kind: Service
metadata:
  name: jaeger-ui
  namespace: jaeger
spec:
  selector:
    app: jaeger
  ports:
  - port: 16686
    targetPort: 16686
    name: ui
  type: ClusterIP
---
apiVersion: v1
kind: Service
metadata:
  name: jaeger-collector
  namespace: jaeger
spec:
  selector:
    app: jaeger
  ports:
  - port: 14250
    targetPort: 14250
    name: grpc
  - port: 14268
    targetPort: 14268
    name: http
  type: ClusterIP
EOF
    
    kubectl apply -f /tmp/otel-manifests/jaeger.yaml
    
    log_info "Waiting for Jaeger to be ready..."
    kubectl wait --for=condition=ready pod -l app=jaeger -n jaeger --timeout=300s
    
    log_success "Jaeger installed successfully"
}

# Create instrumentation configuration
create_instrumentation() {
    log_info "[7/8] Creating auto-instrumentation configuration..."
    
    cat > /tmp/otel-manifests/instrumentation.yaml << 'EOF'
apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: otel-instrumentation
spec:
  exporter:
    endpoint: http://otel-collector:4318
  propagators:
    - tracecontext
    - baggage
    - b3
  sampler:
    type: parentbased_traceidratio
    argument: "1"
  java:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:1.32.0
  nodejs:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-nodejs:0.44.0
  python:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-python:0.41b0
  dotnet:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-dotnet:1.0.0
EOF
    
    sed -i "s/^metadata:/metadata:\n  namespace: $NAMESPACE/" /tmp/otel-manifests/instrumentation.yaml
    kubectl apply -f /tmp/otel-manifests/instrumentation.yaml -n "$NAMESPACE"
    
    log_success "Auto-instrumentation configuration created"
}

# Verify installation
verify_installation() {
    log_info "[8/8] Verifying installation..."
    
    # Check cert-manager
    if ! kubectl get pods -n cert-manager | grep -q Running; then
        log_error "cert-manager pods are not running"
        return 1
    fi
    
    # Check OpenTelemetry Operator
    if ! kubectl get pods -n opentelemetry-operator-system | grep -q Running; then
        log_error "OpenTelemetry Operator pods are not running"
        return 1
    fi
    
    # Check Jaeger
    if ! kubectl get pods -n jaeger | grep -q Running; then
        log_error "Jaeger pods are not running"
        return 1
    fi
    
    # Check OpenTelemetry Collector
    if ! kubectl get opentelemetrycollector otel-collector -n "$NAMESPACE" >/dev/null 2>&1; then
        log_error "OpenTelemetry Collector not found"
        return 1
    fi
    
    # Check Instrumentation
    if ! kubectl get instrumentation otel-instrumentation -n "$NAMESPACE" >/dev/null 2>&1; then
        log_error "Instrumentation configuration not found"
        return 1
    fi
    
    log_success "All components verified successfully"
    
    echo
    log_success "OpenTelemetry auto-instrumentation setup completed!"
    echo
    log_info "Next steps:"
    echo "1. To enable auto-instrumentation for a deployment, add annotation:"
    echo "   instrumentation.opentelemetry.io/inject-java: \"$NAMESPACE/otel-instrumentation\""
    echo "   (replace 'java' with your language: nodejs, python, dotnet)"
    echo
    echo "2. Access Jaeger UI:"
    echo "   kubectl port-forward -n jaeger svc/jaeger-ui 16686:16686"
    echo "   Then open http://localhost:16686"
    echo
    log_info "Configuration files saved to: /tmp/otel-manifests/"
}

# Main execution
main() {
    if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
        usage
        exit 0
    fi
    
    echo -e "${GREEN}OpenTelemetry Auto-Instrumentation Setup${NC}"
    echo "========================================"
    
    detect_os
    check_prerequisites
    install_dependencies
    install_cert_manager
    install_otel_operator
    create_otel_collector
    install_jaeger
    create_instrumentation
    verify_installation
}

main "$@"