Integrate Jaeger with Kubernetes and Istio service mesh for distributed tracing

#!/usr/bin/env bash
set -euo pipefail

# Jaeger + Istio + Kubernetes Distributed Tracing Install Script
# Production-quality installer for distributed tracing with Jaeger

# Colors for output
readonly RED='\033[0;31m'
readonly GREEN='\033[0;32m'
readonly YELLOW='\033[1;33m'
readonly BLUE='\033[0;34m'
readonly NC='\033[0m' # No Color

# Configuration
readonly SCRIPT_NAME=$(basename "$0")
readonly JAEGER_OPERATOR_VERSION="v1.51.0"
readonly ELASTICSEARCH_VERSION="8.11.0"
readonly NAMESPACE="observability"
readonly DOMAIN="${1:-jaeger.local}"

# Print colored output
print_status() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

print_step() {
    echo -e "${BLUE}[$1]${NC} $2"
}

# Usage message
usage() {
    cat << EOF
Usage: $SCRIPT_NAME [DOMAIN]

Install Jaeger with Istio service mesh for distributed tracing on Kubernetes.

Arguments:
    DOMAIN    Domain for Jaeger UI (default: jaeger.local)

Examples:
    $SCRIPT_NAME
    $SCRIPT_NAME jaeger.example.com

Requirements:
    - Kubernetes cluster (v1.24+)
    - Istio service mesh installed
    - kubectl with cluster-admin privileges
    - At least 4GB available memory
EOF
}

# Cleanup function
cleanup() {
    local exit_code=$?
    if [ $exit_code -ne 0 ]; then
        print_error "Installation failed. Cleaning up..."
        kubectl delete namespace "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true
        rm -f /tmp/elasticsearch-deployment.yaml /tmp/jaeger-instance.yaml 2>/dev/null || true
    fi
    exit $exit_code
}

trap cleanup ERR EXIT

# Auto-detect distro and set package manager
detect_distro() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian) 
                PKG_MGR="apt"
                PKG_INSTALL="apt install -y"
                PKG_UPDATE="apt update"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora) 
                PKG_MGR="dnf"
                PKG_INSTALL="dnf install -y"
                PKG_UPDATE="dnf check-update || true"
                ;;
            amzn) 
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                PKG_UPDATE="yum update -y --security"
                ;;
            *) 
                print_error "Unsupported distribution: $ID"
                exit 1
                ;;
        esac
    else
        print_error "Cannot detect OS distribution"
        exit 1
    fi
    export PKG_MGR PKG_INSTALL PKG_UPDATE
}

# Check if running as root or with sudo
check_privileges() {
    if [ "$EUID" -eq 0 ]; then
        print_warning "Running as root. Consider using sudo instead."
        SUDO_CMD=""
    elif command -v sudo >/dev/null 2>&1; then
        SUDO_CMD="sudo"
    else
        print_error "This script requires root privileges or sudo"
        exit 1
    fi
}

# Validate arguments
validate_args() {
    if [ "$#" -gt 1 ]; then
        print_error "Too many arguments"
        usage
        exit 1
    fi

    if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
        usage
        exit 0
    fi
}

# Check prerequisites
check_prerequisites() {
    print_step "1/8" "Checking prerequisites..."

    # Check kubectl
    if ! command -v kubectl >/dev/null 2>&1; then
        print_error "kubectl is not installed"
        exit 1
    fi

    # Check cluster connectivity
    if ! kubectl cluster-info >/dev/null 2>&1; then
        print_error "Cannot connect to Kubernetes cluster"
        exit 1
    fi

    # Check cluster version
    local k8s_version
    k8s_version=$(kubectl version --client -o yaml 2>/dev/null | grep gitVersion | head -1 | awk '{print $2}' | tr -d '"v')
    local major_version
    major_version=$(echo "$k8s_version" | cut -d. -f1)
    local minor_version
    minor_version=$(echo "$k8s_version" | cut -d. -f2)
    
    if [ "$major_version" -lt 1 ] || [ "$major_version" -eq 1 ] && [ "$minor_version" -lt 24 ]; then
        print_error "Kubernetes version 1.24+ required, found: $k8s_version"
        exit 1
    fi

    # Check Istio installation
    if ! kubectl get namespace istio-system >/dev/null 2>&1; then
        print_error "Istio not found. Please install Istio service mesh first"
        exit 1
    fi

    print_status "Prerequisites validated successfully"
}

# Create namespace
create_namespace() {
    print_step "2/8" "Creating observability namespace..."
    
    kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
    print_status "Namespace '$NAMESPACE' ready"
}

# Install Jaeger operator
install_jaeger_operator() {
    print_step "3/8" "Installing Jaeger operator..."
    
    kubectl apply -f "https://github.com/jaegertracing/jaeger-operator/releases/download/${JAEGER_OPERATOR_VERSION}/jaeger-operator.yaml" -n "$NAMESPACE"
    
    # Wait for operator to be ready
    print_status "Waiting for Jaeger operator to be ready..."
    kubectl wait --for=condition=available deployment/jaeger-operator -n "$NAMESPACE" --timeout=300s
    
    print_status "Jaeger operator installed successfully"
}

# Deploy Elasticsearch
deploy_elasticsearch() {
    print_step "4/8" "Deploying Elasticsearch for trace storage..."
    
    cat > /tmp/elasticsearch-deployment.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: elasticsearch
  namespace: $NAMESPACE
spec:
  replicas: 1
  selector:
    matchLabels:
      app: elasticsearch
  template:
    metadata:
      labels:
        app: elasticsearch
    spec:
      containers:
      - name: elasticsearch
        image: docker.elastic.co/elasticsearch/elasticsearch:$ELASTICSEARCH_VERSION
        ports:
        - containerPort: 9200
        - containerPort: 9300
        env:
        - name: discovery.type
          value: single-node
        - name: ES_JAVA_OPTS
          value: "-Xms2g -Xmx2g"
        - name: xpack.security.enabled
          value: "false"
        resources:
          requests:
            memory: 2Gi
            cpu: 500m
          limits:
            memory: 4Gi
            cpu: 1000m
        volumeMounts:
        - name: elasticsearch-storage
          mountPath: /usr/share/elasticsearch/data
        securityContext:
          runAsNonRoot: true
          runAsUser: 1000
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
      volumes:
      - name: elasticsearch-storage
        emptyDir: {}
      securityContext:
        fsGroup: 1000
---
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch
  namespace: $NAMESPACE
spec:
  selector:
    app: elasticsearch
  ports:
  - port: 9200
    targetPort: 9200
    name: http
  - port: 9300
    targetPort: 9300
    name: transport
EOF

    chmod 644 /tmp/elasticsearch-deployment.yaml
    kubectl apply -f /tmp/elasticsearch-deployment.yaml
    
    print_status "Waiting for Elasticsearch to be ready..."
    kubectl wait --for=condition=available deployment/elasticsearch -n "$NAMESPACE" --timeout=300s
    
    print_status "Elasticsearch deployed successfully"
}

# Create Jaeger instance
create_jaeger_instance() {
    print_step "5/8" "Creating Jaeger production instance..."
    
    cat > /tmp/jaeger-instance.yaml << EOF
apiVersion: jaegertracing.io/v1
kind: Jaeger
metadata:
  name: jaeger-production
  namespace: $NAMESPACE
spec:
  strategy: production
  storage:
    type: elasticsearch
    options:
      es:
        server-urls: http://elasticsearch:9200
        index-prefix: jaeger
        username: ""
        password: ""
        tls:
          enabled: false
    esIndexCleaner:
      enabled: true
      numberOfDays: 7
      schedule: "55 23 * * *"
  collector:
    replicas: 2
    resources:
      requests:
        memory: 512Mi
        cpu: 200m
      limits:
        memory: 1Gi
        cpu: 500m
  query:
    replicas: 2
    resources:
      requests:
        memory: 256Mi
        cpu: 100m
      limits:
        memory: 512Mi
        cpu: 200m
  ingress:
    enabled: false
EOF

    chmod 644 /tmp/jaeger-instance.yaml
    kubectl apply -f /tmp/jaeger-instance.yaml
    
    print_status "Waiting for Jaeger components to be ready..."
    sleep 30
    kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=jaeger -n "$NAMESPACE" --timeout=300s
    
    print_status "Jaeger instance created successfully"
}

# Configure Istio telemetry
configure_istio_telemetry() {
    print_step "6/8" "Configuring Istio telemetry for Jaeger..."
    
    kubectl patch configmap istio -n istio-system --type merge -p='{"data":{"mesh":"defaultProviders:\n  tracing:\n  - jaeger\ndefaultConfig:\n  proxyStatsMatcher:\n    inclusionRegexps:\n    - \".*outlier_detection.*\"\n    - \".*circuit_breakers.*\"\n    - \".*upstream_rq_retry.*\"\n    - \".*_cx_.*\"\n  tracing:\n    zipkin:\n      address: jaeger-production-collector.observability:9411\nproxyMetadata:\n  PILOT_ENABLE_WORKLOAD_ENTRY_AUTOREGISTRATION: true"}}'
    
    # Restart Istio components to pick up new configuration
    kubectl rollout restart deployment/istiod -n istio-system
    kubectl rollout status deployment/istiod -n istio-system --timeout=300s
    
    print_status "Istio telemetry configured for Jaeger"
}

# Create service and ingress
create_access() {
    print_step "7/8" "Setting up Jaeger UI access..."
    
    # Create NodePort service for Jaeger UI
    cat << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: jaeger-ui-nodeport
  namespace: $NAMESPACE
spec:
  type: NodePort
  selector:
    app.kubernetes.io/name: jaeger-production-query
  ports:
  - port: 16686
    targetPort: 16686
    nodePort: 30686
    protocol: TCP
EOF
    
    print_status "Jaeger UI accessible via NodePort 30686"
}

# Verify installation
verify_installation() {
    print_step "8/8" "Verifying installation..."
    
    # Check all pods are running
    local failed_pods
    failed_pods=$(kubectl get pods -n "$NAMESPACE" --field-selector=status.phase!=Running --no-headers | wc -l)
    
    if [ "$failed_pods" -gt 0 ]; then
        print_warning "Some pods are not running:"
        kubectl get pods -n "$NAMESPACE" --field-selector=status.phase!=Running
    fi
    
    # Check Jaeger collector service
    if kubectl get service jaeger-production-collector -n "$NAMESPACE" >/dev/null 2>&1; then
        print_status "✓ Jaeger collector service ready"
    else
        print_error "✗ Jaeger collector service not found"
        return 1
    fi
    
    # Check Jaeger query service
    if kubectl get service jaeger-production-query -n "$NAMESPACE" >/dev/null 2>&1; then
        print_status "✓ Jaeger query service ready"
    else
        print_error "✗ Jaeger query service not found"
        return 1
    fi
    
    print_status "Installation completed successfully!"
    echo
    print_status "Access Jaeger UI:"
    echo "  - NodePort: http://<node-ip>:30686"
    echo "  - Port-forward: kubectl port-forward -n $NAMESPACE service/jaeger-production-query 16686:16686"
    echo
    print_status "To enable tracing for a namespace, run:"
    echo "  kubectl label namespace <namespace> istio-injection=enabled"
}

# Main execution
main() {
    if [ "$#" -gt 0 ]; then
        validate_args "$@"
    fi
    
    detect_distro
    check_privileges
    check_prerequisites
    create_namespace
    install_jaeger_operator
    deploy_elasticsearch
    create_jaeger_instance
    configure_istio_telemetry
    create_access
    verify_installation
    
    # Cleanup temp files
    rm -f /tmp/elasticsearch-deployment.yaml /tmp/jaeger-instance.yaml
}

main "$@"