Integrate Jaeger with Kubernetes service mesh for comprehensive distributed tracing

Update system packages and install prerequisites

Start by updating your package manager and installing required tools for Kubernetes and container management.

sudo apt update && sudo apt upgrade -y
sudo apt install -y curl wget gnupg software-properties-common apt-transport-https ca-certificates

sudo dnf update -y
sudo dnf install -y curl wget gnupg2 ca-certificates

Install kubectl and helm

Install kubectl for Kubernetes cluster management and Helm for package deployment.

curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list

sudo apt update
sudo apt install helm

curl https://get.helm.sh/helm-v3.12.0-linux-amd64.tar.gz -o helm.tar.gz
tar -zxvf helm.tar.gz
sudo mv linux-amd64/helm /usr/local/bin/helm

Install Istio service mesh

Download and install Istio service mesh which provides the infrastructure for distributed tracing integration.

curl -L https://istio.io/downloadIstio | sh -
cd istio-*
sudo cp bin/istioctl /usr/local/bin/
istioctl install --set values.defaultRevision=default -y

Enable Istio sidecar injection

Configure automatic sidecar injection for namespaces where you want distributed tracing enabled.

kubectl label namespace default istio-injection=enabled
kubectl create namespace jaeger-system
kubectl label namespace jaeger-system istio-injection=enabled

Install Jaeger Operator

Deploy the Jaeger Operator which manages Jaeger instances and their lifecycle in Kubernetes.

kubectl create namespace observability
kubectl create -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.49.0/jaeger-operator.yaml -n observability

Deploy Elasticsearch for Jaeger storage

Set up Elasticsearch as the backend storage for Jaeger traces with proper configuration for production workloads.

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch
  namespace: jaeger-system
spec:
  serviceName: elasticsearch
  replicas: 3
  selector:
    matchLabels:
      app: elasticsearch
  template:
    metadata:
      labels:
        app: elasticsearch
    spec:
      containers:
      - name: elasticsearch
        image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0
        env:
        - name: discovery.type
          value: single-node
        - name: ES_JAVA_OPTS
          value: "-Xms2g -Xmx2g"
        - name: xpack.security.enabled
          value: "false"
        ports:
        - containerPort: 9200
        - containerPort: 9300
        volumeMounts:
        - name: elasticsearch-data
          mountPath: /usr/share/elasticsearch/data
        resources:
          requests:
            memory: "4Gi"
            cpu: "1000m"
          limits:
            memory: "4Gi"
            cpu: "1000m"
  volumeClaimTemplates:
  - metadata:
      name: elasticsearch-data
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 50Gi
---
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch
  namespace: jaeger-system
spec:
  selector:
    app: elasticsearch
  ports:
  - port: 9200
    targetPort: 9200

kubectl apply -f /tmp/elasticsearch.yaml

Configure Jaeger instance with Elasticsearch backend

Create a production-ready Jaeger instance that uses Elasticsearch for storage and integrates with Istio.

apiVersion: jaegertracing.io/v1
kind: Jaeger
metadata:
  name: jaeger-production
  namespace: jaeger-system
spec:
  strategy: production
  storage:
    type: elasticsearch
    options:
      es:
        server-urls: http://elasticsearch.jaeger-system.svc.cluster.local:9200
        index-prefix: jaeger
        username: ""
        password: ""
    esIndexCleaner:
      enabled: true
      numberOfDays: 7
      schedule: "55 23   *"
  collector:
    replicas: 3
    resources:
      limits:
        cpu: 500m
        memory: 1Gi
      requests:
        cpu: 256m
        memory: 512Mi
    options:
      collector:
        num-workers: 100
        queue-size: 2000
  query:
    replicas: 2
    resources:
      limits:
        cpu: 500m
        memory: 512Mi
      requests:
        cpu: 256m
        memory: 256Mi
  ingester:
    replicas: 2
    resources:
      limits:
        cpu: 500m
        memory: 1Gi
      requests:
        cpu: 256m
        memory: 512Mi

kubectl apply -f /tmp/jaeger-production.yaml

Configure Istio for Jaeger integration

Update Istio configuration to send traces to your Jaeger instance automatically.

apiVersion: v1
kind: ConfigMap
metadata:
  name: istio
  namespace: istio-system
data:
  mesh: |
    extensionProviders:
    - name: jaeger
      envoyOtelAls:
        service: jaeger-production-collector.jaeger-system.svc.cluster.local
        port: 14268
    defaultConfig:
      proxyStatsMatcher:
        inclusionRegexps:
        - ".outlier_detection."
        - ".circuit_breakers."
        - ".upstream_rq_retry."
        - "._cx_."
      tracing:
        sampling: 1.0
        custom_tags:
          my_tag_header:
            header:
              name: x-my-tag
    defaultProviders:
      tracing:
      - jaeger

kubectl apply -f /tmp/istio-jaeger-config.yaml
istioctl proxy-config cluster -n istio-system istio-proxy

Deploy OpenTelemetry Collector

Install OpenTelemetry Collector to handle trace collection and forwarding from applications.

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm repo update

mode: deployment
replicaCount: 3
config:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318
    jaeger:
      protocols:
        grpc:
          endpoint: 0.0.0.0:14250
        thrift_http:
          endpoint: 0.0.0.0:14268
  processors:
    batch:
      timeout: 5s
      send_batch_size: 1024
    memory_limiter:
      limit_mib: 512
  exporters:
    jaeger:
      endpoint: jaeger-production-collector.jaeger-system.svc.cluster.local:14250
      tls:
        insecure: true
  service:
    pipelines:
      traces:
        receivers: [otlp, jaeger]
        processors: [memory_limiter, batch]
        exporters: [jaeger]
resources:
  limits:
    cpu: 500m
    memory: 1Gi
  requests:
    cpu: 256m
    memory: 512Mi

helm install otel-collector open-telemetry/opentelemetry-collector -f /tmp/otel-values.yaml -n observability

Configure tracing for sample application

Deploy a sample microservice application with automatic tracing enabled through Istio sidecar injection.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: frontend
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      app: frontend
  template:
    metadata:
      labels:
        app: frontend
        version: v1
    spec:
      containers:
      - name: frontend
        image: gcr.io/google-samples/microservices-demo/frontend:v0.8.0
        ports:
        - containerPort: 8080
        env:
        - name: PORT
          value: "8080"
        resources:
          requests:
            cpu: 100m
            memory: 64Mi
          limits:
            cpu: 200m
            memory: 128Mi
---
apiVersion: v1
kind: Service
metadata:
  name: frontend
  namespace: default
spec:
  selector:
    app: frontend
  ports:
  - port: 80
    targetPort: 8080
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: backend
  template:
    metadata:
      labels:
        app: backend
        version: v1
    spec:
      containers:
      - name: backend
        image: gcr.io/google-samples/microservices-demo/cartservice:v0.8.0
        ports:
        - containerPort: 7070
        env:
        - name: PORT
          value: "7070"
        resources:
          requests:
            cpu: 100m
            memory: 64Mi
          limits:
            cpu: 200m
            memory: 128Mi
---
apiVersion: v1
kind: Service
metadata:
  name: backend
  namespace: default
spec:
  selector:
    app: backend
  ports:
  - port: 7070
    targetPort: 7070

kubectl apply -f /tmp/sample-app.yaml

Configure sampling and trace retention policies

Set up intelligent sampling to control trace volume and configure retention policies for optimal storage usage.

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: tracing-config
  namespace: istio-system
spec:
  tracing:
  - providers:
    - name: jaeger
  - customTags:
      request_id:
        header:
          name: x-request-id
      user_agent:
        header:
          name: user-agent
  - sampling:
    - value: 10.0
      match:
        headers:
          x-trace-sample:
            exact: "full"
    - value: 1.0

kubectl apply -f /tmp/telemetry-config.yaml

Set up Jaeger UI access and ingress

Configure secure access to Jaeger UI through Kubernetes ingress with proper authentication.

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jaeger-ingress
  namespace: jaeger-system
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
    cert-manager.io/cluster-issuer: letsencrypt-prod
    nginx.ingress.kubernetes.io/auth-type: basic
    nginx.ingress.kubernetes.io/auth-secret: jaeger-auth
    nginx.ingress.kubernetes.io/auth-realm: 'Jaeger Authentication Required'
spec:
  tls:
  - hosts:
    - jaeger.example.com
    secretName: jaeger-tls
  rules:
  - host: jaeger.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: jaeger-production-query
            port:
              number: 16686

kubectl create secret generic jaeger-auth --from-literal=auth=$(htpasswd -nb admin 'YourSecurePassword123!') -n jaeger-system
kubectl apply -f /tmp/jaeger-ingress.yaml

Configure Grafana integration for Jaeger metrics

Set up Grafana dashboards to visualize Jaeger metrics and create alerts for trace anomalies. This step references our existing Grafana monitoring setup.

{
  "dashboard": {
    "id": null,
    "title": "Jaeger Tracing Overview",
    "tags": ["jaeger", "tracing"],
    "timezone": "browser",
    "panels": [
      {
        "title": "Traces per Second",
        "type": "stat",
        "targets": [
          {
            "expr": "rate(jaeger_collector_traces_received_total[5m])",
            "legendFormat": "Traces/sec"
          }
        ]
      },
      {
        "title": "Trace Duration Distribution",
        "type": "heatmap",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, rate(jaeger_query_requests_duration_seconds_bucket[5m]))",
            "legendFormat": "95th percentile"
          }
        ]
      },
      {
        "title": "Error Rate by Service",
        "type": "table",
        "targets": [
          {
            "expr": "rate(jaeger_collector_spans_received_total{result=\"err\"}[5m]) by (service)",
            "legendFormat": "{{service}}"
          }
        ]
      }
    ],
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "refresh": "30s"
  }
}

curl -X POST http://admin:admin@grafana.example.com/api/dashboards/db \
  -H "Content-Type: application/json" \
  -d @/tmp/jaeger-grafana-dashboard.json

Set up automatic instrumentation for Java applications

Configure OpenTelemetry Java agent for automatic trace collection without code changes.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: java-microservice
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      app: java-microservice
  template:
    metadata:
      labels:
        app: java-microservice
    spec:
      initContainers:
      - name: otel-agent-download
        image: busybox:1.35
        command:
        - sh
        - -c
        - |
          wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.32.0/opentelemetry-javaagent.jar -O /shared/opentelemetry-javaagent.jar
        volumeMounts:
        - name: shared-data
          mountPath: /shared
      containers:
      - name: app
        image: openjdk:11-jre
        env:
        - name: JAVA_TOOL_OPTIONS
          value: "-javaagent:/shared/opentelemetry-javaagent.jar"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
          value: "http://otel-collector-opentelemetry-collector.observability.svc.cluster.local:4318"
        - name: OTEL_SERVICE_NAME
          value: "java-microservice"
        - name: OTEL_RESOURCE_ATTRIBUTES
          value: "service.version=1.0,environment=production"
        volumeMounts:
        - name: shared-data
          mountPath: /shared
        ports:
        - containerPort: 8080
      volumes:
      - name: shared-data
        emptyDir: {}

kubectl apply -f /tmp/java-app-instrumentation.yaml

Configure Python application tracing

Set up OpenTelemetry for Python applications with automatic Django and Flask instrumentation.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: python-microservice
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      app: python-microservice
  template:
    metadata:
      labels:
        app: python-microservice
    spec:
      containers:
      - name: app
        image: python:3.11-slim
        command:
        - sh
        - -c
        - |
          pip install opentelemetry-distro opentelemetry-exporter-otlp
          opentelemetry-bootstrap --action=install
          opentelemetry-instrument python app.py
        env:
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
          value: "http://otel-collector-opentelemetry-collector.observability.svc.cluster.local:4318"
        - name: OTEL_SERVICE_NAME
          value: "python-microservice"
        - name: OTEL_RESOURCE_ATTRIBUTES
          value: "service.version=1.0,environment=production"
        - name: OTEL_PYTHON_LOG_CORRELATION
          value: "true"
        ports:
        - containerPort: 5000

kubectl apply -f /tmp/python-app-instrumentation.yaml

Configure Node.js application tracing

Set up automatic tracing for Node.js applications using OpenTelemetry auto-instrumentation. This integrates well with distributed tracing setups for multiple language stacks.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nodejs-microservice
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nodejs-microservice
  template:
    metadata:
      labels:
        app: nodejs-microservice
    spec:
      containers:
      - name: app
        image: node:18-alpine
        command:
        - sh
        - -c
        - |
          npm install @opentelemetry/api @opentelemetry/auto-instrumentations-node @opentelemetry/exporter-trace-otlp-http
          node --require @opentelemetry/auto-instrumentations-node/register app.js
        env:
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
          value: "http://otel-collector-opentelemetry-collector.observability.svc.cluster.local:4318"
        - name: OTEL_SERVICE_NAME
          value: "nodejs-microservice"
        - name: OTEL_RESOURCE_ATTRIBUTES
          value: "service.version=1.0,environment=production"
        ports:
        - containerPort: 3000

kubectl apply -f /tmp/nodejs-app-instrumentation.yaml

Create RBAC policies for Jaeger components

Configure Role-Based Access Control to secure Jaeger components and limit access to sensitive tracing data.

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: jaeger-operator
rules:
apiGroups: [""]
  resources: ["pods", "services", "endpoints", "configmaps", "secrets"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
apiGroups: ["apps"]
  resources: ["deployments", "replicasets"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
apiGroups: ["jaegertracing.io"]
  resources: ["jaegers"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jaeger-operator
  namespace: observability
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: jaeger-operator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: jaeger-operator
subjects:
kind: ServiceAccount
  name: jaeger-operator
  namespace: observability
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: jaeger-reader
  namespace: jaeger-system
rules:
apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "list"]
apiGroups: [""]
  resources: ["services"]
  resourceNames: ["jaeger-production-query"]
  verbs: ["get"]

kubectl apply -f /tmp/jaeger-rbac.yaml

Configure network policies for Jaeger isolation

Set up network policies to control traffic flow and enhance security for Jaeger components.

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: jaeger-collector-policy
  namespace: jaeger-system
spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/component: collector
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          name: default
    - namespaceSelector:
        matchLabels:
          name: observability
    ports:
    - protocol: TCP
      port: 14268
    - protocol: TCP
      port: 14250
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: elasticsearch
    ports:
    - protocol: TCP
      port: 9200
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: jaeger-query-policy
  namespace: jaeger-system
spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/component: query
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          name: istio-system
    ports:
    - protocol: TCP
      port: 16686
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: elasticsearch
    ports:
    - protocol: TCP
      port: 9200

kubectl apply -f /tmp/jaeger-network-policy.yaml