Implement Jaeger security with TLS encryption and authentication for distributed tracing

Install required packages

Install Jaeger components and certificate management tools for the security implementation.

sudo apt update
sudo apt install -y wget curl openssl nginx

sudo dnf update -y
sudo dnf install -y wget curl openssl nginx

Download Jaeger components

Download the latest Jaeger binaries with security features enabled.

cd /opt
sudo wget https://github.com/jaegertracing/jaeger/releases/download/v1.52.0/jaeger-1.52.0-linux-amd64.tar.gz
sudo tar -xzf jaeger-1.52.0-linux-amd64.tar.gz
sudo mv jaeger-1.52.0-linux-amd64 jaeger
sudo chown -R root:root /opt/jaeger

Create certificate authority

Generate a private Certificate Authority (CA) to sign TLS certificates for Jaeger components.

sudo mkdir -p /etc/jaeger/certs
cd /etc/jaeger/certs

Generate CA private key
sudo openssl genrsa -out ca-key.pem 4096

Generate CA certificate
sudo openssl req -new -x509 -days 3650 -key ca-key.pem -out ca-cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=IT/CN=Jaeger-CA"

Generate server certificates

Create TLS certificates for Jaeger Collector and Query services using the CA.

# Generate server private key
sudo openssl genrsa -out server-key.pem 4096

Create certificate signing request
sudo openssl req -new -key server-key.pem -out server.csr -subj "/C=US/ST=State/L=City/O=Organization/OU=IT/CN=example.com"

Generate server certificate
sudo openssl x509 -req -days 365 -in server.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem

Set appropriate permissions
sudo chmod 600 /etc/jaeger/certs/*-key.pem
sudo chmod 644 /etc/jaeger/certs/*-cert.pem
sudo rm server.csr

Generate client certificates

Create client certificates for mutual TLS authentication between Jaeger components.

# Generate client private key
sudo openssl genrsa -out client-key.pem 4096

Create client certificate signing request
sudo openssl req -new -key client-key.pem -out client.csr -subj "/C=US/ST=State/L=City/O=Organization/OU=IT/CN=jaeger-client"

Generate client certificate
sudo openssl x509 -req -days 365 -in client.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem
sudo rm client.csr

Configure Jaeger Collector with TLS

Create the Jaeger Collector configuration with TLS encryption and authentication enabled.

collector:
  grpc-server:
    host-port: "0.0.0.0:14250"
    tls:
      enabled: true
      cert: "/etc/jaeger/certs/server-cert.pem"
      key: "/etc/jaeger/certs/server-key.pem"
      ca: "/etc/jaeger/certs/ca-cert.pem"
      client-ca: "/etc/jaeger/certs/ca-cert.pem"
  http-server:
    host-port: "0.0.0.0:14268"
    tls:
      enabled: true
      cert: "/etc/jaeger/certs/server-cert.pem"
      key: "/etc/jaeger/certs/server-key.pem"
  otlp:
    enabled: true
    grpc:
      host-port: "0.0.0.0:4317"
      tls:
        enabled: true
        cert: "/etc/jaeger/certs/server-cert.pem"
        key: "/etc/jaeger/certs/server-key.pem"
        ca: "/etc/jaeger/certs/ca-cert.pem"
        client-ca: "/etc/jaeger/certs/ca-cert.pem"
span-storage:
  type: memory

Configure Jaeger Query with TLS and JWT

Set up the Query service with TLS encryption and JWT authentication for API access.

query:
  grpc-server:
    host-port: "0.0.0.0:16685"
    tls:
      enabled: true
      cert: "/etc/jaeger/certs/server-cert.pem"
      key: "/etc/jaeger/certs/server-key.pem"
      ca: "/etc/jaeger/certs/ca-cert.pem"
      client-ca: "/etc/jaeger/certs/ca-cert.pem"
  http-server:
    host-port: "0.0.0.0:16686"
    tls:
      enabled: true
      cert: "/etc/jaeger/certs/server-cert.pem"
      key: "/etc/jaeger/certs/server-key.pem"
  bearer-token-propagation: true
  additional-headers:
    - "Authorization"
span-storage:
  type: memory

Generate JWT signing key

Create a JWT signing key for authentication token validation.

sudo mkdir -p /etc/jaeger/jwt
sudo openssl genpkey -algorithm RSA -out /etc/jaeger/jwt/jwt-key.pem -pkcs8 -pass pass:jaeger2024
sudo openssl rsa -pubout -in /etc/jaeger/jwt/jwt-key.pem -out /etc/jaeger/jwt/jwt-public.pem -passin pass:jaeger2024
sudo chmod 600 /etc/jaeger/jwt/jwt-key.pem
sudo chmod 644 /etc/jaeger/jwt/jwt-public.pem

Create JWT validation script

Implement a JWT token validation script for authentication middleware.

#!/usr/bin/env python3
import jwt
import sys
import json
from datetime import datetime, timezone

def validate_token(token, public_key_path):
    try:
        with open(public_key_path, 'r') as f:
            public_key = f.read()
        
        payload = jwt.decode(token, public_key, algorithms=["RS256"])
        
        # Check expiration
        if 'exp' in payload:
            exp_time = datetime.fromtimestamp(payload['exp'], tz=timezone.utc)
            if datetime.now(timezone.utc) > exp_time:
                return False, "Token expired"
        
        # Check required claims
        if 'sub' not in payload or 'role' not in payload:
            return False, "Missing required claims"
        
        return True, payload
    except jwt.InvalidTokenError as e:
        return False, str(e)
    except Exception as e:
        return False, f"Validation error: {str(e)}"

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print("Usage: jwt-validator.py  ")
        sys.exit(1)
    
    token = sys.argv[1]
    public_key_path = sys.argv[2]
    
    valid, result = validate_token(token, public_key_path)
    
    if valid:
        print(json.dumps({"valid": True, "payload": result}))
        sys.exit(0)
    else:
        print(json.dumps({"valid": False, "error": result}))
        sys.exit(1)

sudo chmod 755 /etc/jaeger/jwt-validator.py

Configure NGINX reverse proxy with authentication

Set up NGINX as a reverse proxy with JWT authentication for the Jaeger UI.

upstream jaeger_query {
    server 127.0.0.1:16686;
}

server {
    listen 443 ssl http2;
    server_name example.com;
    
    ssl_certificate /etc/jaeger/certs/server-cert.pem;
    ssl_private_key /etc/jaeger/certs/server-key.pem;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers off;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;
    
    # Security headers
    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
    add_header X-Frame-Options DENY always;
    add_header X-Content-Type-Options nosniff always;
    add_header X-XSS-Protection "1; mode=block" always;
    add_header Referrer-Policy "strict-origin-when-cross-origin" always;
    add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self';" always;
    
    location / {
        # JWT Authentication
        access_by_lua_block {
            local token = ngx.var.http_authorization
            if not token then
                ngx.status = 401
                ngx.header["WWW-Authenticate"] = 'Bearer realm="Jaeger"'
                ngx.say('{"error": "Authorization header required"}')
                ngx.exit(401)
            end
            
            token = string.gsub(token, "Bearer ", "")
            
            local handle = io.popen("/etc/jaeger/jwt-validator.py '" .. token .. "' /etc/jaeger/jwt/jwt-public.pem")
            local result = handle:read("*a")
            local exit_code = handle:close()
            
            if not exit_code then
                ngx.status = 401
                ngx.header["WWW-Authenticate"] = 'Bearer realm="Jaeger"'
                ngx.say('{"error": "Invalid token"}')
                ngx.exit(401)
            end
        }
        
        proxy_pass https://jaeger_query;
        proxy_ssl_verify off;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Authorization $http_authorization;
        
        # WebSocket support for live updates
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
    }
    
    location /api {
        # API endpoints with JWT validation
        access_by_lua_block {
            local token = ngx.var.http_authorization
            if not token then
                ngx.status = 401
                ngx.header["WWW-Authenticate"] = 'Bearer realm="Jaeger API"'
                ngx.say('{"error": "Authorization header required"}')
                ngx.exit(401)
            end
            
            token = string.gsub(token, "Bearer ", "")
            
            local handle = io.popen("/etc/jaeger/jwt-validator.py '" .. token .. "' /etc/jaeger/jwt/jwt-public.pem")
            local result = handle:read("*a")
            local exit_code = handle:close()
            
            if not exit_code then
                ngx.status = 401
                ngx.header["WWW-Authenticate"] = 'Bearer realm="Jaeger API"'
                ngx.say('{"error": "Invalid token"}')
                ngx.exit(401)
            end
        }
        
        proxy_pass https://jaeger_query;
        proxy_ssl_verify off;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Authorization $http_authorization;
    }
}

server {
    listen 80;
    server_name example.com;
    return 301 https://$server_name$request_uri;
}

Install NGINX Lua module

Install the Lua module for NGINX to enable JWT authentication processing.

sudo apt install -y nginx-module-lua python3-jwt
sudo ln -sf /etc/nginx/sites-available/jaeger /etc/nginx/sites-enabled/

sudo dnf install -y nginx-mod-http-lua python3-pyjwt
sudo ln -sf /etc/nginx/sites-available/jaeger /etc/nginx/conf.d/jaeger.conf

Configure NGINX to load Lua module

Enable the Lua module in the main NGINX configuration.

# Add this at the top of nginx.conf
load_module modules/ngx_http_lua_module.so;

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;

events {
    worker_connections 1024;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;
    
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';
    
    access_log /var/log/nginx/access.log main;
    
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    server_tokens off;
    
    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

Create systemd services

Create systemd service files for Jaeger Collector and Query components.

[Unit]
Description=Jaeger Collector
After=network.target

[Service]
Type=simple
User=jaeger
Group=jaeger
ExecStart=/opt/jaeger/jaeger-collector --config-file=/etc/jaeger/collector-config.yaml
Restart=always
RestartSec=5
StandardOutput=journal
StandardError=journal
SyslogIdentifier=jaeger-collector

Security settings
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log/jaeger

[Install]
WantedBy=multi-user.target

[Unit]
Description=Jaeger Query
After=network.target
Requires=jaeger-collector.service

[Service]
Type=simple
User=jaeger
Group=jaeger
ExecStart=/opt/jaeger/jaeger-query --config-file=/etc/jaeger/query-config.yaml
Restart=always
RestartSec=5
StandardOutput=journal
StandardError=journal
SyslogIdentifier=jaeger-query

Security settings
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log/jaeger

[Install]
WantedBy=multi-user.target

Create Jaeger user and directories

Create a dedicated user for running Jaeger services and set up log directories.

sudo useradd -r -s /bin/false -d /opt/jaeger jaeger
sudo mkdir -p /var/log/jaeger
sudo chown jaeger:jaeger /var/log/jaeger
sudo chmod 755 /var/log/jaeger

Allow jaeger user to read certificates
sudo chown -R root:jaeger /etc/jaeger/certs
sudo chmod 640 /etc/jaeger/certs/*-key.pem

Create JWT token generation script

Create a script to generate JWT tokens for authenticated access to Jaeger.

#!/usr/bin/env python3
import jwt
import sys
import argparse
from datetime import datetime, timezone, timedelta

def generate_token(user, role, expires_hours, private_key_path, passphrase):
    try:
        with open(private_key_path, 'r') as f:
            private_key = f.read()
        
        now = datetime.now(timezone.utc)
        payload = {
            'sub': user,
            'role': role,
            'iat': now,
            'exp': now + timedelta(hours=expires_hours),
            'iss': 'jaeger-auth'
        }
        
        token = jwt.encode(payload, private_key, algorithm='RS256', headers={'typ': 'JWT'})
        return token
    except Exception as e:
        print(f"Error generating token: {e}")
        return None

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Generate JWT token for Jaeger access')
    parser.add_argument('--user', required=True, help='Username')
    parser.add_argument('--role', choices=['admin', 'readonly'], default='readonly', help='User role')
    parser.add_argument('--expires', type=int, default=24, help='Expiration in hours')
    parser.add_argument('--key', default='/etc/jaeger/jwt/jwt-key.pem', help='Private key path')
    parser.add_argument('--passphrase', default='jaeger2024', help='Key passphrase')
    
    args = parser.parse_args()
    
    token = generate_token(args.user, args.role, args.expires, args.key, args.passphrase)
    
    if token:
        print(f"JWT Token for {args.user} ({args.role}):")
        print(token)
        print(f"\nUsage: curl -H 'Authorization: Bearer {token}' https://example.com/api/traces")
    else:
        sys.exit(1)

sudo chmod 755 /usr/local/bin/generate-jaeger-token

Start and enable services

Start the Jaeger services and NGINX reverse proxy.

sudo systemctl daemon-reload
sudo systemctl enable --now jaeger-collector
sudo systemctl enable --now jaeger-query
sudo systemctl enable --now nginx

Check service status
sudo systemctl status jaeger-collector
sudo systemctl status jaeger-query
sudo systemctl status nginx