Jaeger Multi-Datacenter Replication Setup

Configure Jaeger distributed tracing with multi-datacenter replication for high availability and disaster recovery. Learn to set up primary and secondary datacenters with automated failover and cross-region data synchronization.

Prerequisites

At least 16GB RAM per datacenter
Network connectivity between datacenters
Docker and Docker Compose installed
Basic understanding of Elasticsearch and distributed systems

What this solves

Jaeger multi-datacenter replication provides high availability and disaster recovery for your distributed tracing infrastructure. This setup ensures that trace data remains available even if an entire datacenter fails, preventing loss of critical observability data during outages or disasters.

Step-by-step configuration

Update system packages

Start by updating your package manager on all servers in both datacenters.

sudo apt update && sudo apt upgrade -y
sudo apt install -y wget curl gnupg2 software-properties-common

sudo dnf update -y
sudo dnf install -y wget curl gnupg2 epel-release

Install Docker and Docker Compose

Docker is required for running Jaeger components with consistent configuration across datacenters.

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list
sudo apt update
sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
sudo usermod -aG docker $USER

sudo dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
sudo systemctl enable --now docker
sudo usermod -aG docker $USER

Configure Elasticsearch cluster for primary datacenter

Set up a three-node Elasticsearch cluster in the primary datacenter to store trace data with high availability.

version: '3.8'
services:
  elasticsearch-1:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-1
    environment:
      - node.name=elasticsearch-1
      - cluster.name=jaeger-primary
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-2:9300,elasticsearch-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-1-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - jaeger-network

  elasticsearch-2:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-2
    environment:
      - node.name=elasticsearch-2
      - cluster.name=jaeger-primary
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-1:9300,elasticsearch-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-2-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9201:9200"
      - "9301:9300"
    networks:
      - jaeger-network

  elasticsearch-3:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-3
    environment:
      - node.name=elasticsearch-3
      - cluster.name=jaeger-primary
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-1:9300,elasticsearch-2:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-3-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9202:9200"
      - "9302:9300"
    networks:
      - jaeger-network

volumes:
  elasticsearch-1-data:
  elasticsearch-2-data:
  elasticsearch-3-data:

networks:
  jaeger-network:
    driver: bridge

Generate Elasticsearch security certificates

Create SSL certificates for secure communication between Elasticsearch nodes.

mkdir -p /opt/jaeger/primary-dc/certs
cd /opt/jaeger/primary-dc

# Generate CA and node certificates
docker run --rm -v $(pwd)/certs:/certs --user $(id -u):$(id -g) \
  elasticsearch:8.11.0 \
  bin/elasticsearch-certutil ca --out /certs/elastic-ca.p12 --pass ""

docker run --rm -v $(pwd)/certs:/certs --user $(id -u):$(id -g) \
  elasticsearch:8.11.0 \
  bin/elasticsearch-certutil cert --ca /certs/elastic-ca.p12 --ca-pass "" \
  --out /certs/elastic-certificates.p12 --pass ""

sudo chown -R $USER:$USER /opt/jaeger/primary-dc/certs
sudo chmod 755 /opt/jaeger/primary-dc/certs
sudo chmod 644 /opt/jaeger/primary-dc/certs/*

Start primary datacenter Elasticsearch cluster

Launch the Elasticsearch cluster in the primary datacenter and verify cluster health.

cd /opt/jaeger/primary-dc
docker compose up -d

# Wait for cluster to be ready
sleep 60

# Verify cluster health
curl -u elastic:YourSecurePassword123 "http://localhost:9200/_cluster/health?pretty"
curl -u elastic:YourSecurePassword123 "http://localhost:9200/_nodes?pretty"

Configure Jaeger components for primary datacenter

Add Jaeger collector, query service, and agent to the primary datacenter configuration.

# Add these services to the existing docker-compose.yml file

  jaeger-collector:
    image: jaegertracing/jaeger-collector:1.52
    container_name: jaeger-collector-primary
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=YourSecurePassword123
      - ES_INDEX_PREFIX=jaeger-primary
      - COLLECTOR_ZIPKIN_HOST_PORT=:9411
      - COLLECTOR_GRPC_TLS=false
      - LOG_LEVEL=info
    ports:
      - "9411:9411"
      - "14250:14250"
      - "14268:14268"
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3
    networks:
      - jaeger-network

  jaeger-query:
    image: jaegertracing/jaeger-query:1.52
    container_name: jaeger-query-primary
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=YourSecurePassword123
      - ES_INDEX_PREFIX=jaeger-primary
      - QUERY_BASE_PATH=/jaeger
      - LOG_LEVEL=info
    ports:
      - "16686:16686"
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3
    networks:
      - jaeger-network

  jaeger-agent:
    image: jaegertracing/jaeger-agent:1.52
    container_name: jaeger-agent-primary
    environment:
      - REPORTER_GRPC_HOST_PORT=jaeger-collector:14250
      - LOG_LEVEL=info
    ports:
      - "6831:6831/udp"
      - "6832:6832/udp"
      - "5778:5778"
    depends_on:
      - jaeger-collector
    networks:
      - jaeger-network

Set up secondary datacenter infrastructure

Create the secondary datacenter configuration with cross-cluster replication settings.

mkdir -p /opt/jaeger/secondary-dc/certs
cd /opt/jaeger/secondary-dc

# Copy certificates from primary datacenter
scp -r primary-server:/opt/jaeger/primary-dc/certs/* ./certs/

version: '3.8'
services:
  elasticsearch-1:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-secondary-1
    environment:
      - node.name=elasticsearch-secondary-1
      - cluster.name=jaeger-secondary
      - cluster.initial_master_nodes=elasticsearch-secondary-1,elasticsearch-secondary-2,elasticsearch-secondary-3
      - discovery.seed_hosts=elasticsearch-secondary-2:9300,elasticsearch-secondary-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
      - xpack.ccr.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-secondary-1-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - jaeger-network

  elasticsearch-2:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-secondary-2
    environment:
      - node.name=elasticsearch-secondary-2
      - cluster.name=jaeger-secondary
      - cluster.initial_master_nodes=elasticsearch-secondary-1,elasticsearch-secondary-2,elasticsearch-secondary-3
      - discovery.seed_hosts=elasticsearch-secondary-1:9300,elasticsearch-secondary-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
      - xpack.ccr.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-secondary-2-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9201:9200"
      - "9301:9300"
    networks:
      - jaeger-network

  elasticsearch-3:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-secondary-3
    environment:
      - node.name=elasticsearch-secondary-3
      - cluster.name=jaeger-secondary
      - cluster.initial_master_nodes=elasticsearch-secondary-1,elasticsearch-secondary-2,elasticsearch-secondary-3
      - discovery.seed_hosts=elasticsearch-secondary-1:9300,elasticsearch-secondary-2:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
      - xpack.ccr.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-secondary-3-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9202:9200"
      - "9302:9300"
    networks:
      - jaeger-network

  jaeger-collector:
    image: jaegertracing/jaeger-collector:1.52
    container_name: jaeger-collector-secondary
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=YourSecurePassword123
      - ES_INDEX_PREFIX=jaeger-secondary
      - COLLECTOR_ZIPKIN_HOST_PORT=:9411
      - COLLECTOR_GRPC_TLS=false
      - LOG_LEVEL=info
    ports:
      - "9411:9411"
      - "14250:14250"
      - "14268:14268"
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3
    networks:
      - jaeger-network

  jaeger-query:
    image: jaegertracing/jaeger-query:1.52
    container_name: jaeger-query-secondary
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=YourSecurePassword123
      - ES_INDEX_PREFIX=jaeger-secondary
      - QUERY_BASE_PATH=/jaeger
      - LOG_LEVEL=info
    ports:
      - "16686:16686"
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3
    networks:
      - jaeger-network

volumes:
  elasticsearch-secondary-1-data:
  elasticsearch-secondary-2-data:
  elasticsearch-secondary-3-data:

networks:
  jaeger-network:
    driver: bridge

Configure cross-cluster replication

Set up Elasticsearch cross-cluster replication to sync data from primary to secondary datacenter.

# Start secondary datacenter cluster
cd /opt/jaeger/secondary-dc
docker compose up -d

# Wait for secondary cluster to be ready
sleep 60

# Configure remote cluster connection on secondary
curl -X PUT -u elastic:YourSecurePassword123 \
  "http://localhost:9200/_cluster/settings" \
  -H "Content-Type: application/json" \
  -d '{
    "persistent": {
      "cluster": {
        "remote": {
          "primary-cluster": {
            "seeds": ["203.0.113.10:9300", "203.0.113.11:9300", "203.0.113.12:9300"]
          }
        }
      }
    }
  }'

# Create follower indices for cross-cluster replication
curl -X PUT -u elastic:YourSecurePassword123 \
  "http://localhost:9200/jaeger-secondary-span-*/_ccr/follow" \
  -H "Content-Type: application/json" \
  -d '{
    "remote_cluster": "primary-cluster",
    "leader_index": "jaeger-primary-span-*",
    "settings": {
      "index.number_of_replicas": 1
    }
  }'

curl -X PUT -u elastic:YourSecurePassword123 \
  "http://localhost:9200/jaeger-secondary-service-*/_ccr/follow" \
  -H "Content-Type: application/json" \
  -d '{
    "remote_cluster": "primary-cluster",
    "leader_index": "jaeger-primary-service-*",
    "settings": {
      "index.number_of_replicas": 1
    }
  }'

Set up automated failover with HAProxy

Configure HAProxy to automatically route traffic to the secondary datacenter when the primary fails.

sudo apt install -y haproxy

sudo dnf install -y haproxy

global
    log 127.0.0.1:514 local0
    stats timeout 30s
    daemon
    user haproxy
    group haproxy

defaults
    log global
    mode http
    option httplog
    option dontlognull
    timeout connect 5000
    timeout client 50000
    timeout server 50000
    errorfile 400 /etc/haproxy/errors/400.http
    errorfile 403 /etc/haproxy/errors/403.http
    errorfile 408 /etc/haproxy/errors/408.http
    errorfile 500 /etc/haproxy/errors/500.http
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    errorfile 504 /etc/haproxy/errors/504.http

frontend jaeger_ui_frontend
    bind *:80
    bind *:443 ssl crt /etc/ssl/certs/jaeger.pem
    redirect scheme https if !{ ssl_fc }
    default_backend jaeger_ui_backend

backend jaeger_ui_backend
    balance roundrobin
    option httpchk GET /api/services
    server primary-jaeger 203.0.113.10:16686 check inter 10s fall 3 rise 2
    server secondary-jaeger 203.0.113.20:16686 check inter 10s fall 3 rise 2 backup

frontend jaeger_collector_frontend
    bind *:14268
    default_backend jaeger_collector_backend

backend jaeger_collector_backend
    balance roundrobin
    option httpchk GET /
    server primary-collector 203.0.113.10:14268 check inter 10s fall 3 rise 2
    server secondary-collector 203.0.113.20:14268 check inter 10s fall 3 rise 2 backup

frontend jaeger_grpc_frontend
    bind *:14250
    mode tcp
    default_backend jaeger_grpc_backend

backend jaeger_grpc_backend
    mode tcp
    balance roundrobin
    server primary-grpc 203.0.113.10:14250 check inter 10s fall 3 rise 2
    server secondary-grpc 203.0.113.20:14250 check inter 10s fall 3 rise 2 backup

listen stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s
    stats admin if TRUE

Create monitoring and alerting scripts

Set up automated monitoring to detect failover events and send alerts.

#!/bin/bash

PRIMARY_DC_URL="http://203.0.113.10:16686"
SECONDARY_DC_URL="http://203.0.113.20:16686"
ALERT_EMAIL="admin@example.com"
LOG_FILE="/var/log/jaeger-failover.log"

check_primary() {
    curl -sf "$PRIMARY_DC_URL/api/services" > /dev/null 2>&1
    return $?
}

check_secondary() {
    curl -sf "$SECONDARY_DC_URL/api/services" > /dev/null 2>&1
    return $?
}

send_alert() {
    local

Automated install script

Run this to automate the entire setup

install.sh

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Default values
DATACENTER_TYPE="primary"
ES_PASSWORD="YourSecurePassword123"
CLUSTER_NAME="jaeger-cluster"

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -t, --type TYPE        Datacenter type (primary|secondary) [default: primary]"
    echo "  -p, --password PASS    Elasticsearch password [default: YourSecurePassword123]"
    echo "  -c, --cluster NAME     Cluster name [default: jaeger-cluster]"
    echo "  -h, --help            Show this help message"
    exit 1
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -t|--type)
            DATACENTER_TYPE="$2"
            shift 2
            ;;
        -p|--password)
            ES_PASSWORD="$2"
            shift 2
            ;;
        -c|--cluster)
            CLUSTER_NAME="$2"
            shift 2
            ;;
        -h|--help)
            usage
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Validate datacenter type
if [[ "$DATACENTER_TYPE" != "primary" && "$DATACENTER_TYPE" != "secondary" ]]; then
    echo -e "${RED}Error: Datacenter type must be 'primary' or 'secondary'${NC}"
    exit 1
fi

# Cleanup function
cleanup() {
    echo -e "${RED}Installation failed. Cleaning up...${NC}"
    docker-compose down 2>/dev/null || true
    docker system prune -f 2>/dev/null || true
}

trap cleanup ERR

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root or with sudo${NC}"
    exit 1
fi

# Detect OS distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update && apt upgrade -y"
            FIREWALL_CMD="ufw"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            FIREWALL_CMD="firewall-cmd"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            FIREWALL_CMD="firewall-cmd"
            ;;
        *) 
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}Cannot detect OS distribution${NC}"
    exit 1
fi

echo -e "${GREEN}Setting up Jaeger multi-datacenter replication ($DATACENTER_TYPE datacenter)${NC}"

# Step 1: Update system packages
echo -e "${YELLOW}[1/7] Updating system packages...${NC}"
$PKG_UPDATE

if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL wget curl gnupg2 software-properties-common
else
    $PKG_INSTALL wget curl gnupg2 epel-release
fi

# Step 2: Install Docker
echo -e "${YELLOW}[2/7] Installing Docker...${NC}"
if ! command -v docker &> /dev/null; then
    if [[ "$PKG_MGR" == "apt" ]]; then
        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
        echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list
        apt update
        $PKG_INSTALL docker-ce docker-ce-cli containerd.io docker-compose-plugin
    else
        $PKG_INSTALL dnf-plugins-core
        dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
        $PKG_INSTALL docker-ce docker-ce-cli containerd.io docker-compose-plugin
        systemctl enable --now docker
    fi
    
    # Add current user to docker group
    if [[ -n "${SUDO_USER:-}" ]]; then
        usermod -aG docker "$SUDO_USER"
    fi
else
    echo -e "${GREEN}Docker already installed${NC}"
fi

# Step 3: Configure firewall
echo -e "${YELLOW}[3/7] Configuring firewall...${NC}"
if [[ "$FIREWALL_CMD" == "ufw" ]]; then
    if command -v ufw &> /dev/null; then
        ufw allow 9200/tcp
        ufw allow 9300/tcp
        ufw allow 14268/tcp
        ufw allow 16686/tcp
        ufw --force enable
    fi
else
    if command -v firewall-cmd &> /dev/null; then
        firewall-cmd --permanent --add-port=9200/tcp
        firewall-cmd --permanent --add-port=9300/tcp
        firewall-cmd --permanent --add-port=14268/tcp
        firewall-cmd --permanent --add-port=16686/tcp
        firewall-cmd --reload
    fi
fi

# Step 4: Create Jaeger directory structure
echo -e "${YELLOW}[4/7] Creating directory structure...${NC}"
mkdir -p /opt/jaeger/{config,certs,data}
chown -R root:root /opt/jaeger
chmod 755 /opt/jaeger /opt/jaeger/config /opt/jaeger/certs
chmod 750 /opt/jaeger/data

# Step 5: Generate SSL certificates for Elasticsearch
echo -e "${YELLOW}[5/7] Generating SSL certificates...${NC}"
cd /opt/jaeger

# Create certificates using Docker (temporary container)
docker run --rm -v /opt/jaeger/certs:/certs \
    docker.elastic.co/elasticsearch/elasticsearch:8.11.0 \
    bin/elasticsearch-certutil cert --silent --pem --in /dev/null --out /certs/bundle.zip

cd /opt/jaeger/certs
if [[ -f bundle.zip ]]; then
    unzip -q bundle.zip
    mv ca/ca.crt .
    mv instance/instance.crt elastic-certificates.crt
    mv instance/instance.key elastic-certificates.key
    rm -rf ca instance bundle.zip
    
    # Create p12 keystore
    openssl pkcs12 -export -out elastic-certificates.p12 \
        -in elastic-certificates.crt -inkey elastic-certificates.key \
        -passin pass: -passout pass:
fi

chown -R root:root /opt/jaeger/certs
chmod 644 /opt/jaeger/certs/*

# Step 6: Create Docker Compose configuration
echo -e "${YELLOW}[6/7] Creating Docker Compose configuration...${NC}"
cat > /opt/jaeger/docker-compose.yml << EOF
version: '3.8'
services:
  elasticsearch-1:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-1
    environment:
      - node.name=elasticsearch-1
      - cluster.name=${CLUSTER_NAME}-${DATACENTER_TYPE}
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-2:9300,elasticsearch-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=${ES_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-1-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - jaeger-network

  elasticsearch-2:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-2
    environment:
      - node.name=elasticsearch-2
      - cluster.name=${CLUSTER_NAME}-${DATACENTER_TYPE}
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-1:9300,elasticsearch-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=${ES_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-2-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9201:9200"
      - "9301:9300"
    networks:
      - jaeger-network

  elasticsearch-3:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-3
    environment:
      - node.name=elasticsearch-3
      - cluster.name=${CLUSTER_NAME}-${DATACENTER_TYPE}
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-1:9300,elasticsearch-2:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=${ES_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-3-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9202:9200"
      - "9302:9300"
    networks:
      - jaeger-network

  jaeger-collector:
    image: jaegertracing/jaeger-collector:1.50
    container_name: jaeger-collector
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=${ES_PASSWORD}
    ports:
      - "14268:14268"
      - "14269:14269"
    networks:
      - jaeger-network
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3

  jaeger-query:
    image: jaegertracing/jaeger-query:1.50
    container_name: jaeger-query
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=${ES_PASSWORD}
    ports:
      - "16686:16686"
    networks:
      - jaeger-network
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3

volumes:
  elasticsearch-1-data:
  elasticsearch-2-data:
  elasticsearch-3-data:

networks:
  jaeger-network:
    driver: bridge
EOF

chmod 644 /opt/jaeger/docker-compose.yml

# Step 7: Start services
echo -e "${YELLOW}[7/7] Starting Jaeger services...${NC}"
cd /opt/jaeger
docker compose up -d

# Wait for services to be ready
echo -e "${YELLOW}Waiting for services to be ready...${NC}"
sleep 30

# Verification
echo -e "${YELLOW}Verifying installation...${NC}"
if curl -s -u "elastic:${ES_PASSWORD}" http://localhost:9200/_cluster/health | grep -q "green\|yellow"; then
    echo -e "${GREEN}✓ Elasticsearch cluster is running${NC}"
else
    echo -e "${RED}✗ Elasticsearch cluster health check failed${NC}"
fi

if curl -s http://localhost:16686 | grep -q "Jaeger"; then
    echo -e "${GREEN}✓ Jaeger UI is accessible${NC}"
else
    echo -e "${RED}✗ Jaeger UI is not responding${NC}"
fi

echo -e "${GREEN}Jaeger multi-datacenter setup complete!${NC}"
echo -e "${GREEN}Jaeger UI: http://$(hostname -I | awk '{print $1}'):16686${NC}"
echo -e "${GREEN}Elasticsearch: http://$(hostname -I | awk '{print $1}'):9200${NC}"
echo -e "${YELLOW}Username: elastic${NC}"
echo -e "${YELLOW}Password: ${ES_PASSWORD}${NC}"

Review the script before running. Execute with: bash install.sh

#jaeger #distributed-tracing #disaster-recovery #elasticsearch #high-availability

Set up Jaeger multi-datacenter replication for disaster recovery and high availability