Configure Jaeger distributed tracing with multi-datacenter replication for high availability and disaster recovery. Learn to set up primary and secondary datacenters with automated failover and cross-region data synchronization.

Prerequisites

At least 16GB RAM per datacenter
Network connectivity between datacenters
Docker and Docker Compose installed
Basic understanding of Elasticsearch and distributed systems

What this solves

Jaeger multi-datacenter replication provides high availability and disaster recovery for your distributed tracing infrastructure. This setup ensures that trace data remains available even if an entire datacenter fails, preventing loss of critical observability data during outages or disasters.

Step-by-step configuration

Update system packages

Start by updating your package manager on all servers in both datacenters.

sudo apt update && sudo apt upgrade -y
sudo apt install -y wget curl gnupg2 software-properties-common

sudo dnf update -y
sudo dnf install -y wget curl gnupg2 epel-release

Install Docker and Docker Compose

Docker is required for running Jaeger components with consistent configuration across datacenters.

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list
sudo apt update
sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
sudo usermod -aG docker $USER

sudo dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
sudo systemctl enable --now docker
sudo usermod -aG docker $USER

Configure Elasticsearch cluster for primary datacenter

Set up a three-node Elasticsearch cluster in the primary datacenter to store trace data with high availability.

version: '3.8'
services:
  elasticsearch-1:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-1
    environment:
      - node.name=elasticsearch-1
      - cluster.name=jaeger-primary
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-2:9300,elasticsearch-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-1-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - jaeger-network

  elasticsearch-2:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-2
    environment:
      - node.name=elasticsearch-2
      - cluster.name=jaeger-primary
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-1:9300,elasticsearch-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-2-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9201:9200"
      - "9301:9300"
    networks:
      - jaeger-network

  elasticsearch-3:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-3
    environment:
      - node.name=elasticsearch-3
      - cluster.name=jaeger-primary
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-1:9300,elasticsearch-2:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-3-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9202:9200"
      - "9302:9300"
    networks:
      - jaeger-network

volumes:
  elasticsearch-1-data:
  elasticsearch-2-data:
  elasticsearch-3-data:

networks:
  jaeger-network:
    driver: bridge

Generate Elasticsearch security certificates

Create SSL certificates for secure communication between Elasticsearch nodes.

mkdir -p /opt/jaeger/primary-dc/certs
cd /opt/jaeger/primary-dc

Generate CA and node certificates
docker run --rm -v $(pwd)/certs:/certs --user $(id -u):$(id -g) \
  elasticsearch:8.11.0 \
  bin/elasticsearch-certutil ca --out /certs/elastic-ca.p12 --pass ""

docker run --rm -v $(pwd)/certs:/certs --user $(id -u):$(id -g) \
  elasticsearch:8.11.0 \
  bin/elasticsearch-certutil cert --ca /certs/elastic-ca.p12 --ca-pass "" \
  --out /certs/elastic-certificates.p12 --pass ""

sudo chown -R $USER:$USER /opt/jaeger/primary-dc/certs
sudo chmod 755 /opt/jaeger/primary-dc/certs
sudo chmod 644 /opt/jaeger/primary-dc/certs/*

Start primary datacenter Elasticsearch cluster

Launch the Elasticsearch cluster in the primary datacenter and verify cluster health.

cd /opt/jaeger/primary-dc
docker compose up -d

Wait for cluster to be ready
sleep 60

Verify cluster health
curl -u elastic:YourSecurePassword123 "http://localhost:9200/_cluster/health?pretty"
curl -u elastic:YourSecurePassword123 "http://localhost:9200/_nodes?pretty"

Configure Jaeger components for primary datacenter

Add Jaeger collector, query service, and agent to the primary datacenter configuration.

# Add these services to the existing docker-compose.yml file

  jaeger-collector:
    image: jaegertracing/jaeger-collector:1.52
    container_name: jaeger-collector-primary
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=YourSecurePassword123
      - ES_INDEX_PREFIX=jaeger-primary
      - COLLECTOR_ZIPKIN_HOST_PORT=:9411
      - COLLECTOR_GRPC_TLS=false
      - LOG_LEVEL=info
    ports:
      - "9411:9411"
      - "14250:14250"
      - "14268:14268"
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3
    networks:
      - jaeger-network

  jaeger-query:
    image: jaegertracing/jaeger-query:1.52
    container_name: jaeger-query-primary
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=YourSecurePassword123
      - ES_INDEX_PREFIX=jaeger-primary
      - QUERY_BASE_PATH=/jaeger
      - LOG_LEVEL=info
    ports:
      - "16686:16686"
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3
    networks:
      - jaeger-network

  jaeger-agent:
    image: jaegertracing/jaeger-agent:1.52
    container_name: jaeger-agent-primary
    environment:
      - REPORTER_GRPC_HOST_PORT=jaeger-collector:14250
      - LOG_LEVEL=info
    ports:
      - "6831:6831/udp"
      - "6832:6832/udp"
      - "5778:5778"
    depends_on:
      - jaeger-collector
    networks:
      - jaeger-network

Set up secondary datacenter infrastructure

Create the secondary datacenter configuration with cross-cluster replication settings.

mkdir -p /opt/jaeger/secondary-dc/certs
cd /opt/jaeger/secondary-dc

Copy certificates from primary datacenter
scp -r primary-server:/opt/jaeger/primary-dc/certs/* ./certs/

version: '3.8'
services:
  elasticsearch-1:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-secondary-1
    environment:
      - node.name=elasticsearch-secondary-1
      - cluster.name=jaeger-secondary
      - cluster.initial_master_nodes=elasticsearch-secondary-1,elasticsearch-secondary-2,elasticsearch-secondary-3
      - discovery.seed_hosts=elasticsearch-secondary-2:9300,elasticsearch-secondary-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
      - xpack.ccr.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-secondary-1-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - jaeger-network

  elasticsearch-2:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-secondary-2
    environment:
      - node.name=elasticsearch-secondary-2
      - cluster.name=jaeger-secondary
      - cluster.initial_master_nodes=elasticsearch-secondary-1,elasticsearch-secondary-2,elasticsearch-secondary-3
      - discovery.seed_hosts=elasticsearch-secondary-1:9300,elasticsearch-secondary-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
      - xpack.ccr.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-secondary-2-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9201:9200"
      - "9301:9300"
    networks:
      - jaeger-network

  elasticsearch-3:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-secondary-3
    environment:
      - node.name=elasticsearch-secondary-3
      - cluster.name=jaeger-secondary
      - cluster.initial_master_nodes=elasticsearch-secondary-1,elasticsearch-secondary-2,elasticsearch-secondary-3
      - discovery.seed_hosts=elasticsearch-secondary-1:9300,elasticsearch-secondary-2:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=YourSecurePassword123
      - xpack.ccr.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-secondary-3-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9202:9200"
      - "9302:9300"
    networks:
      - jaeger-network

  jaeger-collector:
    image: jaegertracing/jaeger-collector:1.52
    container_name: jaeger-collector-secondary
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=YourSecurePassword123
      - ES_INDEX_PREFIX=jaeger-secondary
      - COLLECTOR_ZIPKIN_HOST_PORT=:9411
      - COLLECTOR_GRPC_TLS=false
      - LOG_LEVEL=info
    ports:
      - "9411:9411"
      - "14250:14250"
      - "14268:14268"
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3
    networks:
      - jaeger-network

  jaeger-query:
    image: jaegertracing/jaeger-query:1.52
    container_name: jaeger-query-secondary
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=YourSecurePassword123
      - ES_INDEX_PREFIX=jaeger-secondary
      - QUERY_BASE_PATH=/jaeger
      - LOG_LEVEL=info
    ports:
      - "16686:16686"
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3
    networks:
      - jaeger-network

volumes:
  elasticsearch-secondary-1-data:
  elasticsearch-secondary-2-data:
  elasticsearch-secondary-3-data:

networks:
  jaeger-network:
    driver: bridge

Configure cross-cluster replication

Set up Elasticsearch cross-cluster replication to sync data from primary to secondary datacenter.

# Start secondary datacenter cluster
cd /opt/jaeger/secondary-dc
docker compose up -d

Wait for secondary cluster to be ready
sleep 60

Configure remote cluster connection on secondary
curl -X PUT -u elastic:YourSecurePassword123 \
  "http://localhost:9200/_cluster/settings" \
  -H "Content-Type: application/json" \
  -d '{
    "persistent": {
      "cluster": {
        "remote": {
          "primary-cluster": {
            "seeds": ["203.0.113.10:9300", "203.0.113.11:9300", "203.0.113.12:9300"]
          }
        }
      }
    }
  }'

Create follower indices for cross-cluster replication
curl -X PUT -u elastic:YourSecurePassword123 \
  "http://localhost:9200/jaeger-secondary-span-*/_ccr/follow" \
  -H "Content-Type: application/json" \
  -d '{
    "remote_cluster": "primary-cluster",
    "leader_index": "jaeger-primary-span-*",
    "settings": {
      "index.number_of_replicas": 1
    }
  }'

curl -X PUT -u elastic:YourSecurePassword123 \
  "http://localhost:9200/jaeger-secondary-service-*/_ccr/follow" \
  -H "Content-Type: application/json" \
  -d '{
    "remote_cluster": "primary-cluster",
    "leader_index": "jaeger-primary-service-*",
    "settings": {
      "index.number_of_replicas": 1
    }
  }'

Set up automated failover with HAProxy

Configure HAProxy to automatically route traffic to the secondary datacenter when the primary fails.

sudo apt install -y haproxy

sudo dnf install -y haproxy

global
    log 127.0.0.1:514 local0
    stats timeout 30s
    daemon
    user haproxy
    group haproxy

defaults
    log global
    mode http
    option httplog
    option dontlognull
    timeout connect 5000
    timeout client 50000
    timeout server 50000
    errorfile 400 /etc/haproxy/errors/400.http
    errorfile 403 /etc/haproxy/errors/403.http
    errorfile 408 /etc/haproxy/errors/408.http
    errorfile 500 /etc/haproxy/errors/500.http
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    errorfile 504 /etc/haproxy/errors/504.http

frontend jaeger_ui_frontend
    bind *:80
    bind *:443 ssl crt /etc/ssl/certs/jaeger.pem
    redirect scheme https if !{ ssl_fc }
    default_backend jaeger_ui_backend

backend jaeger_ui_backend
    balance roundrobin
    option httpchk GET /api/services
    server primary-jaeger 203.0.113.10:16686 check inter 10s fall 3 rise 2
    server secondary-jaeger 203.0.113.20:16686 check inter 10s fall 3 rise 2 backup

frontend jaeger_collector_frontend
    bind *:14268
    default_backend jaeger_collector_backend

backend jaeger_collector_backend
    balance roundrobin
    option httpchk GET /
    server primary-collector 203.0.113.10:14268 check inter 10s fall 3 rise 2
    server secondary-collector 203.0.113.20:14268 check inter 10s fall 3 rise 2 backup

frontend jaeger_grpc_frontend
    bind *:14250
    mode tcp
    default_backend jaeger_grpc_backend

backend jaeger_grpc_backend
    mode tcp
    balance roundrobin
    server primary-grpc 203.0.113.10:14250 check inter 10s fall 3 rise 2
    server secondary-grpc 203.0.113.20:14250 check inter 10s fall 3 rise 2 backup

listen stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s
    stats admin if TRUE

Create monitoring and alerting scripts

Set up automated monitoring to detect failover events and send alerts.

#!/bin/bash

PRIMARY_DC_URL="http://203.0.113.10:16686"
SECONDARY_DC_URL="http://203.0.113.20:16686"
ALERT_EMAIL="admin@example.com"
LOG_FILE="/var/log/jaeger-failover.log"

check_primary() {
    curl -sf "$PRIMARY_DC_URL/api/services" > /dev/null 2>&1
    return $?
}

check_secondary() {
    curl -sf "$SECONDARY_DC_URL/api/services" > /dev/null 2>&1
    return $?
}

send_alert() {
    local message="$1"
    echo "$(date): $message" >> "$LOG_FILE"
    echo "$message" | mail -s "Jaeger Failover Alert" "$ALERT_EMAIL"
}

if ! check_primary; then
    if check_secondary; then
        send_alert "Primary Jaeger datacenter is down. Failed over to secondary datacenter."
        # Update HAProxy configuration to prefer secondary
        sed -i 's/backup$//' /etc/haproxy/haproxy.cfg
        sudo systemctl reload haproxy
    else
        send_alert "CRITICAL: Both Jaeger datacenters are down!"
    fi
else
    # Primary is healthy, ensure proper configuration
    if ! grep -q "backup$" /etc/haproxy/haproxy.cfg; then
        send_alert "Primary Jaeger datacenter recovered. Failing back from secondary."
        sed -i '/secondary.*server/s/$/ backup/' /etc/haproxy/haproxy.cfg
        sudo systemctl reload haproxy
    fi
fi

sudo chmod 755 /opt/jaeger/monitor-failover.sh
sudo chown root:root /opt/jaeger/monitor-failover.sh

Configure automated failover with systemd timer

Set up a systemd timer to run the failover monitoring script every 30 seconds.

[Unit]
Description=Jaeger Failover Monitor
After=network.target

[Service]
Type=oneshot
User=root
ExecStart=/opt/jaeger/monitor-failover.sh
StandardOutput=journal
StandardError=journal

[Unit]
Description=Run Jaeger Failover Monitor every 30 seconds
Requires=jaeger-failover.service

[Timer]
OnBootSec=30
OnUnitActiveSec=30

[Install]
WantedBy=timers.target

sudo systemctl daemon-reload
sudo systemctl enable --now jaeger-failover.timer
sudo systemctl enable --now haproxy

Configure data retention and archival

Set up Index Lifecycle Management (ILM) policies to manage trace data retention across both datacenters.

# Create ILM policy on primary cluster
curl -X PUT -u elastic:YourSecurePassword123 \
  "http://203.0.113.10:9200/_ilm/policy/jaeger-trace-policy" \
  -H "Content-Type: application/json" \
  -d '{
    "policy": {
      "phases": {
        "hot": {
          "actions": {
            "rollover": {
              "max_size": "5gb",
              "max_age": "1d"
            }
          }
        },
        "warm": {
          "min_age": "7d",
          "actions": {
            "allocate": {
              "number_of_replicas": 1
            }
          }
        },
        "cold": {
          "min_age": "30d",
          "actions": {
            "allocate": {
              "number_of_replicas": 0
            }
          }
        },
        "delete": {
          "min_age": "90d"
        }
      }
    }
  }'

Apply policy to index templates
curl -X PUT -u elastic:YourSecurePassword123 \
  "http://203.0.113.10:9200/_index_template/jaeger-spans" \
  -H "Content-Type: application/json" \
  -d '{
    "index_patterns": ["jaeger-primary-span-*"],
    "template": {
      "settings": {
        "index.lifecycle.name": "jaeger-trace-policy",
        "index.lifecycle.rollover_alias": "jaeger-spans-write",
        "number_of_shards": 3,
        "number_of_replicas": 1
      }
    }
  }'

Verify your setup

# Check primary datacenter cluster health
curl -u elastic:YourSecurePassword123 "http://203.0.113.10:9200/_cluster/health?pretty"

Check secondary datacenter cluster health
curl -u elastic:YourSecurePassword123 "http://203.0.113.20:9200/_cluster/health?pretty"

Verify cross-cluster replication status
curl -u elastic:YourSecurePassword123 "http://203.0.113.20:9200/_ccr/stats?pretty"

Check Jaeger UI accessibility through HAProxy
curl -I http://your-haproxy-server/api/services

Test failover monitoring script
sudo /opt/jaeger/monitor-failover.sh

Check systemd timer status
sudo systemctl status jaeger-failover.timer

View HAProxy stats
curl http://your-haproxy-server:8404/stats

Common issues

Symptom	Cause	Fix
Cross-cluster replication failing	Network connectivity issues between datacenters	Check firewall rules and network connectivity between Elasticsearch clusters
HAProxy shows backend servers as down	Health check URL incorrect or services not responding	Verify health check URLs and ensure Jaeger services are running
High memory usage in Elasticsearch	JVM heap size not optimized for available memory	Adjust ES_JAVA_OPTS to use 50% of available RAM
Trace data not replicating	Index patterns don't match in CCR configuration	Ensure follower index patterns match leader index naming
Failover script not executing	Permissions issue or mail not configured	Check script permissions and install mail utilities
SSL certificate errors between clusters	Certificate mismatch or expired certificates	Regenerate certificates and ensure they're copied to all nodes

Next steps

Automated install script

Run this to automate the entire setup

install.sh

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Default values
DATACENTER_TYPE="primary"
ES_PASSWORD="YourSecurePassword123"
CLUSTER_NAME="jaeger-cluster"

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -t, --type TYPE        Datacenter type (primary|secondary) [default: primary]"
    echo "  -p, --password PASS    Elasticsearch password [default: YourSecurePassword123]"
    echo "  -c, --cluster NAME     Cluster name [default: jaeger-cluster]"
    echo "  -h, --help            Show this help message"
    exit 1
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -t|--type)
            DATACENTER_TYPE="$2"
            shift 2
            ;;
        -p|--password)
            ES_PASSWORD="$2"
            shift 2
            ;;
        -c|--cluster)
            CLUSTER_NAME="$2"
            shift 2
            ;;
        -h|--help)
            usage
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Validate datacenter type
if [[ "$DATACENTER_TYPE" != "primary" && "$DATACENTER_TYPE" != "secondary" ]]; then
    echo -e "${RED}Error: Datacenter type must be 'primary' or 'secondary'${NC}"
    exit 1
fi

# Cleanup function
cleanup() {
    echo -e "${RED}Installation failed. Cleaning up...${NC}"
    docker-compose down 2>/dev/null || true
    docker system prune -f 2>/dev/null || true
}

trap cleanup ERR

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root or with sudo${NC}"
    exit 1
fi

# Detect OS distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update && apt upgrade -y"
            FIREWALL_CMD="ufw"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y"
            FIREWALL_CMD="firewall-cmd"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            FIREWALL_CMD="firewall-cmd"
            ;;
        *) 
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}Cannot detect OS distribution${NC}"
    exit 1
fi

echo -e "${GREEN}Setting up Jaeger multi-datacenter replication ($DATACENTER_TYPE datacenter)${NC}"

# Step 1: Update system packages
echo -e "${YELLOW}[1/7] Updating system packages...${NC}"
$PKG_UPDATE

if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL wget curl gnupg2 software-properties-common
else
    $PKG_INSTALL wget curl gnupg2 epel-release
fi

# Step 2: Install Docker
echo -e "${YELLOW}[2/7] Installing Docker...${NC}"
if ! command -v docker &> /dev/null; then
    if [[ "$PKG_MGR" == "apt" ]]; then
        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
        echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list
        apt update
        $PKG_INSTALL docker-ce docker-ce-cli containerd.io docker-compose-plugin
    else
        $PKG_INSTALL dnf-plugins-core
        dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
        $PKG_INSTALL docker-ce docker-ce-cli containerd.io docker-compose-plugin
        systemctl enable --now docker
    fi
    
    # Add current user to docker group
    if [[ -n "${SUDO_USER:-}" ]]; then
        usermod -aG docker "$SUDO_USER"
    fi
else
    echo -e "${GREEN}Docker already installed${NC}"
fi

# Step 3: Configure firewall
echo -e "${YELLOW}[3/7] Configuring firewall...${NC}"
if [[ "$FIREWALL_CMD" == "ufw" ]]; then
    if command -v ufw &> /dev/null; then
        ufw allow 9200/tcp
        ufw allow 9300/tcp
        ufw allow 14268/tcp
        ufw allow 16686/tcp
        ufw --force enable
    fi
else
    if command -v firewall-cmd &> /dev/null; then
        firewall-cmd --permanent --add-port=9200/tcp
        firewall-cmd --permanent --add-port=9300/tcp
        firewall-cmd --permanent --add-port=14268/tcp
        firewall-cmd --permanent --add-port=16686/tcp
        firewall-cmd --reload
    fi
fi

# Step 4: Create Jaeger directory structure
echo -e "${YELLOW}[4/7] Creating directory structure...${NC}"
mkdir -p /opt/jaeger/{config,certs,data}
chown -R root:root /opt/jaeger
chmod 755 /opt/jaeger /opt/jaeger/config /opt/jaeger/certs
chmod 750 /opt/jaeger/data

# Step 5: Generate SSL certificates for Elasticsearch
echo -e "${YELLOW}[5/7] Generating SSL certificates...${NC}"
cd /opt/jaeger

# Create certificates using Docker (temporary container)
docker run --rm -v /opt/jaeger/certs:/certs \
    docker.elastic.co/elasticsearch/elasticsearch:8.11.0 \
    bin/elasticsearch-certutil cert --silent --pem --in /dev/null --out /certs/bundle.zip

cd /opt/jaeger/certs
if [[ -f bundle.zip ]]; then
    unzip -q bundle.zip
    mv ca/ca.crt .
    mv instance/instance.crt elastic-certificates.crt
    mv instance/instance.key elastic-certificates.key
    rm -rf ca instance bundle.zip
    
    # Create p12 keystore
    openssl pkcs12 -export -out elastic-certificates.p12 \
        -in elastic-certificates.crt -inkey elastic-certificates.key \
        -passin pass: -passout pass:
fi

chown -R root:root /opt/jaeger/certs
chmod 644 /opt/jaeger/certs/*

# Step 6: Create Docker Compose configuration
echo -e "${YELLOW}[6/7] Creating Docker Compose configuration...${NC}"
cat > /opt/jaeger/docker-compose.yml << EOF
version: '3.8'
services:
  elasticsearch-1:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-1
    environment:
      - node.name=elasticsearch-1
      - cluster.name=${CLUSTER_NAME}-${DATACENTER_TYPE}
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-2:9300,elasticsearch-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=${ES_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-1-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - jaeger-network

  elasticsearch-2:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-2
    environment:
      - node.name=elasticsearch-2
      - cluster.name=${CLUSTER_NAME}-${DATACENTER_TYPE}
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-1:9300,elasticsearch-3:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=${ES_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-2-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9201:9200"
      - "9301:9300"
    networks:
      - jaeger-network

  elasticsearch-3:
    image: elasticsearch:8.11.0
    container_name: elasticsearch-3
    environment:
      - node.name=elasticsearch-3
      - cluster.name=${CLUSTER_NAME}-${DATACENTER_TYPE}
      - cluster.initial_master_nodes=elasticsearch-1,elasticsearch-2,elasticsearch-3
      - discovery.seed_hosts=elasticsearch-1:9300,elasticsearch-2:9300
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
      - ELASTIC_PASSWORD=${ES_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-3-data:/usr/share/elasticsearch/data
      - ./certs:/usr/share/elasticsearch/config
    ports:
      - "9202:9200"
      - "9302:9300"
    networks:
      - jaeger-network

  jaeger-collector:
    image: jaegertracing/jaeger-collector:1.50
    container_name: jaeger-collector
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=${ES_PASSWORD}
    ports:
      - "14268:14268"
      - "14269:14269"
    networks:
      - jaeger-network
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3

  jaeger-query:
    image: jaegertracing/jaeger-query:1.50
    container_name: jaeger-query
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch-1:9200,http://elasticsearch-2:9200,http://elasticsearch-3:9200
      - ES_USERNAME=elastic
      - ES_PASSWORD=${ES_PASSWORD}
    ports:
      - "16686:16686"
    networks:
      - jaeger-network
    depends_on:
      - elasticsearch-1
      - elasticsearch-2
      - elasticsearch-3

volumes:
  elasticsearch-1-data:
  elasticsearch-2-data:
  elasticsearch-3-data:

networks:
  jaeger-network:
    driver: bridge
EOF

chmod 644 /opt/jaeger/docker-compose.yml

# Step 7: Start services
echo -e "${YELLOW}[7/7] Starting Jaeger services...${NC}"
cd /opt/jaeger
docker compose up -d

# Wait for services to be ready
echo -e "${YELLOW}Waiting for services to be ready...${NC}"
sleep 30

# Verification
echo -e "${YELLOW}Verifying installation...${NC}"
if curl -s -u "elastic:${ES_PASSWORD}" http://localhost:9200/_cluster/health | grep -q "green\|yellow"; then
    echo -e "${GREEN}✓ Elasticsearch cluster is running${NC}"
else
    echo -e "${RED}✗ Elasticsearch cluster health check failed${NC}"
fi

if curl -s http://localhost:16686 | grep -q "Jaeger"; then
    echo -e "${GREEN}✓ Jaeger UI is accessible${NC}"
else
    echo -e "${RED}✗ Jaeger UI is not responding${NC}"
fi

echo -e "${GREEN}Jaeger multi-datacenter setup complete!${NC}"
echo -e "${GREEN}Jaeger UI: http://$(hostname -I | awk '{print $1}'):16686${NC}"
echo -e "${GREEN}Elasticsearch: http://$(hostname -I | awk '{print $1}'):9200${NC}"
echo -e "${YELLOW}Username: elastic${NC}"
echo -e "${YELLOW}Password: ${ES_PASSWORD}${NC}"

Review the script before running. Execute with: bash install.sh

#jaeger #distributed-tracing #disaster-recovery #elasticsearch #high-availability

Set up Jaeger multi-datacenter replication for disaster recovery and high availability

Prerequisites

What this solves

Step-by-step configuration

Update system packages

Install Docker and Docker Compose

Configure Elasticsearch cluster for primary datacenter

Generate Elasticsearch security certificates

Generate CA and node certificates

Start primary datacenter Elasticsearch cluster

Wait for cluster to be ready

Verify cluster health

Configure Jaeger components for primary datacenter

Set up secondary datacenter infrastructure

Copy certificates from primary datacenter

Configure cross-cluster replication

Wait for secondary cluster to be ready

Configure remote cluster connection on secondary

Create follower indices for cross-cluster replication

Set up automated failover with HAProxy

Create monitoring and alerting scripts

Configure automated failover with systemd timer

Configure data retention and archival

Apply policy to index templates

Verify your setup

Check secondary datacenter cluster health

Verify cross-cluster replication status

Check Jaeger UI accessibility through HAProxy

Test failover monitoring script

Check systemd timer status

View HAProxy stats

Common issues

Next steps

Related tutorials

Monitor Kubernetes clusters with Prometheus and Grafana for container orchestration insights

Configure Nagios SNMP monitoring for network devices with automated discovery and templates

Integrate Elasticsearch 8 with Prometheus monitoring and Grafana dashboards

Don't want to manage this yourself?