Implement Consul multi-datacenter replication with WAN federation

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Usage function
usage() {
    echo "Usage: $0 <datacenter_name> <node_ip> <node_name> [primary_dc_ip]"
    echo "  datacenter_name: Name of the datacenter (e.g., dc1, dc2)"
    echo "  node_ip: IP address of this Consul node"
    echo "  node_name: Name of this Consul node"
    echo "  primary_dc_ip: IP of primary datacenter (required for secondary DCs)"
    echo ""
    echo "Example:"
    echo "  $0 dc1 203.0.113.10 consul-dc1-01"
    echo "  $0 dc2 203.0.113.20 consul-dc2-01 203.0.113.10"
    exit 1
}

# Check arguments
if [ $# -lt 3 ] || [ $# -gt 4 ]; then
    usage
fi

DATACENTER="$1"
NODE_IP="$2"
NODE_NAME="$3"
PRIMARY_DC_IP="${4:-}"

# Check if running as root
if [ "$EUID" -ne 0 ]; then
    echo -e "${RED}Error: This script must be run as root${NC}"
    exit 1
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            FIREWALL_CMD="ufw"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_UPDATE="dnf check-update || true"
            PKG_INSTALL="dnf install -y"
            FIREWALL_CMD="firewall-cmd"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_UPDATE="yum check-update || true"
            PKG_INSTALL="yum install -y"
            FIREWALL_CMD="firewall-cmd"
            ;;
        *) 
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}Cannot detect distribution${NC}"
    exit 1
fi

# Cleanup function
cleanup() {
    echo -e "${RED}Installation failed. Cleaning up...${NC}"
    systemctl stop consul 2>/dev/null || true
    systemctl disable consul 2>/dev/null || true
    userdel consul 2>/dev/null || true
    rm -rf /opt/consul /etc/consul.d /var/lib/consul
    echo -e "${YELLOW}Cleanup completed${NC}"
}

trap cleanup ERR

echo -e "${GREEN}Starting Consul multi-datacenter installation for $DATACENTER${NC}"

# Step 1: Install prerequisites
echo -e "${GREEN}[1/10] Installing prerequisites...${NC}"
$PKG_UPDATE
if [ "$PKG_MGR" = "apt" ]; then
    $PKG_INSTALL curl wget gnupg lsb-release
    curl -fsSL https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
    echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/hashicorp.list
    apt update
else
    $PKG_INSTALL dnf-plugins-core curl wget
    if [ "$PKG_MGR" = "dnf" ]; then
        dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
    else
        yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
    fi
fi

# Step 2: Install Consul
echo -e "${GREEN}[2/10] Installing Consul...${NC}"
$PKG_INSTALL consul

# Step 3: Create Consul user and directories
echo -e "${GREEN}[3/10] Creating Consul user and directories...${NC}"
if ! id consul &>/dev/null; then
    useradd --system --home /etc/consul --shell /bin/false consul
fi
mkdir -p /opt/consul /etc/consul.d /var/lib/consul
chown -R consul:consul /opt/consul /etc/consul.d /var/lib/consul
chmod 755 /opt/consul /etc/consul.d /var/lib/consul

# Step 4: Generate encryption key
echo -e "${GREEN}[4/10] Generating encryption key...${NC}"
GOSSIP_KEY=$(consul keygen)
echo -e "${YELLOW}Save this gossip encryption key: $GOSSIP_KEY${NC}"

# Step 5: Generate TLS certificates
echo -e "${GREEN}[5/10] Generating TLS certificates...${NC}"
cd /etc/consul.d
consul tls ca create
consul tls cert create -server -dc "$DATACENTER"
chown consul:consul /etc/consul.d/*.pem
chmod 600 /etc/consul.d/*-key.pem
chmod 644 /etc/consul.d/*.pem

# Step 6: Generate bootstrap token
echo -e "${GREEN}[6/10] Generating bootstrap token...${NC}"
BOOTSTRAP_TOKEN=$(openssl rand -hex 16)
echo -e "${YELLOW}Save this bootstrap token: $BOOTSTRAP_TOKEN${NC}"

# Step 7: Create Consul configuration
echo -e "${GREEN}[7/10] Creating Consul configuration...${NC}"
cat > /etc/consul.d/consul.hcl << EOF
datacenter = "$DATACENTER"
data_dir = "/var/lib/consul"
log_level = "INFO"
node_name = "$NODE_NAME"
bind_addr = "$NODE_IP"
client_addr = "0.0.0.0"

server = true
bootstrap_expect = 1

ui_config {
  enabled = true
}

connect {
  enabled = true
}

encrypt = "$GOSSIP_KEY"

tls {
  defaults {
    ca_file = "/etc/consul.d/consul-agent-ca.pem"
    cert_file = "/etc/consul.d/$DATACENTER-server-consul-0.pem"
    key_file = "/etc/consul.d/$DATACENTER-server-consul-0-key.pem"
    verify_incoming = true
    verify_outgoing = true
  }
  internal_rpc {
    verify_server_hostname = true
  }
}

acl = {
  enabled = true
  default_policy = "deny"
  enable_token_persistence = true
EOF

if [ -n "$PRIMARY_DC_IP" ]; then
    # Secondary datacenter configuration
    cat >> /etc/consul.d/consul.hcl << EOF
  enable_token_replication = true
}

primary_datacenter = "dc1"
retry_join_wan = ["$PRIMARY_DC_IP"]
EOF
else
    # Primary datacenter configuration
    cat >> /etc/consul.d/consul.hcl << EOF
  tokens = {
    initial_management = "$BOOTSTRAP_TOKEN"
  }
}
EOF
fi

cat >> /etc/consul.d/consul.hcl << EOF

ports {
  grpc = 8502
  grpc_tls = 8503
}

performance {
  raft_multiplier = 1
}
EOF

chown consul:consul /etc/consul.d/consul.hcl
chmod 640 /etc/consul.d/consul.hcl

# Step 8: Configure firewall
echo -e "${GREEN}[8/10] Configuring firewall...${NC}"
CONSUL_PORTS=(8300 8301 8302 8500 8501 8502 8503 8600)

if [ "$FIREWALL_CMD" = "ufw" ]; then
    if systemctl is-active --quiet ufw; then
        for port in "${CONSUL_PORTS[@]}"; do
            ufw allow "$port" >/dev/null 2>&1 || true
        done
    fi
else
    if systemctl is-active --quiet firewalld; then
        for port in "${CONSUL_PORTS[@]}"; do
            firewall-cmd --permanent --add-port="$port"/tcp >/dev/null 2>&1 || true
            firewall-cmd --permanent --add-port="$port"/udp >/dev/null 2>&1 || true
        done
        firewall-cmd --reload >/dev/null 2>&1 || true
    fi
fi

# Step 9: Start and enable Consul
echo -e "${GREEN}[9/10] Starting Consul service...${NC}"
systemctl daemon-reload
systemctl enable consul
systemctl start consul

# Step 10: Verify installation
echo -e "${GREEN}[10/10] Verifying installation...${NC}"
sleep 10

if systemctl is-active --quiet consul; then
    echo -e "${GREEN}✓ Consul service is running${NC}"
else
    echo -e "${RED}✗ Consul service is not running${NC}"
    exit 1
fi

if consul members >/dev/null 2>&1; then
    echo -e "${GREEN}✓ Consul cluster is accessible${NC}"
else
    echo -e "${YELLOW}⚠ Consul cluster may still be initializing${NC}"
fi

echo -e "${GREEN}Consul installation completed successfully!${NC}"
echo ""
echo -e "${YELLOW}Important information:${NC}"
echo "Gossip encryption key: $GOSSIP_KEY"
echo "Bootstrap token: $BOOTSTRAP_TOKEN"
echo "Web UI: https://$NODE_IP:8501"
echo ""
echo -e "${YELLOW}Next steps:${NC}"
if [ -z "$PRIMARY_DC_IP" ]; then
    echo "1. Configure additional nodes in this datacenter"
    echo "2. Set up secondary datacenters using this node as primary"
else
    echo "1. Configure ACL replication token on primary datacenter"
    echo "2. Join this datacenter to the WAN federation"
fi
echo "3. Configure service mesh and policies as needed"