Set up Nomad multi-node cluster with TLS encryption and Consul integration

#!/usr/bin/env bash
set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -t, --type    Node type: server|client (required)"
    echo "  -j, --join    Comma-separated list of server IPs for retry_join"
    echo "  -h, --help    Show this help message"
    echo ""
    echo "Examples:"
    echo "  $0 --type server --join 10.0.1.10,10.0.1.11,10.0.1.12"
    echo "  $0 --type client --join 10.0.1.10,10.0.1.11,10.0.1.12"
    exit 1
}

# Error handling
cleanup() {
    echo -e "${RED}[ERROR] Installation failed. Cleaning up...${NC}"
    systemctl stop nomad consul 2>/dev/null || true
    systemctl disable nomad consul 2>/dev/null || true
}

trap cleanup ERR

# Parse arguments
NODE_TYPE=""
JOIN_IPS=""

while [[ $# -gt 0 ]]; do
    case $1 in
        -t|--type)
            NODE_TYPE="$2"
            shift 2
            ;;
        -j|--join)
            JOIN_IPS="$2"
            shift 2
            ;;
        -h|--help)
            usage
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Validate arguments
if [[ -z "$NODE_TYPE" || ! "$NODE_TYPE" =~ ^(server|client)$ ]]; then
    echo -e "${RED}Error: Node type must be 'server' or 'client'${NC}"
    usage
fi

if [[ -z "$JOIN_IPS" ]]; then
    echo -e "${RED}Error: Join IPs are required${NC}"
    usage
fi

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root or with sudo${NC}"
    exit 1
fi

# Auto-detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_UPDATE="apt update && apt upgrade -y"
            PKG_INSTALL="apt install -y"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            ;;
        *) 
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}Cannot detect distribution${NC}"
    exit 1
fi

echo -e "${GREEN}Setting up Nomad cluster on $PRETTY_NAME${NC}"
echo -e "${YELLOW}Node type: $NODE_TYPE${NC}"

# Step 1: Update system packages
echo -e "${GREEN}[1/10] Updating system packages...${NC}"
$PKG_UPDATE
$PKG_INSTALL curl unzip gpg

# Step 2: Install HashiCorp repository
echo -e "${GREEN}[2/10] Adding HashiCorp repository...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
    echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/hashicorp.list
    apt update
else
    $PKG_INSTALL dnf-plugins-core 2>/dev/null || true
    if command -v dnf &> /dev/null; then
        dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
    else
        yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
    fi
fi

# Step 3: Install Nomad and Consul
echo -e "${GREEN}[3/10] Installing Nomad and Consul...${NC}"
$PKG_INSTALL nomad consul

# Step 4: Create directories
echo -e "${GREEN}[4/10] Creating directories...${NC}"
mkdir -p /etc/nomad.d /opt/nomad/data /etc/consul.d /opt/consul/data
mkdir -p /opt/nomad/tls /opt/consul/tls
chown -R nomad:nomad /etc/nomad.d /opt/nomad
chown -R consul:consul /etc/consul.d /opt/consul

# Step 5: Generate Consul encryption key and TLS certificates
echo -e "${GREEN}[5/10] Generating Consul TLS certificates and encryption key...${NC}"
cd /opt/consul/tls
consul tls ca create
if [[ "$NODE_TYPE" == "server" ]]; then
    consul tls cert create -server -dc dc1
else
    consul tls cert create -client -dc dc1
fi
chown -R consul:consul /opt/consul/tls
chmod 600 /opt/consul/tls/*.pem

CONSUL_ENCRYPT_KEY=$(consul keygen)

# Step 6: Generate Nomad TLS certificates
echo -e "${GREEN}[6/10] Generating Nomad TLS certificates...${NC}"
cd /opt/nomad/tls
nomad tls ca create
if [[ "$NODE_TYPE" == "server" ]]; then
    nomad tls cert create -server -region global -dc dc1
else
    nomad tls cert create -client -region global -dc dc1
fi
chown -R nomad:nomad /opt/nomad/tls
chmod 600 /opt/nomad/tls/*.pem

# Step 7: Configure Consul
echo -e "${GREEN}[7/10] Configuring Consul...${NC}"
IFS=',' read -ra JOIN_ARRAY <<< "$JOIN_IPS"
JOIN_CONFIG=""
for ip in "${JOIN_ARRAY[@]}"; do
    JOIN_CONFIG="${JOIN_CONFIG}\"${ip}\", "
done
JOIN_CONFIG="[${JOIN_CONFIG%%, }]"

if [[ "$NODE_TYPE" == "server" ]]; then
    cat > /etc/consul.d/consul.hcl << EOF
datacenter = "dc1"
data_dir = "/opt/consul/data"
log_level = "INFO"
server = true
bootstrap_expect = 3
ui_config {
  enabled = true
}
bind_addr = "{{ GetInterfaceIP \"eth0\" }}"
client_addr = "0.0.0.0"
retry_join = $JOIN_CONFIG
encrypt = "$CONSUL_ENCRYPT_KEY"
ca_file = "/opt/consul/tls/consul-agent-ca.pem"
cert_file = "/opt/consul/tls/dc1-server-consul-0.pem"
key_file = "/opt/consul/tls/dc1-server-consul-0-key.pem"
verify_incoming = true
verify_outgoing = true
verify_server_hostname = true
connect {
  enabled = true
}
EOF
else
    cat > /etc/consul.d/consul.hcl << EOF
datacenter = "dc1"
data_dir = "/opt/consul/data"
log_level = "INFO"
server = false
bind_addr = "{{ GetInterfaceIP \"eth0\" }}"
retry_join = $JOIN_CONFIG
encrypt = "$CONSUL_ENCRYPT_KEY"
ca_file = "/opt/consul/tls/consul-agent-ca.pem"
cert_file = "/opt/consul/tls/dc1-client-consul-0.pem"
key_file = "/opt/consul/tls/dc1-client-consul-0-key.pem"
verify_incoming = true
verify_outgoing = true
connect {
  enabled = true
}
EOF
fi

chown consul:consul /etc/consul.d/consul.hcl
chmod 640 /etc/consul.d/consul.hcl

# Step 8: Configure Nomad
echo -e "${GREEN}[8/10] Configuring Nomad...${NC}"
if [[ "$NODE_TYPE" == "server" ]]; then
    cat > /etc/nomad.d/nomad.hcl << EOF
datacenter = "dc1"
data_dir = "/opt/nomad/data"
log_level = "INFO"
server {
  enabled = true
  bootstrap_expect = 3
}
consul {
  address = "127.0.0.1:8500"
  server_service_name = "nomad"
  client_service_name = "nomad-client"
  auto_advertise = true
  server_auto_join = true
  client_auto_join = true
}
tls {
  http = true
  rpc = true
  ca_file = "/opt/nomad/tls/nomad-agent-ca.pem"
  cert_file = "/opt/nomad/tls/global-server-nomad.pem"
  key_file = "/opt/nomad/tls/global-server-nomad-key.pem"
  verify_server_hostname = true
  verify_https_client = true
}
EOF
else
    cat > /etc/nomad.d/nomad.hcl << EOF
datacenter = "dc1"
data_dir = "/opt/nomad/data"
log_level = "INFO"
client {
  enabled = true
}
consul {
  address = "127.0.0.1:8500"
  server_service_name = "nomad"
  client_service_name = "nomad-client"
  auto_advertise = true
  server_auto_join = true
  client_auto_join = true
}
tls {
  http = true
  rpc = true
  ca_file = "/opt/nomad/tls/nomad-agent-ca.pem"
  cert_file = "/opt/nomad/tls/global-client-nomad.pem"
  key_file = "/opt/nomad/tls/global-client-nomad-key.pem"
  verify_server_hostname = true
  verify_https_client = true
}
EOF
fi

chown nomad:nomad /etc/nomad.d/nomad.hcl
chmod 640 /etc/nomad.d/nomad.hcl

# Step 9: Start and enable services
echo -e "${GREEN}[9/10] Starting and enabling services...${NC}"
systemctl enable consul nomad
systemctl start consul
sleep 10
systemctl start nomad

# Step 10: Verify installation
echo -e "${GREEN}[10/10] Verifying installation...${NC}"
sleep 5

if systemctl is-active --quiet consul; then
    echo -e "${GREEN}✓ Consul is running${NC}"
else
    echo -e "${RED}✗ Consul is not running${NC}"
    exit 1
fi

if systemctl is-active --quiet nomad; then
    echo -e "${GREEN}✓ Nomad is running${NC}"
else
    echo -e "${RED}✗ Nomad is not running${NC}"
    exit 1
fi

echo -e "${GREEN}Installation completed successfully!${NC}"
echo -e "${YELLOW}Important information:${NC}"
echo -e "${YELLOW}Consul encryption key: $CONSUL_ENCRYPT_KEY${NC}"
echo -e "${YELLOW}Save this key - you'll need it for other nodes!${NC}"
echo ""
echo -e "${GREEN}Next steps:${NC}"
if [[ "$NODE_TYPE" == "server" ]]; then
    echo "- Consul UI: https://$(hostname -I | awk '{print $1}'):8500"
    echo "- Nomad UI: https://$(hostname -I | awk '{print $1}'):4646"
fi
echo "- Check status: systemctl status consul nomad"
echo "- View logs: journalctl -u consul -u nomad -f"