Configure Elasticsearch 8 cluster with multiple nodes for high availability and scalability

#!/usr/bin/env bash
set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Global variables
CLUSTER_NAME="production-cluster"
NODE_NAME=""
NODE_ROLE=""
NODE_IP=""
DISCOVERY_HOSTS=""
MASTER_NODES=""

# Cleanup function for error handling
cleanup() {
    echo -e "${RED}[ERROR] Installation failed. Cleaning up...${NC}"
    systemctl stop elasticsearch 2>/dev/null || true
    systemctl disable elasticsearch 2>/dev/null || true
    rm -rf /tmp/elastic-*.zip /tmp/ca /tmp/instance 2>/dev/null || true
}
trap cleanup ERR

usage() {
    cat << EOF
Usage: $0 [OPTIONS]
Configure Elasticsearch 8 cluster node

OPTIONS:
    -n, --node-name       Node name (required)
    -r, --role           Node role: master|data|coordinating (required)
    -i, --ip             Node IP address (required)
    -d, --discovery      Comma-separated discovery hosts (required)
    -m, --masters        Comma-separated initial master nodes (required for first master)
    -c, --cluster        Cluster name (default: production-cluster)
    -h, --help           Show this help

Example:
    $0 -n master-node-1 -r master -i 203.0.113.10 -d "203.0.113.10:9300,203.0.113.11:9300" -m "master-node-1,master-node-2"
EOF
    exit 1
}

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -n|--node-name) NODE_NAME="$2"; shift 2 ;;
        -r|--role) NODE_ROLE="$2"; shift 2 ;;
        -i|--ip) NODE_IP="$2"; shift 2 ;;
        -d|--discovery) DISCOVERY_HOSTS="$2"; shift 2 ;;
        -m|--masters) MASTER_NODES="$2"; shift 2 ;;
        -c|--cluster) CLUSTER_NAME="$2"; shift 2 ;;
        -h|--help) usage ;;
        *) echo -e "${RED}Unknown option: $1${NC}"; usage ;;
    esac
done

# Validate required arguments
if [[ -z "$NODE_NAME" || -z "$NODE_ROLE" || -z "$NODE_IP" || -z "$DISCOVERY_HOSTS" ]]; then
    echo -e "${RED}Missing required arguments${NC}"
    usage
fi

if [[ "$NODE_ROLE" != "master" && "$NODE_ROLE" != "data" && "$NODE_ROLE" != "coordinating" ]]; then
    echo -e "${RED}Invalid role. Must be: master, data, or coordinating${NC}"
    exit 1
fi

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root or with sudo${NC}"
    exit 1
fi

echo -e "${BLUE}=== Elasticsearch 8 Cluster Setup ===${NC}"

# Detect distribution
echo -e "${BLUE}[1/9] Detecting distribution...${NC}"
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            JAVA_PACKAGE="openjdk-11-jdk"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            JAVA_PACKAGE="java-11-openjdk"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            JAVA_PACKAGE="java-11-openjdk"
            ;;
        *) 
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
    echo -e "${GREEN}Detected: $PRETTY_NAME${NC}"
else
    echo -e "${RED}Cannot detect distribution${NC}"
    exit 1
fi

# Configure system settings
echo -e "${BLUE}[2/9] Configuring system settings...${NC}"
current_max_map_count=$(sysctl -n vm.max_map_count)
if [[ $current_max_map_count -lt 262144 ]]; then
    sysctl -w vm.max_map_count=262144
    echo 'vm.max_map_count=262144' >> /etc/sysctl.conf
    echo -e "${GREEN}Configured vm.max_map_count=262144${NC}"
else
    echo -e "${GREEN}vm.max_map_count already configured${NC}"
fi

# Install Java and prerequisites
echo -e "${BLUE}[3/9] Installing Java and prerequisites...${NC}"
$PKG_UPDATE
$PKG_INSTALL $JAVA_PACKAGE wget gpg curl unzip

# Add Elasticsearch repository and install
echo -e "${BLUE}[4/9] Installing Elasticsearch...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
    echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" > /etc/apt/sources.list.d/elastic-8.x.list
    apt update
    $PKG_INSTALL elasticsearch
else
    rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
    cat > /etc/yum.repos.d/elasticsearch.repo << 'EOF'
[elasticsearch]
name=Elasticsearch repository for 8.x packages
baseurl=https://artifacts.elastic.co/packages/8.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
    $PKG_INSTALL elasticsearch
fi

# Generate certificates (only for first master node)
echo -e "${BLUE}[5/9] Setting up certificates...${NC}"
mkdir -p /etc/elasticsearch/certs

if [[ "$NODE_ROLE" == "master" && "$NODE_NAME" == *"1"* ]] && [[ -n "$MASTER_NODES" ]]; then
    echo -e "${YELLOW}Generating certificates for cluster...${NC}"
    /usr/share/elasticsearch/bin/elasticsearch-certutil ca --pem --out /tmp/elastic-ca.zip --silent
    unzip -q /tmp/elastic-ca.zip -d /tmp/
    /usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca-cert /tmp/ca/ca.crt --ca-key /tmp/ca/ca.key --pem --out /tmp/elastic-certificates.zip --silent
    unzip -q /tmp/elastic-certificates.zip -d /tmp/
    cp /tmp/ca/ca.crt /tmp/instance/instance.crt /tmp/instance/instance.key /etc/elasticsearch/certs/
    chown -R elasticsearch:elasticsearch /etc/elasticsearch/certs/
    chmod -R 640 /etc/elasticsearch/certs/*
    echo -e "${YELLOW}Copy these certificates to other nodes:${NC}"
    echo -e "${YELLOW}/etc/elasticsearch/certs/ca.crt${NC}"
    echo -e "${YELLOW}/etc/elasticsearch/certs/instance.crt${NC}"
    echo -e "${YELLOW}/etc/elasticsearch/certs/instance.key${NC}"
else
    echo -e "${YELLOW}Please copy certificates from the first master node to /etc/elasticsearch/certs/${NC}"
    echo -e "${YELLOW}Press Enter when certificates are in place...${NC}"
    read -r
    chown -R elasticsearch:elasticsearch /etc/elasticsearch/certs/
    chmod -R 640 /etc/elasticsearch/certs/*
fi

# Configure node role
echo -e "${BLUE}[6/9] Configuring node role...${NC}"
case "$NODE_ROLE" in
    master)
        NODE_ROLES="[master]"
        ;;
    data)
        NODE_ROLES="[data, data_content, data_hot, data_warm, data_cold]"
        ;;
    coordinating)
        NODE_ROLES="[]"
        ;;
esac

# Format discovery hosts and master nodes
FORMATTED_DISCOVERY=$(echo "$DISCOVERY_HOSTS" | sed 's/,/", "/g' | sed 's/^/"/' | sed 's/$/"/')
FORMATTED_MASTERS=""
if [[ -n "$MASTER_NODES" ]]; then
    FORMATTED_MASTERS=$(echo "$MASTER_NODES" | sed 's/,/", "/g' | sed 's/^/"/' | sed 's/$/"/')
fi

# Create Elasticsearch configuration
echo -e "${BLUE}[7/9] Creating Elasticsearch configuration...${NC}"
cat > /etc/elasticsearch/elasticsearch.yml << EOF
# Cluster configuration
cluster.name: $CLUSTER_NAME
node.name: $NODE_NAME
node.roles: $NODE_ROLES

# Paths
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch

# Network
network.host: $NODE_IP
http.port: 9200
transport.port: 9300

# Discovery
discovery.seed_hosts: [$FORMATTED_DISCOVERY]
EOF

if [[ -n "$MASTER_NODES" ]]; then
    echo "cluster.initial_master_nodes: [$FORMATTED_MASTERS]" >> /etc/elasticsearch/elasticsearch.yml
fi

cat >> /etc/elasticsearch/elasticsearch.yml << 'EOF'

# Security
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.certificate: certs/instance.crt
xpack.security.transport.ssl.key: certs/instance.key
xpack.security.transport.ssl.certificate_authorities: certs/ca.crt
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.certificate: certs/instance.crt
xpack.security.http.ssl.key: certs/instance.key
xpack.security.http.ssl.certificate_authorities: certs/ca.crt

# Monitoring
xpack.monitoring.collection.enabled: true

# Performance
bootstrap.memory_lock: true
EOF

# Configure JVM heap size based on node role
echo -e "${BLUE}[8/9] Configuring JVM settings...${NC}"
TOTAL_RAM=$(free -m | awk 'NR==2{print $2}')
case "$NODE_ROLE" in
    master)
        HEAP_SIZE="2g"
        ;;
    data)
        HEAP_SIZE=$((TOTAL_RAM / 2 / 1024))g
        ;;
    coordinating)
        HEAP_SIZE="1g"
        ;;
esac

sed -i "s/-Xms1g/-Xms$HEAP_SIZE/" /etc/elasticsearch/jvm.options
sed -i "s/-Xmx1g/-Xmx$HEAP_SIZE/" /etc/elasticsearch/jvm.options

# Enable and start Elasticsearch
systemctl daemon-reload
systemctl enable elasticsearch
systemctl start elasticsearch

echo -e "${BLUE}[9/9] Verifying installation...${NC}"
sleep 10

# Check service status
if systemctl is-active --quiet elasticsearch; then
    echo -e "${GREEN}✓ Elasticsearch service is running${NC}"
else
    echo -e "${RED}✗ Elasticsearch service failed to start${NC}"
    journalctl -u elasticsearch --no-pager -n 20
    exit 1
fi

# Check if port is listening
if netstat -tlnp 2>/dev/null | grep -q ":9200.*LISTEN" || ss -tlnp 2>/dev/null | grep -q ":9200.*LISTEN"; then
    echo -e "${GREEN}✓ Elasticsearch is listening on port 9200${NC}"
else
    echo -e "${YELLOW}⚠ Elasticsearch port 9200 not yet available${NC}"
fi

echo -e "${GREEN}=== Installation Complete ===${NC}"
echo -e "${BLUE}Node: $NODE_NAME ($NODE_ROLE)${NC}"
echo -e "${BLUE}IP: $NODE_IP${NC}"
echo -e "${BLUE}Cluster: $CLUSTER_NAME${NC}"
echo ""
echo -e "${YELLOW}Next steps:${NC}"
echo -e "1. Configure other cluster nodes"
echo -e "2. Set up built-in user passwords: /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto"
echo -e "3. Configure firewall to allow ports 9200 and 9300"
echo -e "4. Test cluster health: curl -k https://$NODE_IP:9200/_cluster/health"

# Cleanup temporary files
rm -f /tmp/elastic-*.zip
rm -rf /tmp/ca /tmp/instance