Configure MinIO high availability clustering for production with multi-node setup and automatic failover

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Global variables
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CLEANUP_FILES=()

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -n, --nodes     Comma-separated list of node IPs (default: 203.0.113.10,203.0.113.11,203.0.113.12,203.0.113.13,203.0.113.14,203.0.113.15)"
    echo "  -d, --domain    Cluster domain name (default: minio-cluster.example.com)"
    echo "  -h, --help      Show this help message"
    echo ""
    echo "Example:"
    echo "  $0 -n 10.0.1.10,10.0.1.11,10.0.1.12,10.0.1.13,10.0.1.14,10.0.1.15 -d minio.company.com"
    exit 1
}

# Error handling with cleanup
cleanup() {
    if [ ${#CLEANUP_FILES[@]} -gt 0 ]; then
        echo -e "${YELLOW}[CLEANUP] Removing temporary files...${NC}"
        for file in "${CLEANUP_FILES[@]}"; do
            [ -f "$file" ] && rm -f "$file"
        done
    fi
}

trap cleanup ERR EXIT

# Logging functions
log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Check if running as root or with sudo
check_privileges() {
    if [[ $EUID -ne 0 ]]; then
        log_error "This script must be run as root or with sudo"
        exit 1
    fi
}

# Detect distribution and set package manager
detect_distro() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian)
                PKG_MGR="apt"
                PKG_INSTALL="apt install -y"
                PKG_UPDATE="apt update && apt upgrade -y"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora)
                PKG_MGR="dnf"
                PKG_INSTALL="dnf install -y"
                PKG_UPDATE="dnf update -y"
                ;;
            amzn)
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                PKG_UPDATE="yum update -y"
                ;;
            *)
                log_error "Unsupported distribution: $ID"
                exit 1
                ;;
        esac
        export PKG_MGR PKG_INSTALL PKG_UPDATE
        log_info "Detected distribution: $ID"
    else
        log_error "Cannot detect distribution. /etc/os-release not found."
        exit 1
    fi
}

# Parse command line arguments
parse_args() {
    NODES="203.0.113.10,203.0.113.11,203.0.113.12,203.0.113.13,203.0.113.14,203.0.113.15"
    DOMAIN="minio-cluster.example.com"
    
    while [[ $# -gt 0 ]]; do
        case $1 in
            -n|--nodes)
                NODES="$2"
                shift 2
                ;;
            -d|--domain)
                DOMAIN="$2"
                shift 2
                ;;
            -h|--help)
                usage
                ;;
            *)
                log_error "Unknown option: $1"
                usage
                ;;
        esac
    done
    
    # Validate nodes
    IFS=',' read -ra NODE_ARRAY <<< "$NODES"
    if [ ${#NODE_ARRAY[@]} -lt 4 ]; then
        log_error "MinIO clustering requires at least 4 nodes"
        exit 1
    fi
    
    export NODES DOMAIN NODE_ARRAY
}

# Update system packages
update_system() {
    echo -e "${BLUE}[1/10] Updating system packages...${NC}"
    eval "$PKG_UPDATE"
    eval "$PKG_INSTALL wget curl systemd openssl"
    log_success "System packages updated"
}

# Create storage directories
create_storage() {
    echo -e "${BLUE}[2/10] Creating storage directories...${NC}"
    mkdir -p /data/disk{1,2,3,4}
    chmod 755 /data/disk{1,2,3,4}
    log_success "Storage directories created"
}

# Download and install MinIO binary
install_minio() {
    echo -e "${BLUE}[3/10] Downloading and installing MinIO binary...${NC}"
    cd /tmp
    wget -q https://dl.min.io/server/minio/release/linux-amd64/minio
    chmod +x minio
    mv minio /usr/local/bin/
    ln -sf /usr/local/bin/minio /usr/local/bin/minio
    log_success "MinIO binary installed"
}

# Create MinIO user and group
create_user() {
    echo -e "${BLUE}[4/10] Creating MinIO user and group...${NC}"
    if ! getent group minio-user > /dev/null 2>&1; then
        groupadd -r minio-user
    fi
    if ! getent passwd minio-user > /dev/null 2>&1; then
        useradd -M -r -g minio-user minio-user
    fi
    chown minio-user:minio-user /data/disk{1,2,3,4}
    log_success "MinIO user created"
}

# Configure host resolution
configure_hosts() {
    echo -e "${BLUE}[5/10] Configuring host resolution...${NC}"
    
    # Remove existing MinIO entries
    sed -i '/# MinIO Cluster Nodes/,/^$/d' /etc/hosts
    
    # Add new entries
    echo "# MinIO Cluster Nodes" >> /etc/hosts
    local i=1
    for node in "${NODE_ARRAY[@]}"; do
        echo "$node    minio-$i" >> /etc/hosts
        ((i++))
    done
    echo "" >> /etc/hosts
    
    log_success "Host resolution configured"
}

# Create MinIO configuration
create_config() {
    echo -e "${BLUE}[6/10] Creating MinIO configuration...${NC}"
    
    mkdir -p /etc/minio
    chmod 750 /etc/minio
    
    # Generate credentials
    ACCESS_KEY=$(openssl rand -base64 32 | tr -d "=+/" | cut -c1-20)
    SECRET_KEY=$(openssl rand -base64 48 | tr -d "=+/" | cut -c1-40)
    
    # Create volume string
    local volume_string="http://minio-{1...${#NODE_ARRAY[@]}}/data/disk{1...4}"
    
    # Create configuration file
    cat > /etc/minio/minio.conf << EOF
# MinIO Cluster Configuration
MINIO_ROOT_USER="$ACCESS_KEY"
MINIO_ROOT_PASSWORD="$SECRET_KEY"
MINIO_VOLUMES="$volume_string"
MINIO_OPTS="--console-address :9001 --address :9000"
MINIO_SERVER_URL="https://$DOMAIN:9000"
MINIO_BROWSER_REDIRECT_URL="https://$DOMAIN:9001"

# Security and Performance
MINIO_PROMETHEUS_AUTH_TYPE="public"
MINIO_HEAL_WORKERS="4"
MINIO_HEAL_DRIVE_WORKERS="2"
EOF
    
    chown minio-user:minio-user /etc/minio/minio.conf
    chmod 640 /etc/minio/minio.conf
    chown minio-user:minio-user /etc/minio
    
    log_success "MinIO configuration created"
    log_warning "Access Key: $ACCESS_KEY"
    log_warning "Secret Key: $SECRET_KEY"
    log_warning "Please save these credentials securely!"
}

# Create systemd service
create_service() {
    echo -e "${BLUE}[7/10] Creating systemd service...${NC}"
    
    cat > /etc/systemd/system/minio.service << EOF
[Unit]
Description=MinIO Object Storage Server
Documentation=https://docs.min.io
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/usr/local/bin/minio

[Service]
WorkingDirectory=/usr/local/
User=minio-user
Group=minio-user
ProtectProc=invisible
EnvironmentFile=/etc/minio/minio.conf
ExecStartPre=/bin/bash -c "if [ -z \"\${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/minio/minio.conf\"; exit 1; fi"
ExecStart=/usr/local/bin/minio server \$MINIO_VOLUMES \$MINIO_OPTS
Restart=always
LimitNOFILE=65536
TasksMax=infinity
TimeoutStopSec=infinity
SendSIGKILL=no

[Install]
WantedBy=multi-user.target
EOF
    
    chmod 644 /etc/systemd/system/minio.service
    systemctl daemon-reload
    log_success "Systemd service created"
}

# Configure firewall
configure_firewall() {
    echo -e "${BLUE}[8/10] Configuring firewall...${NC}"
    
    case "$PKG_MGR" in
        apt)
            if command -v ufw > /dev/null; then
                ufw allow 9000/tcp comment "MinIO API"
                ufw allow 9001/tcp comment "MinIO Console"
            fi
            ;;
        dnf|yum)
            if command -v firewall-cmd > /dev/null; then
                firewall-cmd --permanent --add-port=9000/tcp
                firewall-cmd --permanent --add-port=9001/tcp
                firewall-cmd --reload
            fi
            ;;
    esac
    
    log_success "Firewall configured"
}

# Enable and start service
start_service() {
    echo -e "${BLUE}[9/10] Enabling and starting MinIO service...${NC}"
    
    systemctl enable minio
    systemctl start minio
    
    # Wait for service to start
    sleep 5
    
    if systemctl is-active --quiet minio; then
        log_success "MinIO service started successfully"
    else
        log_error "MinIO service failed to start"
        systemctl status minio --no-pager
        exit 1
    fi
}

# Verify installation
verify_installation() {
    echo -e "${BLUE}[10/10] Verifying installation...${NC}"
    
    # Check if service is running
    if systemctl is-active --quiet minio; then
        log_success "MinIO service is running"
    else
        log_error "MinIO service is not running"
        return 1
    fi
    
    # Check if ports are listening
    if netstat -tuln 2>/dev/null | grep -q ":9000\|:9001" || ss -tuln 2>/dev/null | grep -q ":9000\|:9001"; then
        log_success "MinIO is listening on ports 9000 and 9001"
    else
        log_warning "MinIO ports may not be accessible yet"
    fi
    
    log_success "MinIO cluster installation completed!"
    echo ""
    echo -e "${GREEN}Next steps:${NC}"
    echo "1. Run this script on all cluster nodes with the same configuration"
    echo "2. Access MinIO console at: http://$(hostname -I | awk '{print $1}'):9001"
    echo "3. Configure load balancer for production use"
    echo "4. Set up SSL/TLS certificates for HTTPS access"
}

# Main execution
main() {
    echo -e "${GREEN}MinIO High Availability Cluster Installation${NC}"
    echo "=============================================="
    
    check_privileges
    detect_distro
    parse_args "$@"
    
    update_system
    create_storage
    install_minio
    create_user
    configure_hosts
    create_config
    create_service
    configure_firewall
    start_service
    verify_installation
    
    log_success "Installation completed successfully!"
}

# Run main function with all arguments
main "$@"