Setup Elasticsearch monitoring with Metricbeat and Kibana dashboards

#!/usr/bin/env bash
set -euo pipefail

# Elasticsearch Monitoring Setup with Metricbeat and Kibana
# Production-grade installation script

# Color codes
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Global variables
ELASTICSEARCH_HOST="${1:-localhost}"
ELASTICSEARCH_PORT="${2:-9200}"
KIBANA_HOST="${3:-localhost}"
KIBANA_PORT="${4:-5601}"

# Usage function
usage() {
    echo "Usage: $0 [elasticsearch_host] [elasticsearch_port] [kibana_host] [kibana_port]"
    echo "Example: $0 localhost 9200 localhost 5601"
    exit 1
}

# Logging functions
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Cleanup function for rollback
cleanup() {
    log_error "Installation failed. Performing cleanup..."
    systemctl stop metricbeat 2>/dev/null || true
    systemctl disable metricbeat 2>/dev/null || true
    if [ "$PKG_MGR" = "apt" ]; then
        apt remove -y metricbeat 2>/dev/null || true
    else
        $PKG_INSTALL remove -y metricbeat 2>/dev/null || true
    fi
    rm -f /etc/metricbeat/metricbeat.yml.bak
}

trap cleanup ERR

# Check if running as root or with sudo
check_privileges() {
    if [ "$EUID" -ne 0 ]; then
        log_error "This script must be run as root or with sudo"
        exit 1
    fi
}

# Detect OS and set package manager
detect_os() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian)
                PKG_MGR="apt"
                PKG_INSTALL="apt install -y"
                PKG_UPDATE="apt update && apt upgrade -y"
                REPO_SETUP="debian"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora)
                PKG_MGR="dnf"
                PKG_INSTALL="dnf install -y"
                PKG_UPDATE="dnf update -y"
                REPO_SETUP="rhel"
                ;;
            amzn)
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                PKG_UPDATE="yum update -y"
                REPO_SETUP="rhel"
                ;;
            *)
                log_error "Unsupported distribution: $ID"
                exit 1
                ;;
        esac
        log_info "Detected OS: $PRETTY_NAME"
    else
        log_error "Cannot detect OS. /etc/os-release not found"
        exit 1
    fi
}

# Check prerequisites
check_prerequisites() {
    echo "[1/8] Checking prerequisites..."
    
    # Check if elasticsearch and kibana are accessible
    if ! command -v curl >/dev/null 2>&1; then
        log_warn "curl not found, installing..."
        $PKG_INSTALL curl
    fi
    
    # Test Elasticsearch connectivity
    if ! curl -s -f "http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}" >/dev/null 2>&1; then
        log_warn "Cannot connect to Elasticsearch at ${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"
        log_warn "Please ensure Elasticsearch is running and accessible"
    fi
    
    log_info "Prerequisites check completed"
}

# Update system packages
update_system() {
    echo "[2/8] Updating system packages..."
    log_info "Running system update..."
    $PKG_UPDATE
    log_info "System update completed"
}

# Add Elastic repository
add_elastic_repo() {
    echo "[3/8] Adding Elastic repository..."
    
    if [ "$REPO_SETUP" = "debian" ]; then
        # Install required packages
        $PKG_INSTALL wget gpg apt-transport-https
        
        # Add GPG key
        wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
        
        # Add repository
        echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" > /etc/apt/sources.list.d/elastic-8.x.list
        
        # Update package list
        apt update
    else
        # RHEL-based systems
        # Import GPG key
        rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
        
        # Create repository file
        cat > /etc/yum.repos.d/elastic.repo << EOF
[elastic-8.x]
name=Elastic repository for 8.x packages
baseurl=https://artifacts.elastic.co/packages/8.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
    fi
    
    log_info "Elastic repository added successfully"
}

# Install Metricbeat
install_metricbeat() {
    echo "[4/8] Installing Metricbeat..."
    
    $PKG_INSTALL metricbeat
    
    log_info "Metricbeat installed successfully"
}

# Configure Metricbeat
configure_metricbeat() {
    echo "[5/8] Configuring Metricbeat..."
    
    # Backup original config
    cp /etc/metricbeat/metricbeat.yml /etc/metricbeat/metricbeat.yml.bak
    
    # Create new configuration
    cat > /etc/metricbeat/metricbeat.yml << EOF
# Metricbeat configuration for Elasticsearch monitoring
metricbeat.config.modules:
  path: /etc/metricbeat/modules.d/*.yml
  reload.enabled: true
  reload.period: 30s

metricbeat.modules:
- module: elasticsearch
  metricsets:
    - node
    - node_stats
    - cluster_stats
    - index
    - index_summary
    - index_recovery
    - ccr
  period: 30s
  hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
  
- module: system
  metricsets:
    - cpu
    - memory
    - network
    - process
    - process_summary
    - filesystem
    - diskio
  enabled: true
  period: 30s
  processes: ['.*']

output.elasticsearch:
  hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
  index: "metricbeat-%{+yyyy.MM.dd}"

setup.kibana:
  host: "${KIBANA_HOST}:${KIBANA_PORT}"

setup.dashboards.enabled: true
setup.template.enabled: true

processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded

logging.level: info
logging.to_files: true
logging.files:
  path: /var/log/metricbeat
  name: metricbeat
  keepfiles: 7
  permissions: 0644
EOF

    # Set proper ownership and permissions
    chown root:root /etc/metricbeat/metricbeat.yml
    chmod 644 /etc/metricbeat/metricbeat.yml
    
    # Create log directory
    mkdir -p /var/log/metricbeat
    chown metricbeat:metricbeat /var/log/metricbeat
    chmod 755 /var/log/metricbeat
    
    log_info "Metricbeat configuration completed"
}

# Setup Kibana dashboards and templates
setup_kibana_assets() {
    echo "[6/8] Setting up Kibana dashboards and index templates..."
    
    # Setup Elasticsearch template and Kibana dashboards
    metricbeat setup --dashboards --index-management
    
    log_info "Kibana assets setup completed"
}

# Start and enable services
start_services() {
    echo "[7/8] Starting and enabling Metricbeat service..."
    
    # Enable and start metricbeat
    systemctl enable metricbeat
    systemctl start metricbeat
    
    # Wait a moment for the service to start
    sleep 3
    
    # Check service status
    if systemctl is-active --quiet metricbeat; then
        log_info "Metricbeat service started successfully"
    else
        log_error "Failed to start Metricbeat service"
        journalctl -u metricbeat --no-pager -l
        exit 1
    fi
}

# Verify installation
verify_installation() {
    echo "[8/8] Verifying installation..."
    
    # Check service status
    if systemctl is-active --quiet metricbeat; then
        log_info "✓ Metricbeat service is running"
    else
        log_error "✗ Metricbeat service is not running"
        return 1
    fi
    
    # Wait for data to be indexed
    log_info "Waiting 30 seconds for metrics to be collected..."
    sleep 30
    
    # Check if data is being indexed
    local data_check
    data_check=$(curl -s -f "http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/metricbeat-*/_search?q=metricset.module:elasticsearch&size=1" 2>/dev/null || echo "")
    
    if echo "$data_check" | grep -q "elasticsearch"; then
        log_info "✓ Elasticsearch metrics are being collected"
    else
        log_warn "! No Elasticsearch metrics found yet. This may be normal for a new installation."
    fi
    
    # Show recent logs
    log_info "Recent Metricbeat logs:"
    journalctl -u metricbeat --no-pager -l -n 5
    
    log_info "Installation completed successfully!"
    echo ""
    log_info "Next steps:"
    echo "1. Access Kibana at http://${KIBANA_HOST}:${KIBANA_PORT}"
    echo "2. Go to Dashboard and search for 'Elasticsearch' to view monitoring dashboards"
    echo "3. Check Stack Management > Index Patterns for 'metricbeat-*' pattern"
    echo "4. Configure alerts in Stack Management > Rules and Connectors"
    echo ""
    log_info "Useful commands:"
    echo "- Check status: sudo systemctl status metricbeat"
    echo "- View logs: sudo journalctl -u metricbeat -f"
    echo "- Test config: sudo metricbeat test config"
}

# Main execution
main() {
    log_info "Starting Elasticsearch Monitoring Setup with Metricbeat"
    
    check_privileges
    detect_os
    check_prerequisites
    update_system
    add_elastic_repo
    install_metricbeat
    configure_metricbeat
    setup_kibana_assets
    start_services
    verify_installation
    
    log_info "Elasticsearch monitoring setup completed successfully!"
}

# Run main function
main "$@"