Setup MinIO monitoring with Prometheus and Grafana dashboards

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
readonly RED='\033[0;31m'
readonly GREEN='\033[0;32m'
readonly YELLOW='\033[0;33m'
readonly NC='\033[0m' # No Color

# Configuration
readonly PROMETHEUS_VERSION="2.45.0"
readonly PROMETHEUS_USER="prometheus"
readonly PROMETHEUS_HOME="/var/lib/prometheus"
readonly PROMETHEUS_CONFIG="/etc/prometheus"

# Usage
usage() {
    echo "Usage: $0 [MINIO_HOST] [MINIO_PORT]"
    echo "  MINIO_HOST: MinIO server host (default: localhost)"
    echo "  MINIO_PORT: MinIO server port (default: 9000)"
    exit 1
}

# Parse arguments
MINIO_HOST="${1:-localhost}"
MINIO_PORT="${2:-9000}"

# Validate port is numeric
if ! [[ "$MINIO_PORT" =~ ^[0-9]+$ ]]; then
    echo -e "${RED}Error: Port must be numeric${NC}"
    usage
fi

# Error handling
cleanup() {
    echo -e "${RED}Installation failed. Rolling back changes...${NC}"
    systemctl stop prometheus 2>/dev/null || true
    systemctl disable prometheus 2>/dev/null || true
    rm -f /etc/systemd/system/prometheus.service
    rm -rf /usr/local/bin/prometheus /usr/local/bin/promtool
    rm -rf "$PROMETHEUS_CONFIG" "$PROMETHEUS_HOME"
    userdel "$PROMETHEUS_USER" 2>/dev/null || true
    systemctl daemon-reload
}

trap cleanup ERR

# Color output functions
info() { echo -e "${GREEN}$1${NC}"; }
warn() { echo -e "${YELLOW}$1${NC}"; }
error() { echo -e "${RED}$1${NC}"; }

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    error "This script must be run as root"
    exit 1
fi

# Detect OS and package manager
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf update -y --refresh"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum update -y"
            ;;
        *) 
            error "Unsupported distribution: $ID"
            exit 1
            ;;
    esac
else
    error "Cannot detect OS distribution"
    exit 1
fi

info "[1/8] Updating package repositories..."
$PKG_UPDATE

info "[2/8] Installing required packages..."
$PKG_INSTALL wget tar

info "[3/8] Creating prometheus user..."
if ! id "$PROMETHEUS_USER" &>/dev/null; then
    useradd --no-create-home --shell /bin/false "$PROMETHEUS_USER"
fi

info "[4/8] Downloading and installing Prometheus..."
cd /tmp
wget -O prometheus.tar.gz "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"
tar xzf prometheus.tar.gz
cp "prometheus-${PROMETHEUS_VERSION}.linux-amd64/prometheus" /usr/local/bin/
cp "prometheus-${PROMETHEUS_VERSION}.linux-amd64/promtool" /usr/local/bin/
chown "$PROMETHEUS_USER:$PROMETHEUS_USER" /usr/local/bin/prometheus
chown "$PROMETHEUS_USER:$PROMETHEUS_USER" /usr/local/bin/promtool
chmod 755 /usr/local/bin/prometheus /usr/local/bin/promtool
rm -rf "prometheus-${PROMETHEUS_VERSION}.linux-amd64" prometheus.tar.gz

info "[5/8] Creating Prometheus directories..."
mkdir -p "$PROMETHEUS_CONFIG" "$PROMETHEUS_HOME"
chown "$PROMETHEUS_USER:$PROMETHEUS_USER" "$PROMETHEUS_CONFIG" "$PROMETHEUS_HOME"
chmod 755 "$PROMETHEUS_CONFIG" "$PROMETHEUS_HOME"

info "[6/8] Creating Prometheus configuration..."
cat > "$PROMETHEUS_CONFIG/prometheus.yml" << EOF
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "minio_alerts.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'minio-cluster'
    metrics_path: /minio/v2/metrics/cluster
    static_configs:
      - targets: ['${MINIO_HOST}:${MINIO_PORT}']
    scrape_interval: 30s
    scrape_timeout: 10s

  - job_name: 'minio-node'
    metrics_path: /minio/v2/metrics/node
    static_configs:
      - targets: ['${MINIO_HOST}:${MINIO_PORT}']
    scrape_interval: 30s
    scrape_timeout: 10s

  - job_name: 'minio-bucket'
    metrics_path: /minio/v2/metrics/bucket
    static_configs:
      - targets: ['${MINIO_HOST}:${MINIO_PORT}']
    scrape_interval: 60s
    scrape_timeout: 15s
EOF

cat > "$PROMETHEUS_CONFIG/minio_alerts.yml" << 'EOF'
groups:
  - name: minio
    rules:
      - alert: MinIODiskUsageHigh
        expr: (minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) * 100 < 10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "MinIO cluster disk usage is critically high"
          description: "MinIO cluster has less than 10% free disk space remaining."

      - alert: MinIONodeOffline
        expr: up{job="minio-node"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "MinIO node is offline"
          description: "MinIO node {{ $labels.instance }} has been offline for more than 2 minutes."

      - alert: MinIOAPIRequestsHigh
        expr: rate(minio_s3_requests_total[5m]) > 1000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High MinIO API request rate"
          description: "MinIO is receiving {{ $value }} requests per second."
EOF

chown -R "$PROMETHEUS_USER:$PROMETHEUS_USER" "$PROMETHEUS_CONFIG"
find "$PROMETHEUS_CONFIG" -type f -exec chmod 644 {} \;

info "[7/8] Creating systemd service..."
cat > /etc/systemd/system/prometheus.service << EOF
[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/prometheus \\
    --config.file /etc/prometheus/prometheus.yml \\
    --storage.tsdb.path /var/lib/prometheus/ \\
    --web.console.templates=/etc/prometheus/consoles \\
    --web.console.libraries=/etc/prometheus/console_libraries \\
    --web.listen-address=0.0.0.0:9090 \\
    --web.enable-lifecycle

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable prometheus
systemctl start prometheus

info "[8/8] Configuring firewall..."
case "$ID" in
    ubuntu|debian)
        if command -v ufw &> /dev/null; then
            ufw allow 9090/tcp
        fi
        ;;
    almalinux|rocky|centos|rhel|ol|fedora|amzn)
        if command -v firewall-cmd &> /dev/null; then
            firewall-cmd --permanent --add-port=9090/tcp
            firewall-cmd --reload
        fi
        ;;
esac

# Verification
info "Verifying installation..."
sleep 5

if systemctl is-active --quiet prometheus; then
    info "✓ Prometheus service is running"
else
    error "✗ Prometheus service failed to start"
    exit 1
fi

if /usr/local/bin/promtool check config "$PROMETHEUS_CONFIG/prometheus.yml" &>/dev/null; then
    info "✓ Prometheus configuration is valid"
else
    error "✗ Prometheus configuration is invalid"
    exit 1
fi

# Display information
info ""
info "Installation completed successfully!"
info "Prometheus is accessible at: http://localhost:9090"
info "MinIO metrics will be scraped from: ${MINIO_HOST}:${MINIO_PORT}"
warn ""
warn "Next steps:"
warn "1. Ensure MinIO is configured with:"
warn "   MINIO_PROMETHEUS_AUTH_TYPE=public"
warn "   MINIO_PROMETHEUS_URL=http://${MINIO_HOST}:${MINIO_PORT}/minio/v2/metrics/cluster"
warn "   MINIO_PROMETHEUS_JOB_ID=minio-job"
warn "2. Install Grafana and import MinIO dashboards"
warn "3. Configure alerting channels in your monitoring setup"