Configure Grafana dashboards for InfluxDB collectd metrics with advanced visualization and alerting

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
readonly RED='\033[0;31m'
readonly GREEN='\033[0;32m'
readonly YELLOW='\033[1;33m'
readonly NC='\033[0m'

# Global variables
INFLUXDB_USER="${1:-collectd_user}"
INFLUXDB_PASSWORD="${2:-$(openssl rand -base64 32)}"
GRAFANA_ADMIN_PASSWORD="${3:-$(openssl rand -base64 32)}"

usage() {
    echo "Usage: $0 [influxdb_user] [influxdb_password] [grafana_admin_password]"
    echo "  influxdb_user: InfluxDB user for Grafana (default: collectd_user)"
    echo "  influxdb_password: Password for InfluxDB user (default: auto-generated)"
    echo "  grafana_admin_password: Grafana admin password (default: auto-generated)"
    exit 1
}

log() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1" >&2
}

cleanup() {
    error "Script failed. Check logs above for details."
    exit 1
}

trap cleanup ERR

check_prerequisites() {
    log "Checking prerequisites..."
    
    if [[ $EUID -ne 0 ]]; then
        error "This script must be run as root"
        exit 1
    fi
    
    if ! command -v curl &> /dev/null; then
        error "curl is required but not installed"
        exit 1
    fi
    
    if ! systemctl is-active --quiet influxdb; then
        error "InfluxDB is not running. Please install and configure InfluxDB first."
        exit 1
    fi
}

detect_distro() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian) 
                PKG_MGR="apt"
                PKG_INSTALL="apt install -y"
                PKG_UPDATE="apt update"
                GRAFANA_CONFIG="/etc/grafana/grafana.ini"
                FIREWALL_CMD="ufw"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora)
                PKG_MGR="dnf"
                PKG_INSTALL="dnf install -y"
                PKG_UPDATE="dnf update -y"
                GRAFANA_CONFIG="/etc/grafana/grafana.ini"
                FIREWALL_CMD="firewall-cmd"
                ;;
            amzn)
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                PKG_UPDATE="yum update -y"
                GRAFANA_CONFIG="/etc/grafana/grafana.ini"
                FIREWALL_CMD="firewall-cmd"
                ;;
            *)
                error "Unsupported distribution: $ID"
                exit 1
                ;;
        esac
    else
        error "Cannot detect distribution"
        exit 1
    fi
}

install_grafana() {
    echo "[2/7] Installing Grafana..."
    
    if command -v grafana-server &> /dev/null; then
        log "Grafana already installed"
        return
    fi
    
    case "$PKG_MGR" in
        apt)
            curl -fsSL https://packages.grafana.com/gpg.key | gpg --dearmor -o /usr/share/keyrings/grafana.gpg
            echo "deb [signed-by=/usr/share/keyrings/grafana.gpg] https://packages.grafana.com/oss/deb stable main" > /etc/apt/sources.list.d/grafana.list
            $PKG_UPDATE
            $PKG_INSTALL grafana
            ;;
        dnf|yum)
            cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
            $PKG_INSTALL grafana
            ;;
    esac
    
    systemctl daemon-reload
    systemctl enable grafana-server
}

configure_grafana() {
    echo "[3/7] Configuring Grafana..."
    
    # Set admin password
    grafana-cli admin reset-admin-password "$GRAFANA_ADMIN_PASSWORD"
    
    # Configure Grafana settings
    sed -i "s/^;http_port = 3000/http_port = 3000/" "$GRAFANA_CONFIG"
    sed -i "s/^;domain = localhost/domain = $(hostname -f)/" "$GRAFANA_CONFIG"
    
    systemctl start grafana-server
    
    # Wait for Grafana to start
    local count=0
    while ! curl -s http://localhost:3000 > /dev/null; do
        sleep 2
        count=$((count + 1))
        if [ $count -gt 30 ]; then
            error "Grafana failed to start"
            exit 1
        fi
    done
}

configure_firewall() {
    echo "[4/7] Configuring firewall..."
    
    case "$FIREWALL_CMD" in
        ufw)
            if command -v ufw &> /dev/null && ufw status | grep -q "Status: active"; then
                ufw allow 3000/tcp
                log "UFW rule added for Grafana port 3000"
            fi
            ;;
        firewall-cmd)
            if systemctl is-active --quiet firewalld; then
                firewall-cmd --permanent --add-port=3000/tcp
                firewall-cmd --reload
                log "Firewall rule added for Grafana port 3000"
            fi
            ;;
    esac
}

setup_influxdb_user() {
    echo "[5/7] Setting up InfluxDB user..."
    
    # Create InfluxDB user for Grafana
    influx -execute "CREATE USER \"$INFLUXDB_USER\" WITH PASSWORD '$INFLUXDB_PASSWORD'"
    influx -execute "GRANT READ ON \"collectd\" TO \"$INFLUXDB_USER\""
    
    log "InfluxDB user created: $INFLUXDB_USER"
}

configure_datasource() {
    echo "[6/7] Configuring InfluxDB data source in Grafana..."
    
    # Wait a bit more for Grafana API to be ready
    sleep 5
    
    # Create InfluxDB data source
    cat > /tmp/datasource.json << EOF
{
  "name": "InfluxDB-collectd",
  "type": "influxdb",
  "url": "http://localhost:8086",
  "access": "proxy",
  "database": "collectd",
  "user": "$INFLUXDB_USER",
  "password": "$INFLUXDB_PASSWORD",
  "isDefault": true
}
EOF

    curl -X POST \
        -H "Content-Type: application/json" \
        -d @/tmp/datasource.json \
        http://admin:$GRAFANA_ADMIN_PASSWORD@localhost:3000/api/datasources
    
    rm -f /tmp/datasource.json
    log "InfluxDB data source configured"
}

import_dashboard() {
    echo "[7/7] Creating collectd dashboard..."
    
    # Create a basic collectd dashboard
    cat > /tmp/dashboard.json << 'EOF'
{
  "dashboard": {
    "id": null,
    "title": "Collectd System Monitoring",
    "tags": ["collectd"],
    "timezone": "browser",
    "panels": [
      {
        "id": 1,
        "title": "CPU Usage",
        "type": "timeseries",
        "targets": [
          {
            "query": "SELECT 100 - mean(\"value\") FROM \"cpu_value\" WHERE \"type_instance\" = 'idle' AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
      },
      {
        "id": 2,
        "title": "Memory Usage",
        "type": "timeseries",
        "targets": [
          {
            "query": "SELECT mean(\"value\") FROM \"memory_value\" WHERE \"type_instance\" =~ /^(used|cached|free|buffered)$/ AND $timeFilter GROUP BY time($__interval), \"type_instance\" fill(null)"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
      }
    ],
    "time": {"from": "now-1h", "to": "now"},
    "refresh": "30s"
  },
  "overwrite": true
}
EOF

    curl -X POST \
        -H "Content-Type: application/json" \
        -d @/tmp/dashboard.json \
        http://admin:$GRAFANA_ADMIN_PASSWORD@localhost:3000/api/dashboards/db
    
    rm -f /tmp/dashboard.json
    log "Collectd dashboard imported"
}

verify_installation() {
    log "Verifying installation..."
    
    if ! systemctl is-active --quiet grafana-server; then
        error "Grafana service is not running"
        return 1
    fi
    
    if ! curl -s http://localhost:3000 > /dev/null; then
        error "Grafana web interface is not accessible"
        return 1
    fi
    
    log "Installation completed successfully!"
    log "Grafana URL: http://$(hostname -I | awk '{print $1}'):3000"
    log "Username: admin"
    log "Password: $GRAFANA_ADMIN_PASSWORD"
    log "InfluxDB User: $INFLUXDB_USER"
    log "InfluxDB Password: $INFLUXDB_PASSWORD"
}

main() {
    log "Starting Grafana dashboard setup for collectd metrics..."
    
    check_prerequisites
    detect_distro
    install_grafana
    configure_grafana
    configure_firewall
    setup_influxdb_user
    configure_datasource
    import_dashboard
    verify_installation
    
    log "Setup completed! Access Grafana at http://localhost:3000"
}

main "$@"