Implement Grafana alerting with Prometheus and InfluxDB for comprehensive monitoring

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Global variables
PKG_MGR=""
PKG_INSTALL=""
DISTRO=""
GRAFANA_CONFIG=""
PROMETHEUS_CONFIG=""
INFLUXDB_CONFIG=""

# Cleanup function
cleanup() {
    local exit_code=$?
    if [ $exit_code -ne 0 ]; then
        echo -e "${RED}[ERROR] Installation failed. Cleaning up...${NC}"
        systemctl stop grafana-server prometheus influxdb 2>/dev/null || true
        systemctl disable grafana-server prometheus influxdb 2>/dev/null || true
    fi
    exit $exit_code
}

trap cleanup ERR

usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -h, --help          Show this help message"
    echo "  --influxdb-password Password for InfluxDB admin user (default: randomly generated)"
    echo "  --grafana-port      Grafana port (default: 3000)"
    echo "  --prometheus-port   Prometheus port (default: 9090)"
    echo "  --influxdb-port     InfluxDB port (default: 8086)"
    exit 1
}

# Parse arguments
INFLUXDB_PASSWORD=""
GRAFANA_PORT="3000"
PROMETHEUS_PORT="9090"
INFLUXDB_PORT="8086"

while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            usage
            ;;
        --influxdb-password)
            INFLUXDB_PASSWORD="$2"
            shift 2
            ;;
        --grafana-port)
            GRAFANA_PORT="$2"
            shift 2
            ;;
        --prometheus-port)
            PROMETHEUS_PORT="$2"
            shift 2
            ;;
        --influxdb-port)
            INFLUXDB_PORT="$2"
            shift 2
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Generate random password if not provided
if [ -z "$INFLUXDB_PASSWORD" ]; then
    INFLUXDB_PASSWORD=$(openssl rand -base64 32 | tr -d "=+/" | cut -c1-16)
fi

# Check prerequisites
check_prerequisites() {
    echo -e "${BLUE}[1/8] Checking prerequisites...${NC}"
    
    if [[ $EUID -ne 0 ]]; then
        echo -e "${RED}This script must be run as root${NC}"
        exit 1
    fi
    
    if ! command -v wget &> /dev/null; then
        echo -e "${YELLOW}wget not found, will install it${NC}"
    fi
    
    if ! command -v openssl &> /dev/null; then
        echo -e "${RED}openssl is required but not installed${NC}"
        exit 1
    fi
}

# Detect distribution
detect_distro() {
    echo -e "${BLUE}[2/8] Detecting distribution...${NC}"
    
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        DISTRO="$ID"
        case "$ID" in
            ubuntu|debian)
                PKG_MGR="apt"
                PKG_INSTALL="apt install -y"
                GRAFANA_CONFIG="/etc/grafana/grafana.ini"
                PROMETHEUS_CONFIG="/etc/prometheus"
                INFLUXDB_CONFIG="/etc/influxdb"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora)
                PKG_MGR="dnf"
                PKG_INSTALL="dnf install -y"
                GRAFANA_CONFIG="/etc/grafana/grafana.ini"
                PROMETHEUS_CONFIG="/etc/prometheus"
                INFLUXDB_CONFIG="/etc/influxdb"
                ;;
            amzn)
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                GRAFANA_CONFIG="/etc/grafana/grafana.ini"
                PROMETHEUS_CONFIG="/etc/prometheus"
                INFLUXDB_CONFIG="/etc/influxdb"
                ;;
            *)
                echo -e "${RED}Unsupported distribution: $ID${NC}"
                exit 1
                ;;
        esac
        echo -e "${GREEN}Detected: $PRETTY_NAME${NC}"
    else
        echo -e "${RED}Cannot detect distribution${NC}"
        exit 1
    fi
}

# Install Grafana
install_grafana() {
    echo -e "${BLUE}[3/8] Installing Grafana...${NC}"
    
    if [ "$PKG_MGR" = "apt" ]; then
        apt update
        $PKG_INSTALL software-properties-common wget
        wget -q -O - https://packages.grafana.com/gpg.key | apt-key add -
        echo "deb https://packages.grafana.com/oss/deb stable main" > /etc/apt/sources.list.d/grafana.list
        apt update
        $PKG_INSTALL grafana
    else
        $PKG_INSTALL wget
        cat > /etc/yum.repos.d/grafana.repo << EOF
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
        $PKG_INSTALL grafana
    fi
    
    # Configure Grafana
    sed -i "s/;http_port = 3000/http_port = $GRAFANA_PORT/" "$GRAFANA_CONFIG"
    
    systemctl enable grafana-server
    systemctl start grafana-server
}

# Install Prometheus
install_prometheus() {
    echo -e "${BLUE}[4/8] Installing Prometheus...${NC}"
    
    if [ "$PKG_MGR" = "apt" ]; then
        $PKG_INSTALL prometheus
    else
        $PKG_INSTALL prometheus2
        PROMETHEUS_CONFIG="/etc/prometheus2"
    fi
    
    # Configure Prometheus
    mkdir -p "$PROMETHEUS_CONFIG"
    cat > "$PROMETHEUS_CONFIG/prometheus.yml" << EOF
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:$PROMETHEUS_PORT']
  
  - job_name: 'node'
    static_configs:
      - targets: ['localhost:9100']
EOF
    
    chown -R prometheus:prometheus "$PROMETHEUS_CONFIG" 2>/dev/null || true
    chmod 644 "$PROMETHEUS_CONFIG/prometheus.yml"
    
    systemctl enable prometheus
    systemctl start prometheus
}

# Install InfluxDB
install_influxdb() {
    echo -e "${BLUE}[5/8] Installing InfluxDB...${NC}"
    
    if [ "$PKG_MGR" = "apt" ]; then
        wget -qO- https://repos.influxdata.com/influxdb.key | apt-key add -
        echo "deb https://repos.influxdata.com/debian stable main" > /etc/apt/sources.list.d/influxdb.list
        apt update
        $PKG_INSTALL influxdb
    else
        cat > /etc/yum.repos.d/influxdb.repo << EOF
[influxdb]
name = InfluxDB Repository - RHEL
baseurl = https://repos.influxdata.com/rhel/\$releasever/\$basearch/stable
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdb.key
EOF
        $PKG_INSTALL influxdb
    fi
    
    # Configure InfluxDB
    mkdir -p "$INFLUXDB_CONFIG"
    cat > "$INFLUXDB_CONFIG/influxdb.conf" << EOF
[meta]
  dir = "/var/lib/influxdb/meta"

[data]
  dir = "/var/lib/influxdb/data"
  engine = "tsm1"
  wal-dir = "/var/lib/influxdb/wal"

[http]
  enabled = true
  bind-address = ":$INFLUXDB_PORT"
EOF
    
    chown -R influxdb:influxdb "$INFLUXDB_CONFIG" 2>/dev/null || true
    chown -R influxdb:influxdb /var/lib/influxdb 2>/dev/null || true
    chmod 644 "$INFLUXDB_CONFIG/influxdb.conf"
    
    systemctl enable influxdb
    systemctl start influxdb
}

# Configure data sources
configure_datasources() {
    echo -e "${BLUE}[6/8] Configuring Grafana data sources...${NC}"
    
    # Wait for services to be ready
    sleep 5
    
    # Configure Prometheus data source
    curl -s -X POST \
      http://admin:admin@localhost:$GRAFANA_PORT/api/datasources \
      -H "Content-Type: application/json" \
      -d '{
        "name": "Prometheus",
        "type": "prometheus",
        "url": "http://localhost:'$PROMETHEUS_PORT'",
        "access": "proxy",
        "isDefault": true
      }' || echo -e "${YELLOW}Prometheus datasource may already exist${NC}"
    
    # Configure InfluxDB data source
    curl -s -X POST \
      http://admin:admin@localhost:$GRAFANA_PORT/api/datasources \
      -H "Content-Type: application/json" \
      -d '{
        "name": "InfluxDB",
        "type": "influxdb",
        "url": "http://localhost:'$INFLUXDB_PORT'",
        "access": "proxy",
        "database": "monitoring"
      }' || echo -e "${YELLOW}InfluxDB datasource may already exist${NC}"
}

# Configure firewall
configure_firewall() {
    echo -e "${BLUE}[7/8] Configuring firewall...${NC}"
    
    if command -v ufw &> /dev/null; then
        ufw allow $GRAFANA_PORT/tcp
        ufw allow $PROMETHEUS_PORT/tcp
        ufw allow $INFLUXDB_PORT/tcp
    elif command -v firewall-cmd &> /dev/null; then
        firewall-cmd --permanent --add-port=$GRAFANA_PORT/tcp
        firewall-cmd --permanent --add-port=$PROMETHEUS_PORT/tcp
        firewall-cmd --permanent --add-port=$INFLUXDB_PORT/tcp
        firewall-cmd --reload
    fi
}

# Verify installation
verify_installation() {
    echo -e "${BLUE}[8/8] Verifying installation...${NC}"
    
    # Check services
    for service in grafana-server prometheus influxdb; do
        if systemctl is-active --quiet $service; then
            echo -e "${GREEN}✓ $service is running${NC}"
        else
            echo -e "${RED}✗ $service is not running${NC}"
            exit 1
        fi
    done
    
    # Check ports
    for port in $GRAFANA_PORT $PROMETHEUS_PORT $INFLUXDB_PORT; do
        if ss -tlnp | grep -q ":$port "; then
            echo -e "${GREEN}✓ Port $port is open${NC}"
        else
            echo -e "${RED}✗ Port $port is not accessible${NC}"
            exit 1
        fi
    done
}

# Main execution
main() {
    check_prerequisites
    detect_distro
    install_grafana
    install_prometheus
    install_influxdb
    configure_datasources
    configure_firewall
    verify_installation
    
    echo -e "${GREEN}Installation completed successfully!${NC}"
    echo -e "${BLUE}Access Grafana at: http://localhost:$GRAFANA_PORT${NC}"
    echo -e "${BLUE}Default login: admin/admin${NC}"
    echo -e "${BLUE}Prometheus: http://localhost:$PROMETHEUS_PORT${NC}"
    echo -e "${BLUE}InfluxDB: http://localhost:$INFLUXDB_PORT${NC}"
    echo -e "${YELLOW}InfluxDB admin password: $INFLUXDB_PASSWORD${NC}"
}

main "$@"