Configure NGINX log analysis with Loki and Grafana for centralized monitoring

#!/usr/bin/env bash
set -euo pipefail

# NGINX Loki Grafana Monitoring Setup Script
# Configures centralized log monitoring with structured JSON logging

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Variables
LOKI_VERSION="2.9.4"
GRAFANA_VERSION="10.2.2"
LOKI_USER="loki"
PROMTAIL_USER="promtail"

# Usage
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -h, --help     Show this help message"
    echo "Example: $0"
    exit 1
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help) usage ;;
        *) echo -e "${RED}Unknown option: $1${NC}"; usage ;;
    esac
done

# Cleanup function
cleanup() {
    echo -e "${RED}Error occurred. Cleaning up...${NC}"
    systemctl stop loki promtail grafana-server 2>/dev/null || true
}
trap cleanup ERR

# Check prerequisites
check_prerequisites() {
    if [[ $EUID -ne 0 ]]; then
        echo -e "${RED}This script must be run as root${NC}"
        exit 1
    fi
    
    if ! command -v wget >/dev/null 2>&1; then
        echo -e "${RED}wget is required but not installed${NC}"
        exit 1
    fi
    
    if ! command -v unzip >/dev/null 2>&1; then
        echo -e "${RED}unzip is required but not installed${NC}"
        exit 1
    fi
}

# Detect distribution
detect_distro() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian)
                PKG_MGR="apt"
                PKG_INSTALL="apt install -y"
                PKG_UPDATE="apt update && apt upgrade -y"
                NGINX_CONFIG_DIR="/etc/nginx"
                NGINX_SITES_DIR="/etc/nginx/sites-available"
                NGINX_LOG_DIR="/var/log/nginx"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora)
                PKG_MGR="dnf"
                PKG_INSTALL="dnf install -y"
                PKG_UPDATE="dnf update -y"
                NGINX_CONFIG_DIR="/etc/nginx"
                NGINX_SITES_DIR="/etc/nginx/conf.d"
                NGINX_LOG_DIR="/var/log/nginx"
                ;;
            amzn)
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                PKG_UPDATE="yum update -y"
                NGINX_CONFIG_DIR="/etc/nginx"
                NGINX_SITES_DIR="/etc/nginx/conf.d"
                NGINX_LOG_DIR="/var/log/nginx"
                ;;
            *)
                echo -e "${RED}Unsupported distribution: $ID${NC}"
                exit 1
                ;;
        esac
    else
        echo -e "${RED}Cannot detect distribution${NC}"
        exit 1
    fi
}

echo "[1/12] Checking prerequisites..."
check_prerequisites

echo "[2/12] Detecting distribution..."
detect_distro
echo -e "${GREEN}Detected: $PRETTY_NAME${NC}"

echo "[3/12] Updating system packages..."
$PKG_UPDATE

echo "[4/12] Installing required packages..."
$PKG_INSTALL nginx wget unzip curl

echo "[5/12] Configuring NGINX structured logging..."
# Backup original nginx.conf
cp $NGINX_CONFIG_DIR/nginx.conf $NGINX_CONFIG_DIR/nginx.conf.backup

# Create JSON logging format
cat > /tmp/nginx_json_log.conf << 'EOF'
    log_format json_logs escape=json
    '{'
        '"timestamp":"$time_iso8601",'
        '"remote_addr":"$remote_addr",'
        '"remote_user":"$remote_user",'
        '"request":"$request",'
        '"status":"$status",'
        '"body_bytes_sent":"$body_bytes_sent",'
        '"request_time":"$request_time",'
        '"http_referrer":"$http_referer",'
        '"http_user_agent":"$http_user_agent",'
        '"http_x_forwarded_for":"$http_x_forwarded_for",'
        '"upstream_addr":"$upstream_addr",'
        '"upstream_response_time":"$upstream_response_time"'
    '}';
    
    access_log /var/log/nginx/access.log json_logs;
    error_log /var/log/nginx/error.log warn;
EOF

# Add JSON logging to nginx.conf
sed -i '/http {/r /tmp/nginx_json_log.conf' $NGINX_CONFIG_DIR/nginx.conf
rm /tmp/nginx_json_log.conf

# Test and reload NGINX
nginx -t
systemctl enable nginx
systemctl reload nginx

echo "[6/12] Installing Loki..."
cd /tmp
wget -q https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/loki-linux-amd64.zip
unzip -q loki-linux-amd64.zip
mv loki-linux-amd64 /usr/local/bin/loki
chmod 755 /usr/local/bin/loki

echo "[7/12] Configuring Loki..."
useradd --system --no-create-home --shell /bin/false $LOKI_USER || true
mkdir -p /etc/loki /var/lib/loki
chown -R $LOKI_USER:$LOKI_USER /var/lib/loki

cat > /etc/loki/loki.yml << 'EOF'
auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  path_prefix: /var/lib/loki
  storage:
    filesystem:
      chunks_directory: /var/lib/loki/chunks
      rules_directory: /var/lib/loki/rules
  replication_factor: 1
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

limits_config:
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  ingestion_rate_mb: 16
  ingestion_burst_size_mb: 32

chunk_store_config:
  max_look_back_period: 0s

table_manager:
  retention_deletes_enabled: true
  retention_period: 168h
EOF

chown $LOKI_USER:$LOKI_USER /etc/loki/loki.yml
chmod 644 /etc/loki/loki.yml

echo "[8/12] Creating Loki systemd service..."
cat > /etc/systemd/system/loki.service << EOF
[Unit]
Description=Loki log aggregation system
After=network-online.target

[Service]
Type=simple
User=$LOKI_USER
Group=$LOKI_USER
ExecStart=/usr/local/bin/loki -config.file=/etc/loki/loki.yml
Restart=on-failure
RestartSec=5
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable --now loki

echo "[9/12] Installing Promtail..."
wget -q https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/promtail-linux-amd64.zip
unzip -q promtail-linux-amd64.zip
mv promtail-linux-amd64 /usr/local/bin/promtail
chmod 755 /usr/local/bin/promtail

echo "[10/12] Configuring Promtail..."
useradd --system --no-create-home --shell /bin/false $PROMTAIL_USER || true
usermod -a -G adm $PROMTAIL_USER

mkdir -p /etc/promtail
cat > /etc/promtail/promtail.yml << EOF
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://localhost:3100/loki/api/v1/push

scrape_configs:
  - job_name: nginx
    static_configs:
    - targets:
        - localhost
      labels:
        job: nginx
        __path__: /var/log/nginx/access.log
        
  - job_name: nginx-error
    static_configs:
    - targets:
        - localhost
      labels:
        job: nginx-error
        __path__: /var/log/nginx/error.log
EOF

chown $PROMTAIL_USER:$PROMTAIL_USER /etc/promtail/promtail.yml
chmod 644 /etc/promtail/promtail.yml

cat > /etc/systemd/system/promtail.service << EOF
[Unit]
Description=Promtail service
After=network.target

[Service]
Type=simple
User=$PROMTAIL_USER
ExecStart=/usr/local/bin/promtail -config.file /etc/promtail/promtail.yml
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable --now promtail

echo "[11/12] Installing and configuring Grafana..."
if [[ "$PKG_MGR" == "apt" ]]; then
    wget -q -O - https://packages.grafana.com/gpg.key | apt-key add -
    echo "deb https://packages.grafana.com/oss/deb stable main" > /etc/apt/sources.list.d/grafana.list
    apt update
    $PKG_INSTALL grafana
else
    cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
    $PKG_INSTALL grafana
fi

systemctl enable --now grafana-server

echo "[12/12] Verifying installation..."
sleep 5

# Check services
for service in nginx loki promtail grafana-server; do
    if systemctl is-active --quiet $service; then
        echo -e "${GREEN}✓ $service is running${NC}"
    else
        echo -e "${RED}✗ $service is not running${NC}"
    fi
done

# Check ports
for port in 80 3100 9080 3000; do
    if netstat -tuln 2>/dev/null | grep -q ":$port " || ss -tuln 2>/dev/null | grep -q ":$port "; then
        echo -e "${GREEN}✓ Port $port is listening${NC}"
    else
        echo -e "${YELLOW}! Port $port may not be listening${NC}"
    fi
done

echo -e "\n${GREEN}Installation completed successfully!${NC}"
echo -e "\nAccess points:"
echo -e "• Grafana: http://localhost:3000 (admin/admin)"
echo -e "• Loki: http://localhost:3100"
echo -e "• Promtail: http://localhost:9080"
echo -e "\nNext steps:"
echo -e "1. Configure Grafana data source: http://localhost:3100"
echo -e "2. Import NGINX dashboard or create custom queries"
echo -e "3. Generate some traffic to see logs flowing"