Set up NGINX log analysis and monitoring with Prometheus and Grafana dashboards

Update system packages

Start with fresh package repositories to ensure you get the latest versions of all monitoring tools.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install required packages

Install NGINX if not already present, plus tools for downloading and managing the monitoring stack.

sudo apt install -y nginx curl wget tar

sudo dnf install -y nginx curl wget tar

Configure NGINX status module

Enable the NGINX status module to expose basic server metrics that Prometheus can scrape. Add a status location block to your main server configuration.

server {
    listen 80 default_server;
    listen [::]:80 default_server;
    
    # Your existing configuration
    root /var/www/html;
    index index.html index.htm index.nginx-debian.html;
    server_name _;
    
    location / {
        try_files $uri $uri/ =404;
    }
    
    # Status endpoint for monitoring
    location /nginx_status {
        stub_status on;
        access_log off;
        allow 127.0.0.1;
        allow 10.0.0.0/8;
        allow 172.16.0.0/12;
        allow 192.168.0.0/16;
        deny all;
    }
}

Configure structured NGINX logging

Replace the default NGINX log format with a structured JSON format that includes response times, upstream information, and request details for better parsing.

http {
    # JSON log format for better parsing
    log_format json_analytics escape=json '
    {
        "time": "$time_iso8601",
        "remote_addr": "$remote_addr",
        "remote_user": "$remote_user",
        "request": "$request",
        "status": $status,
        "body_bytes_sent": $body_bytes_sent,
        "request_time": $request_time,
        "upstream_response_time": "$upstream_response_time",
        "http_referrer": "$http_referer",
        "http_user_agent": "$http_user_agent",
        "http_x_forwarded_for": "$http_x_forwarded_for",
        "server_name": "$server_name"
    }';
    
    # Apply to access logs
    access_log /var/log/nginx/access.log json_analytics;
    error_log /var/log/nginx/error.log warn;
    
    # Your existing configuration
    include /etc/nginx/mime.types;
    default_type application/octet-stream;
    
    # Performance settings
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    
    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

Test and reload NGINX

Verify the configuration syntax and reload NGINX to apply the new logging format and status endpoint.

sudo nginx -t
sudo systemctl reload nginx
sudo systemctl status nginx

Install NGINX Prometheus Exporter

Download and install the official NGINX Prometheus exporter that will scrape the status endpoint and convert metrics to Prometheus format.

cd /tmp
wget https://github.com/nginxinc/nginx-prometheus-exporter/releases/download/v1.1.0/nginx-prometheus-exporter_1.1.0_linux_amd64.tar.gz
tar -xzf nginx-prometheus-exporter_1.1.0_linux_amd64.tar.gz
sudo mv nginx-prometheus-exporter /usr/local/bin/
sudo chmod +x /usr/local/bin/nginx-prometheus-exporter

Create nginx-exporter systemd service

Create a systemd service file to manage the NGINX exporter as a proper system service with automatic startup and logging.

[Unit]
Description=NGINX Prometheus Exporter
After=network.target nginx.service
Requires=nginx.service

[Service]
Type=simple
User=nginx
Group=nginx
ExecStart=/usr/local/bin/nginx-prometheus-exporter -nginx.scrape-uri=http://127.0.0.1/nginx_status
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

Install Prometheus log exporter

Install a log parsing exporter to convert NGINX JSON logs into Prometheus metrics for request rates, response times, and status codes.

cd /tmp
wget https://github.com/martin-helmich/prometheus-nginxlog-exporter/releases/download/v1.10.0/prometheus-nginxlog-exporter_1.10.0_linux_amd64.tar.gz
tar -xzf prometheus-nginxlog-exporter_1.10.0_linux_amd64.tar.gz
sudo mv prometheus-nginxlog-exporter /usr/local/bin/
sudo chmod +x /usr/local/bin/prometheus-nginxlog-exporter

Configure log exporter

Create a configuration file that defines how to parse NGINX logs and what metrics to extract from them.

listen:
  port: 4040
  address: "0.0.0.0"

consul:
  enable: false

namespaces:
  - name: nginx
    format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
    source:
      files:
        - /var/log/nginx/access.log
    labels:
      service: "nginx"
      environment: "production"
    histogram_buckets: [.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10]
    print_log: false
    
    metrics:
      - name: response_time_seconds
        type: histogram
        help: "Response time in seconds"
        match: '$request_time'
        value: '$request_time'
        labels:
          method: '$request_method'
          status: '$status'
          
      - name: bytes_sent_total
        type: counter
        help: "Total bytes sent"
        match: '$body_bytes_sent'
        value: '$body_bytes_sent'
        labels:
          method: '$request_method'
          status: '$status'
          
      - name: requests_total
        type: counter
        help: "Total number of requests"
        match: '.*'
        labels:
          method: '$request_method'
          status: '$status'

Create log exporter systemd service

Set up the log exporter as a systemd service with proper permissions to read NGINX log files.

[Unit]
Description=Prometheus NGINX Log Exporter
After=network.target

[Service]
Type=simple
User=root
Group=adm
ExecStart=/usr/local/bin/prometheus-nginxlog-exporter -config-file /etc/prometheus-nginxlog-exporter.yml
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

Start the exporters

Enable and start both Prometheus exporters to begin collecting NGINX metrics.

sudo systemctl daemon-reload
sudo systemctl enable --now nginx-exporter
sudo systemctl enable --now prometheus-nginxlog-exporter
sudo systemctl status nginx-exporter
sudo systemctl status prometheus-nginxlog-exporter

Install Prometheus

Download and install Prometheus server to scrape metrics from the NGINX exporters and store time-series data.

cd /tmp
wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prometheus-2.45.0.linux-amd64.tar.gz
tar -xzf prometheus-2.45.0.linux-amd64.tar.gz
sudo mv prometheus-2.45.0.linux-amd64 /opt/prometheus
sudo useradd --no-create-home --shell /bin/false prometheus
sudo mkdir -p /etc/prometheus /var/lib/prometheus
sudo chown prometheus:prometheus /etc/prometheus /var/lib/prometheus
sudo chown -R prometheus:prometheus /opt/prometheus

Configure Prometheus

Create a Prometheus configuration file that defines scraping intervals and targets for the NGINX exporters.

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "nginx_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'nginx-exporter'
    static_configs:
      - targets: ['localhost:9113']
    scrape_interval: 10s
    metrics_path: /metrics
    
  - job_name: 'nginx-log-exporter'
    static_configs:
      - targets: ['localhost:4040']
    scrape_interval: 10s
    metrics_path: /metrics

Create Prometheus alerting rules

Define alerting rules for common NGINX issues like high error rates, slow response times, and service availability.

groups:
  - name: nginx
    rules:
      - alert: NginxDown
        expr: up{job="nginx-exporter"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "NGINX is down"
          description: "NGINX has been down for more than 1 minute"

      - alert: NginxHighErrorRate
        expr: rate(nginx_requests_total{status=~"5.."}[5m]) / rate(nginx_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High NGINX 5xx error rate"
          description: "NGINX 5xx error rate is {{ $value | humanizePercentage }} for 5 minutes"

      - alert: NginxHighLatency
        expr: histogram_quantile(0.95, rate(nginx_response_time_seconds_bucket[5m])) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High NGINX response time"
          description: "95th percentile response time is {{ $value }}s for 5 minutes"

      - alert: NginxHighConnections
        expr: nginx_connections_active > 1000
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High number of NGINX connections"
          description: "NGINX has {{ $value }} active connections"

Set Prometheus permissions

Ensure the prometheus user owns all configuration files and the data directory.

sudo chown prometheus:prometheus /etc/prometheus/prometheus.yml
sudo chown prometheus:prometheus /etc/prometheus/nginx_rules.yml

Create Prometheus systemd service

Set up Prometheus as a systemd service with appropriate resource limits and security settings.

[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/opt/prometheus/prometheus \
    --config.file /etc/prometheus/prometheus.yml \
    --storage.tsdb.path /var/lib/prometheus/ \
    --web.console.templates=/opt/prometheus/consoles \
    --web.console.libraries=/opt/prometheus/console_libraries \
    --web.listen-address=0.0.0.0:9090 \
    --web.enable-lifecycle
Restart=always

[Install]
WantedBy=multi-user.target

Install Grafana

Install Grafana to create visual dashboards and set up alerting based on the Prometheus metrics.

wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -
echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
sudo apt update
sudo apt install -y grafana

sudo tee /etc/yum.repos.d/grafana.repo<