Set up NGINX web application firewall with ModSecurity 3 and OWASP Core Rule Set

Update system packages

Update your package manager to ensure you get the latest security patches and package versions.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install dependencies and build tools

Install the required packages for compiling ModSecurity and the NGINX connector module.

sudo apt install -y build-essential git autoconf automake libtool pkg-config
sudo apt install -y libcurl4-openssl-dev liblmdb-dev libpcre3-dev libyajl-dev
sudo apt install -y nginx-core nginx-dev libmaxminddb-dev

sudo dnf groupinstall -y "Development Tools"
sudo dnf install -y git autoconf automake libtool pkgconfig
sudo dnf install -y libcurl-devel lmdb-devel pcre-devel yajl-devel
sudo dnf install -y nginx nginx-devel libmaxminddb-devel

Download and compile ModSecurity 3

Clone the ModSecurity repository and compile the library from source for optimal performance and latest features.

cd /opt
sudo git clone --depth 1 -b v3.0.12 --single-branch https://github.com/owasp-modsecurity/ModSecurity
sudo chown -R $USER:$USER ModSecurity
cd ModSecurity
git submodule init
git submodule update

Configure and compile ModSecurity with optimized settings.

./build.sh
./configure --with-lmdb
make -j$(nproc)
sudo make install

Build the NGINX ModSecurity connector

Download and compile the ModSecurity-nginx connector to integrate ModSecurity with NGINX.

cd /opt
sudo git clone --depth 1 https://github.com/owasp-modsecurity/ModSecurity-nginx.git
sudo chown -R $USER:$USER ModSecurity-nginx

Get NGINX source code to build the dynamic module.

nginx -v
NGINX_VERSION=$(nginx -v 2>&1 | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+')
wget http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz
tar -xzf nginx-${NGINX_VERSION}.tar.gz
cd nginx-${NGINX_VERSION}

Configure NGINX with the ModSecurity module and compile.

./configure --with-compat --add-dynamic-module=../ModSecurity-nginx
make modules
sudo cp objs/ngx_http_modsecurity_module.so /etc/nginx/modules/

Configure NGINX to load ModSecurity

Enable the ModSecurity module in NGINX configuration.

load_module modules/ngx_http_modsecurity_module.so;

events {
    worker_connections 1024;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    
    # Security headers
    add_header X-Frame-Options DENY always;
    add_header X-Content-Type-Options nosniff always;
    add_header X-XSS-Protection "1; mode=block" always;
    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
    
    # Logging format with security info
    log_format security '$remote_addr - $remote_user [$time_local] '
                       '"$request" $status $body_bytes_sent '
                       '"$http_referer" "$http_user_agent" '
                       'rt=$request_time uct="$upstream_connect_time" '
                       'uht="$upstream_header_time" urt="$upstream_response_time"';
    
    access_log /var/log/nginx/access.log security;
    error_log /var/log/nginx/error.log warn;
    
    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

Download and configure OWASP Core Rule Set

Download the latest OWASP CRS rules for comprehensive protection against web application attacks.

sudo mkdir -p /etc/nginx/modsec
cd /etc/nginx/modsec
sudo wget https://github.com/coreruleset/coreruleset/archive/refs/tags/v4.0.0.tar.gz
sudo tar -xzf v4.0.0.tar.gz
sudo mv coreruleset-4.0.0 crs
sudo chown -R www-data:www-data /etc/nginx/modsec

Configure the main ModSecurity configuration file.

# ModSecurity configuration
SecRuleEngine On
SecRequestBodyAccess On
SecResponseBodyAccess Off
SecRequestBodyLimit 13107200
SecRequestBodyNoFilesLimit 131072
SecRequestBodyLimitAction Reject
SecRequestBodyJsonDepthLimit 512
SecRequestBodyInMemoryLimit 131072
SecDataDir /tmp/
SecTmpDir /tmp/

# Upload directory
SecUploadDir /tmp/
SecUploadKeepFiles RelevantOnly
SecUploadFileMode 0600

# Debug and audit logging
SecDebugLog /var/log/nginx/modsec_debug.log
SecDebugLogLevel 0
SecAuditEngine RelevantOnly
SecAuditLogRelevantStatus "^(?:5|4(?!04))"
SecAuditLogParts ABDEFHIJZ
SecAuditLogType Serial
SecAuditLog /var/log/nginx/modsec_audit.log

# Request/Response handling
SecArgumentSeparator &
SecCookieFormat 0
SecUnicodeMapFile unicode.mapping 20127

# Rule compatibility
SecCollectionTimeout 600
SecHttpBlKey whdkfieyhtnf

# Include OWASP CRS
Include /etc/nginx/modsec/crs/crs-setup.conf
Include /etc/nginx/modsec/crs/rules/*.conf

Configure CRS setup and exclusions

Create a custom CRS configuration with exclusions for common false positives.

# Custom CRS configuration and exclusions

# Set paranoia level (1-4, higher = more strict)
SecAction "id:900000,phase:1,nolog,pass,t:none,setvar:tx.paranoia_level=2"

# Set anomaly scoring thresholds
SecAction "id:900110,phase:1,nolog,pass,t:none,setvar:tx.inbound_anomaly_score_threshold=5"
SecAction "id:900111,phase:1,nolog,pass,t:none,setvar:tx.outbound_anomaly_score_threshold=4"

# Allowed request methods
SecAction "id:900200,phase:1,nolog,pass,t:none,setvar:'tx.allowed_methods=GET HEAD POST OPTIONS PUT PATCH DELETE'"

# Allowed file extensions
SecAction "id:900220,phase:1,nolog,pass,t:none,setvar:'tx.allowed_request_content_type=|application/x-www-form-urlencoded| |multipart/form-data| |multipart/related| |text/xml| |application/xml| |application/soap+xml| |application/json| |application/cloudevents+json| |application/cloudevents-batch+json|'"

# Common exclusions for legitimate traffic
# WordPress admin exclusions
SecRule REQUEST_FILENAME "@beginsWith /wp-admin/" "id:1001,phase:1,pass,nolog,ctl:ruleRemoveById=942100"
SecRule REQUEST_FILENAME "@beginsWith /wp-admin/" "id:1002,phase:1,pass,nolog,ctl:ruleRemoveById=941160"

# API endpoint exclusions
SecRule REQUEST_FILENAME "@beginsWith /api/" "id:1003,phase:1,pass,nolog,ctl:ruleRemoveById=942100"
SecRule REQUEST_FILENAME "@beginsWith /api/" "id:1004,phase:1,pass,nolog,ctl:ruleRemoveById=920230"

# Upload form exclusions
SecRule REQUEST_FILENAME "@contains upload" "id:1005,phase:1,pass,nolog,ctl:ruleRemoveById=200003"

# Rate limiting rules
SecRule IP:BF_COUNTER "@gt 20" "id:1100,phase:1,deny,status:429,msg:'Rate limit exceeded',logdata:'Rate limit exceeded for IP %{REMOTE_ADDR}'"
SecAction "id:1101,phase:5,pass,nolog,setvar:IP.BF_COUNTER=+1,expirevar:IP.BF_COUNTER=300"

Create virtual host with ModSecurity

Configure an NGINX virtual host with ModSecurity protection enabled.

server {
    listen 80;
    server_name example.com www.example.com;
    
    # Enable ModSecurity
    modsecurity on;
    modsecurity_rules_file /etc/nginx/modsec/main.conf;
    
    # Additional security headers
    add_header X-Frame-Options DENY always;
    add_header X-Content-Type-Options nosniff always;
    add_header X-XSS-Protection "1; mode=block" always;
    add_header Referrer-Policy "strict-origin-when-cross-origin" always;
    
    # Rate limiting
    limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;
    limit_req_zone $binary_remote_addr zone=login:10m rate=1r/s;
    
    # Document root
    root /var/www/html;
    index index.html index.php;
    
    # Main location
    location / {
        try_files $uri $uri/ =404;
        
        # Apply rate limiting
        limit_req zone=api burst=20 nodelay;
    }
    
    # API endpoints with stricter limiting
    location /api/ {
        limit_req zone=api burst=5 nodelay;
        proxy_pass http://backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }
    
    # Login endpoints
    location /login {
        limit_req zone=login burst=3 nodelay;
        proxy_pass http://backend;
    }
    
    # Block common attack patterns
    location ~* \.(sql|bak|old|backup)$ {
        deny all;
        return 404;
    }
    
    # Security logging
    access_log /var/log/nginx/waf_access.log security;
    error_log /var/log/nginx/waf_error.log;
}

# Backend upstream
upstream backend {
    server 127.0.0.1:8080;
    keepalive 32;
}

Enable the site and create necessary directories.

sudo ln -s /etc/nginx/sites-available/waf-protected /etc/nginx/sites-enabled/
sudo mkdir -p /var/www/html
sudo chown -R www-data:www-data /var/www/html
sudo chmod 755 /var/www/html

Configure log rotation

Set up log rotation for ModSecurity and NGINX logs to prevent disk space issues.

/var/log/nginx/modsec_*.log {
    daily
    missingok
    rotate 14
    compress
    delaycompress
    notifempty
    create 0640 www-data adm
    sharedscripts
    prerotate
        if [ -d /etc/logrotate.d/httpd-prerotate ]; then \
            run-parts /etc/logrotate.d/httpd-prerotate; \
        fi 
    endscript
    postrotate
        invoke-rc.d nginx reload >/dev/null 2>&1
    endscript
}

/var/log/nginx/waf_*.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    create 0640 www-data adm
    sharedscripts
    postrotate
        invoke-rc.d nginx reload >/dev/null 2>&1
    endscript
}

Install and configure fail2ban integration

Install fail2ban to automatically block IPs that trigger ModSecurity rules repeatedly.

sudo apt install -y fail2ban

sudo dnf install -y epel-release
sudo dnf install -y fail2ban

Create a custom fail2ban filter for ModSecurity.

[Definition]
failregex = ^.*\[.*\] ModSecurity: Access denied with code 403.*\[client *$
            ^.*\[.*\] ModSecurity: Warning.*\[client *$
            ^

Configure fail2ban jail for ModSecurity violations.

[nginx-modsecurity]
enabled = true
port = http,https
filter = nginx-modsecurity
logpath = /var/log/nginx/modsec_audit.log
          /var/log/nginx/error.log
maxretry = 3
bantime = 3600
findtime = 600
action = iptables-multiport[name=nginx-modsecurity, port="http,https", protocol=tcp]
         sendmail-whois[name=nginx-modsecurity, dest=admin@example.com]

Start and enable services

Enable and start NGINX and fail2ban services with proper startup configuration.

sudo nginx -t
sudo systemctl enable nginx fail2ban
sudo systemctl restart nginx fail2ban
sudo systemctl status nginx fail2ban

Set up log monitoring script

Create a monitoring script to track ModSecurity activity and send alerts.

#!/bin/bash

# ModSecurity monitoring script
LOGFILE="/var/log/nginx/modsec_audit.log"
ALERT_EMAIL="admin@example.com"
LAST_CHECK="/tmp/modsec_last_check"

# Create last check file if it doesn't exist
if [ ! -f "$LAST_CHECK" ]; then
    echo $(date +%s) > "$LAST_CHECK"
fi

LAST_TIME=$(cat "$LAST_CHECK")
CURRENT_TIME=$(date +%s)

# Find new attacks since last check
NEW_ATTACKS=$(awk -v start="$LAST_TIME" '
    /ModSecurity: Access denied/ {
        cmd="date -d \"" $4 " " $5 "\" +%s"
        cmd | getline timestamp
        close(cmd)
        if (timestamp > start) {
            attacks++
            print $0
        }
    }
    END { print "Total new attacks:" attacks+0 }
' "$LOGFILE")

ATTACK_COUNT=$(echo "$NEW_ATTACKS" | tail -n1 | grep -o '[0-9]\+' || echo "0")

# Send alert if attacks detected
if [ "$ATTACK_COUNT" -gt 0 ]; then
    echo "ModSecurity detected $ATTACK_COUNT new attacks" | \
    mail -s "WAF Alert: $ATTACK_COUNT attacks blocked" "$ALERT_EMAIL"
fi

# Update last check time
echo "$CURRENT_TIME" > "$LAST_CHECK"

Make the script executable and set up a cron job.

sudo chmod +x /opt/modsec-monitor.sh
sudo chown root:root /opt/modsec-monitor.sh

# Add to crontab for monitoring every 15 minutes
echo "*/15 * * * * /opt/modsec-monitor.sh" | sudo crontab -

Configure Prometheus metrics (optional)

If you use Prometheus monitoring, create a metrics exporter for ModSecurity.

#!/usr/bin/env python3

import re
import time
from prometheus_client import start_http_server, Counter, Histogram
from prometheus_client.core import CollectorRegistry

# Metrics
registry = CollectorRegistry()
attacks_total = Counter('modsecurity_attacks_total', 
                       'Total number of blocked attacks',
                       ['rule_id', 'severity'], registry=registry)
response_time = Histogram('modsecurity_response_time_seconds',
                         'Response time for requests', registry=registry)

def parse_modsec_log():
    """Parse ModSecurity audit log for metrics"""
    try:
        with open('/var/log/nginx/modsec_audit.log', 'r') as f:
            # Read new lines since last position
            for line in f:
                if 'Access denied' in line:
                    # Extract rule ID and severity
                    rule_match = re.search(r'\[id "(\d+)"\]', line)
                    severity_match = re.search(r'\[severity "(\w+)"\]', line)
                    
                    rule_id = rule_match.group(1) if rule_match else 'unknown'
                    severity = severity_match.group(1) if severity_match else 'unknown'
                    
                    attacks_total.labels(rule_id=rule_id, severity=severity).inc()
    except FileNotFoundError:
        pass

if __name__ == '__main__':
    start_http_server(9113, registry=registry)
    
    while True:
        parse_modsec_log()
        time.sleep(30)

Install Python dependencies and create a systemd service.

sudo apt install -y python3-pip
sudo pip3 install prometheus_client

sudo dnf install -y python3-pip
sudo pip3 install prometheus_client

[Unit]
Description=ModSecurity Prometheus Exporter
After=network.target

[Service]
Type=simple
User=www-data
Group=www-data
ExecStart=/usr/bin/python3 /opt/modsec-prometheus.py
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

sudo systemctl daemon-reload
sudo systemctl enable --now modsec-exporter

Optimize ModSecurity performance

Adjust ModSecurity settings for better performance in high-traffic environments.

# Performance optimizations
SecRequestBodyInMemoryLimit 262144
SecRequestBodyLimit 52428800
SecRequestBodyNoFilesLimit 262144

# Reduce rule processing overhead
SecRuleEngine DetectionOnly
SecAuditEngine Off
SecDebugLogLevel 0

# Optimize rule matching
SecPcreMatchLimit 100000
SecPcreMatchLimitRecursion 100000

# Collection timeout optimization
SecCollectionTimeout 3600

Configure response caching

Enable response caching for non-sensitive content to reduce server load.

# Cache configuration
proxy_cache_path /var/cache/nginx/waf levels=1:2 keys_zone=waf_cache:100m