Update system packages

Start by updating your package manager to ensure you get the latest security patches and package versions.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install Nginx and development tools

Install Nginx web server along with development packages needed to compile ModSecurity from source.

sudo apt install -y nginx build-essential libpcre3-dev libssl-dev zlib1g-dev libxml2-dev libgeoip-dev liblmdb-dev libyajl-dev libcurl4-openssl-dev libpcre2-dev doxygen git

sudo dnf install -y nginx gcc gcc-c++ make pcre-devel openssl-devel zlib-devel libxml2-devel GeoIP-devel lmdb-devel yajl-devel libcurl-devel pcre2-devel doxygen git

Download and compile ModSecurity 3

Clone the ModSecurity repository and compile the library with optimizations for production use.

cd /tmp
git clone --depth 1 -b v3/master --single-branch https://github.com/SpiderLabs/ModSecurity
cd ModSecurity
git submodule init
git submodule update

Configure and compile ModSecurity with all required features enabled.

./build.sh
./configure --enable-pcre2 --enable-static --disable-shared
make -j$(nproc)
sudo make install

Download and compile ModSecurity-nginx connector

The connector module integrates ModSecurity 3 with Nginx using the dynamic module system.

cd /tmp
git clone --depth 1 https://github.com/SpiderLabs/ModSecurity-nginx.git

Get the Nginx version and download matching source code for module compilation.

nginx -v
NGINX_VERSION=$(nginx -v 2>&1 | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+')
wget http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz
tar zxvf nginx-${NGINX_VERSION}.tar.gz

Compile the ModSecurity-nginx module as a dynamic module.

cd nginx-${NGINX_VERSION}
./configure --with-compat --add-dynamic-module=../ModSecurity-nginx
make modules
sudo cp objs/ngx_http_modsecurity_module.so /etc/nginx/modules/

Configure Nginx to load ModSecurity module

Add the ModSecurity module to Nginx main configuration and verify it loads correctly.

load_module modules/ngx_http_modsecurity_module.so;

user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

events {
    worker_connections 768;
    use epoll;
    multi_accept on;
}

http {
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    server_tokens off;

    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    # Logging Configuration
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                   '$status $body_bytes_sent "$http_referer" '
                   '"$http_user_agent" "$http_x_forwarded_for"';

    access_log /var/log/nginx/access.log main;
    error_log /var/log/nginx/error.log warn;

    # Gzip Settings
    gzip on;
    gzip_vary on;
    gzip_proxied any;
    gzip_comp_level 6;
    gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml+rss application/json;

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

Download OWASP Core Rule Set

The OWASP CRS provides comprehensive protection rules against common web application attacks.

sudo mkdir -p /etc/nginx/modsec
cd /tmp
wget https://github.com/coreruleset/coreruleset/archive/v4.0.0.tar.gz
tar -xzf v4.0.0.tar.gz
sudo cp -R coreruleset-4.0.0/* /etc/nginx/modsec/
sudo mv /etc/nginx/modsec/crs-setup.conf.example /etc/nginx/modsec/crs-setup.conf

Create ModSecurity main configuration

Configure ModSecurity with recommended settings for production use including audit logging and rule engine.

# ModSecurity Core Rules Set configuration file
Enable ModSecurity, attaching it to every transaction
SecRuleEngine On

Request body handling
SecRequestBodyAccess On
SecRequestBodyLimit 13107200
SecRequestBodyNoFilesLimit 131072
SecRequestBodyLimitAction Reject
SecRequestBodyJsonDepthLimit 512
SecRequestBodyInMemoryLimit 131072

Response body handling
SecResponseBodyAccess On
SecResponseBodyMimeType text/plain text/html text/xml
SecResponseBodyLimit 524288
SecResponseBodyLimitAction ProcessPartial

Filesystem configuration
SecTmpDir /tmp/
SecDataDir /tmp/

Debug log
SecDebugLog /var/log/nginx/modsec_debug.log
SecDebugLogLevel 0

Audit log
SecAuditEngine RelevantOnly
SecAuditLogRelevantStatus "^(?:5|4(?!04))"
SecAuditLogParts ABIJDEFHZ
SecAuditLogType Serial
SecAuditLog /var/log/nginx/modsec_audit.log

Upload handling
SecUploadDir /tmp/
SecUploadKeepFiles Off

Generic attack detection rules
SecRule ARGS "@detectSQLi" \
    "id:1001,\
    phase:2,\
    block,\
    msg:'SQL Injection Attack Detected',\
    logdata:'Matched Data: %{MATCHED_VAR} found within %{MATCHED_VAR_NAME}',\
    tag:'attack-sqli',\
    severity:'CRITICAL',\
    setvar:'tx.anomaly_score=+%{tx.critical_anomaly_score}'"

SecRule ARGS "@detectXSS" \
    "id:1002,\
    phase:2,\
    block,\
    msg:'XSS Attack Detected',\
    logdata:'Matched Data: %{MATCHED_VAR} found within %{MATCHED_VAR_NAME}',\
    tag:'attack-xss',\
    severity:'CRITICAL',\
    setvar:'tx.anomaly_score=+%{tx.critical_anomaly_score}'"

Collaborative detection variables
SecAction \
    "id:900000,\
    phase:1,\
    nolog,\
    pass,\
    t:none,\
    setvar:tx.critical_anomaly_score=5,\
    setvar:tx.error_anomaly_score=4,\
    setvar:tx.warning_anomaly_score=3,\
    setvar:tx.notice_anomaly_score=2,\
    setvar:tx.anomaly_score_threshold=5"

Blocking based on anomaly scores
SecRule TX:ANOMALY_SCORE "@ge %{tx.anomaly_score_threshold}" \
    "id:949110,\
    phase:2,\
    deny,\
    status:403,\
    msg:'Inbound Anomaly Score Exceeded (Total Score: %{TX.ANOMALY_SCORE})',\
    tag:'application-multi',\
    tag:'language-multi',\
    tag:'platform-multi',\
    tag:'attack-generic'"

Include /etc/nginx/modsec/crs-setup.conf
Include /etc/nginx/modsec/rules/*.conf

Create ModSecurity include file

Create a simple include file that enables ModSecurity for virtual hosts.

Include /etc/nginx/modsec/modsecurity.conf

Configure virtual host with ModSecurity

Create a sample virtual host configuration that demonstrates ModSecurity integration with proper security headers.

server {
    listen 80;
    listen [::]:80;
    server_name example.com www.example.com;
    root /var/www/html;
    index index.html index.htm index.nginx-debian.html;

    # Enable ModSecurity
    modsecurity on;
    modsecurity_rules_file /etc/nginx/modsec/main.conf;

    # Security headers
    add_header X-Frame-Options "SAMEORIGIN" always;
    add_header X-Content-Type-Options "nosniff" always;
    add_header X-XSS-Protection "1; mode=block" always;
    add_header Referrer-Policy "no-referrer-when-downgrade" always;
    add_header Content-Security-Policy "default-src 'self' http: https: data: blob: 'unsafe-inline'" always;
    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;

    # Rate limiting zone
    limit_req_zone $binary_remote_addr zone=login:10m rate=1r/s;
    limit_req_zone $binary_remote_addr zone=global:10m rate=10r/s;

    location / {
        limit_req zone=global burst=20 nodelay;
        try_files $uri $uri/ =404;
    }

    location /login {
        limit_req zone=login burst=5 nodelay;
        try_files $uri $uri/ =404;
    }

    # Deny access to sensitive files
    location ~ /\. {
        deny all;
        access_log off;
        log_not_found off;
    }

    location ~ \.(htaccess|htpasswd|ini|log|sh|sql|conf)$ {
        deny all;
        access_log off;
        log_not_found off;
    }

    # Cache static assets
    location ~* \.(jpg|jpeg|png|gif|ico|css|js)$ {
        expires 1y;
        add_header Cache-Control "public, immutable";
        access_log off;
    }

    # Error pages
    error_page 403 /403.html;
    error_page 404 /404.html;
    error_page 500 502 503 504 /50x.html;

    location = /50x.html {
        root /var/www/html;
    }
}

Create log directories and set permissions

Set up proper log directories with correct ownership for the Nginx user to write ModSecurity logs.

sudo mkdir -p /var/log/nginx
sudo chown www-data:www-data /var/log/nginx
sudo chmod 755 /var/log/nginx
sudo touch /var/log/nginx/modsec_debug.log /var/log/nginx/modsec_audit.log
sudo chown www-data:www-data /var/log/nginx/modsec_*.log
sudo chmod 644 /var/log/nginx/modsec_*.log

Test configuration and restart Nginx

Verify the Nginx configuration is valid and restart the service to apply ModSecurity settings.

sudo nginx -t
sudo systemctl restart nginx
sudo systemctl enable nginx

Configure log rotation

Set up automatic log rotation for ModSecurity logs to prevent disk space issues in production.

/var/log/nginx/modsec_*.log {
    daily
    missingok
    rotate 52
    compress
    delaycompress
    notifempty
    create 644 www-data www-data
    postrotate
        if [ -f /var/run/nginx.pid ]; then
            kill -USR1 cat /var/run/nginx.pid
        fi
    endscript
}

Create test web pages

Create simple test pages to verify ModSecurity is blocking malicious requests.

    


    
Web Application Firewall Active
    
This server is protected by ModSecurity 3 with OWASP Core Rule Set.
    

        
        
    

Data received: " . htmlspecialchars($_POST['data']) . "";
} else {
    echo "
No data submitted
";
}
?>

Test SQL injection protection

Attempt a basic SQL injection attack to verify ModSecurity blocks malicious requests.

curl -X POST -d "data=' OR '1'='1" http://localhost/test.php

You should see a 403 Forbidden response indicating the WAF blocked the request.

Test XSS protection

Test cross-site scripting protection with a simple XSS payload.

curl -X POST -d "data=" http://localhost/test.php

This should also result in a 403 Forbidden response from ModSecurity.

Tune ModSecurity for performance

Optimize ModSecurity settings for production workloads by adjusting buffer sizes and processing limits.

# Performance tuning for ModSecurity
SecRequestBodyInMemoryLimit 262144
SecRequestBodyLimit 52428800
SecResponseBodyLimit 1048576

Reduce rule processing overhead
SecRuleEngine DetectionOnly
SecAuditEngine Off

Optimize rule exclusions for known good traffic
SecRuleRemoveById 920100 # Host header missing
SecRuleRemoveById 920230 # Multiple URL encoding detected

Custom performance rules
SecRule REQUEST_FILENAME "@beginsWith /static/" \
    "id:1100,\
    phase:1,\
    pass,\
    nolog,\
    ctl:ruleEngine=off"

SecRule REQUEST_FILENAME "@beginsWith /api/health" \
    "id:1101,\
    phase:1,\
    pass,\
    nolog,\
    ctl:ruleEngine=off"

Include this performance configuration in your main ModSecurity config.

Include /etc/nginx/modsec/modsecurity.conf
Include /etc/nginx/modsec/performance.conf

Configure Nginx worker optimization

Optimize Nginx worker processes and connections for better performance with ModSecurity enabled.

worker_processes auto;
worker_cpu_affinity auto;
worker_rlimit_nofile 65535;

events {
    worker_connections 2048;
    use epoll;
    multi_accept on;
    accept_mutex off;
}

http {
    # Connection and timeout optimization
    keepalive_timeout 30;
    keepalive_requests 100;
    send_timeout 10;
    client_body_timeout 12;
    client_header_timeout 12;
    
    # Buffer optimization for ModSecurity
    client_body_buffer_size 16k;
    client_header_buffer_size 4k;
    client_max_body_size 50m;
    large_client_header_buffers 4 8k;
}

Set up real-time monitoring

Monitor ModSecurity activity and blocked requests using log analysis tools.

# View recent blocked requests
sudo tail -f /var/log/nginx/modsec_audit.log

Count blocked requests by rule ID
sudo grep -o '\[id "[0-9]*"\]' /var/log/nginx/modsec_audit.log | sort | uniq -c | sort -nr

Monitor error log for ModSecurity issues
sudo tail -f /var/log/nginx/error.log | grep -i modsec

Create monitoring script

Set up automated monitoring for ModSecurity blocking activity and performance metrics.

#!/bin/bash

ModSecurity monitoring script
LOGFILE="/var/log/nginx/modsec_audit.log"
ALERT_EMAIL="admin@example.com"
THRESHOLD=100

Count blocks in last hour
BLOCKS=$(grep "$(date -d '1 hour ago' '+%d/%b/%Y:%H')" $LOGFILE | wc -l)

if [ $BLOCKS -gt $THRESHOLD ]; then
    echo "ModSecurity Alert: $BLOCKS requests blocked in the last hour" | \
    mail -s "High WAF Activity Alert" $ALERT_EMAIL
fi

Log summary statistics
echo "$(date): $BLOCKS requests blocked in last hour" >> /var/log/modsec-stats.log

sudo chmod +x /usr/local/bin/modsec-monitor.sh

Add to crontab for hourly monitoring
echo "0     /usr/local/bin/modsec-monitor.sh" | sudo crontab -