Install OpenResty

OpenResty extends Nginx with Lua scripting capabilities for advanced load balancing features.

wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add -
echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/openresty.list
sudo apt update
sudo apt install -y openresty

sudo yum install -y yum-utils
sudo yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo
sudo yum install -y openresty

Install lua-resty-upstream-healthcheck

This module provides active health checking capabilities for upstream servers.

sudo apt install -y lua-resty-upstream-healthcheck
sudo apt install -y lua-resty-lock

sudo yum install -y lua-resty-upstream-healthcheck
sudo yum install -y lua-resty-lock

Create OpenResty configuration directory

Set up the directory structure for OpenResty configuration files.

sudo mkdir -p /usr/local/openresty/nginx/conf/conf.d
sudo mkdir -p /usr/local/openresty/nginx/logs
sudo mkdir -p /var/log/openresty

Configure main OpenResty configuration

Create the main configuration file with load balancing and health check modules.

user www-data;
worker_processes auto;
error_log /var/log/openresty/error.log warn;
pid /var/run/openresty.pid;

events {
    worker_connections 1024;
    use epoll;
    multi_accept on;
}

http {
    include /usr/local/openresty/nginx/conf/mime.types;
    default_type application/octet-stream;

    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for" '
                    'upstream: $upstream_addr response_time: $upstream_response_time';

    access_log /var/log/openresty/access.log main;

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;

    # Shared memory for health checks
    lua_shared_dict healthcheck 1m;
    lua_shared_dict locks 1m;

    # Load health check module
    lua_package_path "/usr/local/openresty/lualib/?.lua;;";
    init_worker_by_lua_block {
        local hc = require "resty.upstream.healthcheck"
        
        local ok, err = hc.spawn_checker {
            shm = "healthcheck",
            upstream = "backend_servers",
            type = "http",
            http_req = "GET /health HTTP/1.0\r\nHost: backend\r\n\r\n",
            interval = 2000,  -- 2 seconds
            timeout = 1000,   -- 1 second
            fall = 3,         -- # successive failures before marking unhealthy
            rise = 2,         -- # successive successes before marking healthy
            valid_statuses = {200, 302},
            concurrency = 10,
        }
        if not ok then
            ngx.log(ngx.ERR, "failed to spawn health checker: ", err)
            return
        end
    }

    # Upstream backend servers
    upstream backend_servers {
        server 203.0.113.10:8080 max_fails=0 fail_timeout=0;
        server 203.0.113.11:8080 max_fails=0 fail_timeout=0;
        server 203.0.113.12:8080 max_fails=0 fail_timeout=0;
        
        # Load balancing method
        least_conn;
        
        # Connection keepalive
        keepalive 32;
    }

    # Health check status endpoint
    server {
        listen 8090;
        location /status {
            access_log off;
            allow 127.0.0.1;
            allow 203.0.113.0/24;
            deny all;
            
            content_by_lua_block {
                local hc = require "resty.upstream.healthcheck"
                ngx.say("Nginx health check status page")
                hc.status_page()
            }
        }
    }

    include /usr/local/openresty/nginx/conf/conf.d/*.conf;
}

Configure load balancer virtual host

Create the main load balancer configuration with failover logic.

server {
    listen 80;
    server_name example.com;

    # Real IP configuration for proxy
    set_real_ip_from 203.0.113.0/24;
    real_ip_header X-Forwarded-For;
    real_ip_recursive on;

    # Proxy settings
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    
    # Timeouts
    proxy_connect_timeout 5s;
    proxy_send_timeout 10s;
    proxy_read_timeout 30s;
    
    # Buffer settings
    proxy_buffering on;
    proxy_buffer_size 4k;
    proxy_buffers 8 4k;
    proxy_busy_buffers_size 8k;
    
    # Main location with health-aware load balancing
    location / {
        proxy_pass http://backend_servers;
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
        proxy_next_upstream_tries 3;
        proxy_next_upstream_timeout 10s;
        
        # Add response headers for debugging
        add_header X-Upstream-Server $upstream_addr always;
        add_header X-Response-Time $upstream_response_time always;
    }
    
    # Health check endpoint for backends
    location /health {
        access_log off;
        return 200 "healthy";
        add_header Content-Type text/plain;
    }
    
    # Load balancer status page
    location /lb-status {
        access_log off;
        allow 127.0.0.1;
        allow 203.0.113.0/24;
        deny all;
        
        content_by_lua_block {
            local upstream = require "ngx.upstream"
            local hc = require "resty.upstream.healthcheck"
            
            ngx.say("Backend Server Status:")
            ngx.say("========================")
            
            local ups = upstream.get_servers("backend_servers")
            if not ups then
                ngx.say("No upstream servers found")
                return
            end
            
            for _, server in ipairs(ups) do
                local status = "unknown"
                if server.backup then
                    status = "backup"
                elseif server.down then
                    status = "down"
                elseif server.fail_timeout and server.max_fails then
                    status = "active"
                end
                
                ngx.say(string.format("%s:%s - %s (weight: %d)", 
                    server.name or server.addr, 
                    server.port, 
                    status, 
                    server.weight or 1))
            end
        }
    }
}

Create SSL configuration for production

Add HTTPS support with automatic HTTP to HTTPS redirection.

server {
    listen 80;
    server_name example.com;
    return 301 https://$server_name$request_uri;
}

server {
    listen 443 ssl http2;
    server_name example.com;

    ssl_certificate /etc/ssl/certs/example.com.pem;
    ssl_certificate_key /etc/ssl/private/example.com.key;
    
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers off;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;
    ssl_session_tickets off;
    
    # HSTS
    add_header Strict-Transport-Security "max-age=31536000; includeSubdomains; preload" always;
    
    # Security headers
    add_header X-Frame-Options DENY always;
    add_header X-Content-Type-Options nosniff always;
    add_header Referrer-Policy strict-origin-when-cross-origin always;

    # Real IP configuration
    set_real_ip_from 203.0.113.0/24;
    real_ip_header X-Forwarded-For;
    real_ip_recursive on;

    # Proxy settings
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    
    # Main location
    location / {
        proxy_pass http://backend_servers;
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
        proxy_next_upstream_tries 3;
        proxy_next_upstream_timeout 10s;
        
        add_header X-Upstream-Server $upstream_addr always;
        add_header X-Response-Time $upstream_response_time always;
    }
}

Create systemd service file

Set up OpenResty to run as a system service with proper permissions.

[Unit]
Description=OpenResty (nginx)
Documentation=https://openresty.org/
After=network.target
Wants=network.target

[Service]
Type=forking
PIDFile=/var/run/openresty.pid
ExecStartPre=/usr/local/openresty/bin/openresty -t
ExecStart=/usr/local/openresty/bin/openresty
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true

[Install]
WantedBy=multi-user.target

Set proper permissions

Configure file ownership and permissions for OpenResty directories.

sudo chown -R www-data:www-data /var/log/openresty
sudo chmod 755 /var/log/openresty
sudo chown www-data:www-data /var/run/openresty.pid 2>/dev/null || true
sudo chmod 644 /usr/local/openresty/nginx/conf/nginx.conf
sudo chmod 644 /usr/local/openresty/nginx/conf/conf.d/*.conf

Configure log rotation

Set up automatic log rotation to prevent disk space issues.

/var/log/openresty/*.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    copytruncate
    postrotate
        if [ -f /var/run/openresty.pid ]; then
            kill -USR1 cat /var/run/openresty.pid
        fi
    endscript
}

Start and enable OpenResty

Enable the service to start automatically and verify it's running.

sudo systemctl daemon-reload
sudo systemctl enable openresty
sudo systemctl start openresty
sudo systemctl status openresty

Create advanced health check configuration

Implement custom health check logic with detailed monitoring.

# Advanced health check monitoring
server {
    listen 8091;
    server_name localhost;
    
    location /health-check {
        access_log off;
        allow 127.0.0.1;
        allow 203.0.113.0/24;
        deny all;
        
        content_by_lua_block {
            local json = require "cjson"
            local hc = require "resty.upstream.healthcheck"
            local upstream = require "ngx.upstream"
            
            local function check_backend(host, port)
                local sock = ngx.socket.tcp()
                sock:settimeout(1000)
                
                local ok, err = sock:connect(host, port)
                if not ok then
                    return false, err
                end
                
                local req = "GET /health HTTP/1.0\r\nHost: " .. host .. "\r\n\r\n"
                local bytes, err = sock:send(req)
                if not bytes then
                    sock:close()
                    return false, err
                end
                
                local line, err = sock:receive("*l")
                sock:close()
                
                if line and string.find(line, "200") then
                    return true, "healthy"
                else
                    return false, "unhealthy response"
                end
            end
            
            local backends = {
                {host = "203.0.113.10", port = 8080},
                {host = "203.0.113.11", port = 8080},
                {host = "203.0.113.12", port = 8080}
            }
            
            local status = {}
            local healthy_count = 0
            local total_count = #backends
            
            for _, backend in ipairs(backends) do
                local is_healthy, msg = check_backend(backend.host, backend.port)
                status[backend.host .. ":" .. backend.port] = {
                    healthy = is_healthy,
                    message = msg,
                    checked_at = ngx.now()
                }
                if is_healthy then
                    healthy_count = healthy_count + 1
                end
            end
            
            local response = {
                timestamp = ngx.now(),
                total_backends = total_count,
                healthy_backends = healthy_count,
                status = status,
                load_balancer_healthy = healthy_count > 0
            }
            
            ngx.header.content_type = "application/json"
            ngx.say(json.encode(response))
        }
    }
}

Add performance monitoring

Configure metrics collection for load balancing performance.

server {
    listen 8092;
    server_name localhost;
    
    location /metrics {
        access_log off;
        allow 127.0.0.1;
        allow 203.0.113.0/24;
        deny all;
        
        content_by_lua_block {
            local json = require "cjson"
            local upstream = require "ngx.upstream"
            
            -- Get basic upstream statistics
            local ups = upstream.get_servers("backend_servers")
            local metrics = {
                timestamp = ngx.now(),
                upstream_servers = {},
                requests_total = 0,
                active_connections = 0
            }
            
            if ups then
                for _, server in ipairs(ups) do
                    local server_info = {
                        address = server.name or (server.addr .. ":" .. server.port),
                        weight = server.weight or 1,
                        max_fails = server.max_fails or 1,
                        fail_timeout = server.fail_timeout or 10,
                        backup = server.backup or false,
                        down = server.down or false
                    }
                    table.insert(metrics.upstream_servers, server_info)
                end
            end
            
            ngx.header.content_type = "application/json"
            ngx.say(json.encode(metrics))
        }
    }
    
    location /prometheus-metrics {
        access_log off;
        allow 127.0.0.1;
        allow 203.0.113.0/24;
        deny all;
        
        content_by_lua_block {
            local upstream = require "ngx.upstream"
            
            ngx.say("# HELP openresty_upstream_servers_total Total number of upstream servers")
            ngx.say("# TYPE openresty_upstream_servers_total gauge")
            
            local ups = upstream.get_servers("backend_servers")
            local total_servers = 0
            local active_servers = 0
            
            if ups then
                for _, server in ipairs(ups) do
                    total_servers = total_servers + 1
                    if not server.down then
                        active_servers = active_servers + 1
                    end
                end
            end
            
            ngx.say(string.format("openresty_upstream_servers_total{upstream=\"backend_servers\"} %d", total_servers))
            ngx.say(string.format("openresty_upstream_servers_active{upstream=\"backend_servers\"} %d", active_servers))
        }
    }
}

Configure backup server

Add a backup server that activates when all primary servers fail.

# Enhanced upstream with backup server
upstream backend_servers_with_backup {
    # Primary servers
    server 203.0.113.10:8080 max_fails=3 fail_timeout=30s;
    server 203.0.113.11:8080 max_fails=3 fail_timeout=30s;
    server 203.0.113.12:8080 max_fails=3 fail_timeout=30s;
    
    # Backup server - only used when all primary servers are down
    server 203.0.113.20:8080 backup;
    
    # Load balancing settings
    least_conn;
    keepalive 32;
    keepalive_requests 100;
    keepalive_timeout 60s;
}

server {
    listen 81;
    server_name example.com;
    
    # Enhanced error handling
    error_page 502 503 504 /maintenance.html;
    
    location = /maintenance.html {
        root /usr/local/openresty/nginx/html;
        internal;
    }
    
    location / {
        # Try backup upstream if primary fails
        proxy_pass http://backend_servers_with_backup;
        
        # Aggressive failover settings
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_next_upstream_tries 5;
        proxy_next_upstream_timeout 15s;
        
        # Shorter timeouts for faster failover
        proxy_connect_timeout 3s;
        proxy_send_timeout 5s;
        proxy_read_timeout 15s;
        
        # Headers
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        
        # Failover debugging
        add_header X-Upstream-Server $upstream_addr always;
        add_header X-Response-Time $upstream_response_time always;
        add_header X-Upstream-Status $upstream_status always;
    }
}

Create maintenance page

Design a user-friendly maintenance page for complete outages.

    
    
    
    


    

        
503
        
Service Temporarily Unavailable
        
We're currently experiencing technical difficulties. Our team has been notified and is working to restore service as quickly as possible.
        
Please try again in a few minutes. We apologize for any inconvenience.
    

Set up access log analysis

Configure detailed logging for performance monitoring and troubleshooting.

sudo mkdir -p /var/log/openresty/analysis
sudo chown www-data:www-data /var/log/openresty/analysis

Create log analysis script

Build a script to analyze load balancer performance from access logs.

#!/bin/bash

LOG_FILE="/var/log/openresty/access.log"
OUTPUT_DIR="/var/log/openresty/analysis"
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")

Create analysis report
cat > "$OUTPUT_DIR/report_$TIMESTAMP.txt" << EOF
OpenResty Load Balancer Analysis Report
Generated: $(date)
========================================

Top 10 Response Times:
EOF

Extract response times and sort
awk '{print $(NF)}' "$LOG_FILE" | grep -E '^[0-9.]+$' | sort -n | tail -10 >> "$OUTPUT_DIR/report_$TIMESTAMP.txt"

echo -e "\nBackend Server Distribution:" >> "$OUTPUT_DIR/report_$TIMESTAMP.txt"
awk '{for(i=1;i<=NF;i++) if($i ~ /upstream:/) print $(i+1)}' "$LOG_FILE" | sort | uniq -c | sort -nr >> "$OUTPUT_DIR/report_$TIMESTAMP.txt"

echo -e "\nHTTP Status Codes:" >> "$OUTPUT_DIR/report_$TIMESTAMP.txt"
awk '{print $9}' "$LOG_FILE" | sort | uniq -c | sort -nr >> "$OUTPUT_DIR/report_$TIMESTAMP.txt"

echo -e "\nTop 10 Slowest Requests:" >> "$OUTPUT_DIR/report_$TIMESTAMP.txt"
awk '{print $(NF), $0}' "$LOG_FILE" | grep -E '^[0-9.]+' | sort -nr | head -10 | cut -d' ' -f2- >> "$OUTPUT_DIR/report_$TIMESTAMP.txt"

echo "Analysis complete. Report saved to: $OUTPUT_DIR/report_$TIMESTAMP.txt"

Make analysis script executable and schedule it

Set proper permissions and create a cron job for regular analysis.

sudo chmod +x /usr/local/bin/analyze-openresty-logs.sh
sudo chown root:root /usr/local/bin/analyze-openresty-logs.sh

Create monitoring dashboard endpoint

Set up a real-time monitoring dashboard accessible via web browser.

    
    
    
    


    

        
OpenResty Load Balancer Dashboard
        
        

            
System Status
            Refresh Data
            

        
        
        

            

                
Health Check Status
                
Loading...
            
            
            

                
Metrics
                
Loading...