Configure OpenResty load balancing with upstream health checks and automatic failover

Update system packages

Start by updating your package manager to ensure you have the latest security patches.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install OpenResty

Add the OpenResty repository and install the package with all necessary modules for load balancing.

wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add -
echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/openresty.list
sudo apt update
sudo apt install -y openresty openresty-opm

sudo dnf install -y wget
wget https://openresty.org/package/centos/openresty.repo
sudo mv openresty.repo /etc/yum.repos.d/
sudo dnf install -y openresty openresty-opm

Install health check module

Install the upstream health check module that monitors backend server availability.

sudo opm install ledgetech/lua-resty-upstream-healthcheck

Create main configuration

Configure OpenResty with upstream server definitions and load balancing settings.

worker_processes auto;
worker_rlimit_nofile 65535;

events {
    worker_connections 4096;
    use epoll;
    multi_accept on;
}

http {
    include       mime.types;
    default_type  application/octet-stream;
    
    # Logging
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                   '$status $body_bytes_sent "$http_referer" '
                   '"$http_user_agent" "$http_x_forwarded_for" '
                   'upstream: $upstream_addr response_time: $upstream_response_time';
    
    access_log /var/log/openresty/access.log main;
    error_log /var/log/openresty/error.log warn;
    
    # Basic settings
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    
    # Gzip compression
    gzip on;
    gzip_vary on;
    gzip_proxied any;
    gzip_comp_level 6;
    gzip_types text/plain text/css application/json application/javascript text/xml application/xml;
    
    # Define upstream servers
    upstream web_servers {
        least_conn;  # Load balancing algorithm
        
        server 203.0.113.10:80 weight=3 max_fails=3 fail_timeout=30s;
        server 203.0.113.11:80 weight=2 max_fails=3 fail_timeout=30s;
        server 203.0.113.12:80 weight=1 max_fails=3 fail_timeout=30s backup;
        
        keepalive 32;
        keepalive_requests 100;
        keepalive_timeout 60s;
    }
    
    # Health check configuration
    lua_shared_dict healthcheck 1m;
    lua_socket_log_errors off;
    
    init_worker_by_lua_block {
        local hc = require "resty.upstream.healthcheck"
        
        local ok, err = hc.spawn_checker{
            shm = "healthcheck",
            upstream = "web_servers",
            type = "http",
            
            http_req = "GET /health HTTP/1.0\r\nHost: example.com\r\n\r\n",
            interval = 3000,  # 3 seconds
            timeout = 1000,   # 1 second
            fall = 3,         # consider dead after 3 failures
            rise = 2,         # consider healthy after 2 successes
            valid_statuses = {200, 302},
            concurrency = 10,
        }
        
        if not ok then
            ngx.log(ngx.ERR, "failed to spawn health checker: ", err)
            return
        end
    }
    
    server {
        listen 80;
        server_name example.com www.example.com;
        
        location / {
            proxy_pass http://web_servers;
            
            # Proxy headers
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            
            # Connection settings
            proxy_connect_timeout 5s;
            proxy_send_timeout 30s;
            proxy_read_timeout 30s;
            
            # HTTP version for keepalive
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            
            # Error handling
            proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
            proxy_next_upstream_tries 3;
            proxy_next_upstream_timeout 10s;
        }
        
        # Health check status endpoint
        location /nginx_status {
            access_log off;
            allow 127.0.0.1;
            deny all;
            
            content_by_lua_block {
                local hc = require "resty.upstream.healthcheck"
                ngx.say("Nginx is running")
                ngx.say("Upstream status:")
                ngx.print(hc.status_page())
            }
        }
    }
}

Configure SSL termination

Set up SSL certificates and configure HTTPS with secure headers for production use.

sudo mkdir -p /etc/ssl/openresty
sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
    -keyout /etc/ssl/openresty/server.key \
    -out /etc/ssl/openresty/server.crt \
    -subj "/C=US/ST=State/L=City/O=Organization/CN=example.com"

Add SSL server configuration

Create a secure HTTPS configuration with modern security headers and SSL best practices.

server {
    listen 443 ssl http2;
    server_name example.com www.example.com;
    
    # SSL configuration
    ssl_certificate /etc/ssl/openresty/server.crt;
    ssl_certificate_key /etc/ssl/openresty/server.key;
    
    ssl_session_timeout 1d;
    ssl_session_cache shared:SSL:50m;
    ssl_session_tickets off;
    
    # Modern SSL configuration
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers off;
    
    # OCSP stapling
    ssl_stapling on;
    ssl_stapling_verify on;
    
    # Security headers
    add_header Strict-Transport-Security "max-age=63072000" always;
    add_header X-Frame-Options DENY;
    add_header X-Content-Type-Options nosniff;
    add_header X-XSS-Protection "1; mode=block";
    add_header Referrer-Policy "no-referrer-when-downgrade";
    
    location / {
        proxy_pass http://web_servers;
        
        # Enhanced proxy headers for SSL
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header X-Forwarded-Port $server_port;
        
        # Connection settings
        proxy_connect_timeout 5s;
        proxy_send_timeout 30s;
        proxy_read_timeout 30s;
        
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        
        # SSL proxy settings
        proxy_ssl_verify off;
        
        # Enhanced error handling
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_next_upstream_tries 3;
        proxy_next_upstream_timeout 10s;
        
        # Enable buffering for better performance
        proxy_buffering on;
        proxy_buffer_size 4k;
        proxy_buffers 8 4k;
    }
}

Redirect HTTP to HTTPS
server {
    listen 80;
    server_name example.com www.example.com;
    return 301 https://$server_name$request_uri;
}

Include SSL configuration

Add the SSL configuration to the main nginx.conf file.

sudo sed -i '/server {/,$d' /usr/local/openresty/nginx/conf/nginx.conf
echo "    include ssl.conf;" | sudo tee -a /usr/local/openresty/nginx/conf/nginx.conf
echo "}" | sudo tee -a /usr/local/openresty/nginx/conf/nginx.conf

Create log directories

Set up log directories with proper permissions for OpenResty logging.

sudo mkdir -p /var/log/openresty
sudo chown nobody:nogroup /var/log/openresty
sudo chmod 755 /var/log/openresty

Configure logrotate

Set up automatic log rotation to prevent disk space issues.

/var/log/openresty/*.log {
    daily
    missingok
    rotate 52
    compress
    delaycompress
    notifempty
    create 644 nobody nogroup
    postrotate
        if [ -f /usr/local/openresty/nginx/logs/nginx.pid ]; then
            kill -USR1 cat /usr/local/openresty/nginx/logs/nginx.pid
        fi
    endscript
}

Create systemd service

Set up OpenResty as a system service for automatic startup and management.

[Unit]
Description=OpenResty Web Server
After=network.target

[Service]
Type=forking
PIDFile=/usr/local/openresty/nginx/logs/nginx.pid
ExecStartPre=/usr/local/openresty/bin/openresty -t
ExecStart=/usr/local/openresty/bin/openresty
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true

[Install]
WantedBy=multi-user.target

Configure firewall rules

Open the necessary ports for HTTP and HTTPS traffic.

sudo ufw allow 80/tcp
sudo ufw allow 443/tcp
sudo ufw allow 22/tcp
sudo ufw --force enable

sudo firewall-cmd --permanent --add-service=http
sudo firewall-cmd --permanent --add-service=https
sudo firewall-cmd --permanent --add-service=ssh
sudo firewall-cmd --reload

Test configuration and start service

Validate the configuration syntax and start OpenResty with automatic startup enabled.

sudo /usr/local/openresty/bin/openresty -t
sudo systemctl daemon-reload
sudo systemctl enable openresty
sudo systemctl start openresty

Create health check endpoint

Set up a monitoring script to check upstream server health and send alerts.

#!/bin/bash

Configuration
UPSTREAM_SERVERS=("203.0.113.10:80" "203.0.113.11:80" "203.0.113.12:80")
HEALTH_ENDPOINT="/health"
TIMEOUT=5
EMAIL="admin@example.com"
LOG_FILE="/var/log/openresty/health-check.log"

Function to check server health
check_server() {
    local server=$1
    local status_code=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout $TIMEOUT "http://$server$HEALTH_ENDPOINT")
    
    if [ "$status_code" = "200" ]; then
        echo "$(date): Server $server is healthy" >> $LOG_FILE
        return 0
    else
        echo "$(date): Server $server is DOWN (HTTP $status_code)" >> $LOG_FILE
        return 1
    fi
}

Check all servers
failed_servers=()
for server in "${UPSTREAM_SERVERS[@]}"; do
    if ! check_server $server; then
        failed_servers+=("$server")
    fi
done

Send alert if any servers are down
if [ ${#failed_servers[@]} -gt 0 ]; then
    alert_message="ALERT: The following upstream servers are down: ${failed_servers[*]}"
    echo "$alert_message" | logger -t "openresty-health"
    
    # Uncomment to enable email alerts
    # echo "$alert_message" | mail -s "OpenResty Upstream Alert" $EMAIL
fi

Make monitoring script executable

Set appropriate permissions for the health check script.

sudo chmod +x /usr/local/bin/check-upstream-health.sh
sudo chown root:root /usr/local/bin/check-upstream-health.sh

Schedule health checks

Add a cron job to run health checks every minute.

echo "    * /usr/local/bin/check-upstream-health.sh" | sudo crontab -

Configure session persistence

Enable sticky sessions using IP hash for applications requiring session affinity.

# IP hash for session persistence
upstream web_servers_sticky {
    ip_hash;
    
    server 203.0.113.10:80 weight=3 max_fails=3 fail_timeout=30s;
    server 203.0.113.11:80 weight=2 max_fails=3 fail_timeout=30s;
    server 203.0.113.12:80 weight=1 max_fails=3 fail_timeout=30s backup;
    
    keepalive 32;
}

Round robin for stateless requests
upstream api_servers {
    server 203.0.113.10:8080 weight=1 max_fails=2 fail_timeout=15s;
    server 203.0.113.11:8080 weight=1 max_fails=2 fail_timeout=15s;
    server 203.0.113.12:8080 weight=1 max_fails=2 fail_timeout=15s;
    
    keepalive 16;
}

Configure different upstream pools

Create location-specific upstream routing for different application components.

# Add to server block in ssl.conf

Static content (round robin)
location ~* \.(css|js|png|jpg|jpeg|gif|ico|svg)$ {
    proxy_pass http://web_servers;
    
    # Extended caching for static files
    proxy_cache_valid 200 302 1d;
    proxy_cache_valid 404 1m;
    
    # Standard proxy headers
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    
    expires 1d;
    add_header Cache-Control "public, immutable";
}

API requests (round robin)
location /api/ {
    proxy_pass http://api_servers;
    
    # API-specific headers
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    proxy_set_header Content-Type application/json;
    
    # Shorter timeouts for API
    proxy_connect_timeout 3s;
    proxy_send_timeout 10s;
    proxy_read_timeout 10s;
}

Web application (sticky sessions)
location / {
    proxy_pass http://web_servers_sticky;
    
    # Session affinity headers
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    
    proxy_connect_timeout 5s;
    proxy_send_timeout 30s;
    proxy_read_timeout 30s;
}