Setup OpenResty load balancing with health checks and automatic failover

Intermediate 45 min Jun 12, 2026 452 views
Ubuntu 24.04 Debian 12 AlmaLinux 9 Rocky Linux 9

Configure OpenResty with upstream backend servers, implement health monitoring, and set up automatic failover for high availability load balancing.

Prerequisites

  • Root or sudo access
  • Multiple backend servers
  • Basic understanding of HTTP load balancing

What this solves

OpenResty provides advanced load balancing capabilities with built-in health checks and automatic failover. This setup distributes traffic across multiple backend servers while continuously monitoring their availability and removing unhealthy servers from rotation.

Step-by-step installation

Install OpenResty

OpenResty extends Nginx with Lua scripting capabilities for advanced load balancing features.

wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add -
echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/openresty.list
sudo apt update
sudo apt install -y openresty
sudo yum install -y yum-utils
sudo yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo
sudo yum install -y openresty

Install lua-resty-upstream-healthcheck

This module provides active health checking capabilities for upstream servers.

sudo apt install -y lua-resty-upstream-healthcheck
sudo apt install -y lua-resty-lock
sudo yum install -y lua-resty-upstream-healthcheck
sudo yum install -y lua-resty-lock

Create OpenResty configuration directory

Set up the directory structure for OpenResty configuration files.

sudo mkdir -p /usr/local/openresty/nginx/conf/conf.d
sudo mkdir -p /usr/local/openresty/nginx/logs
sudo mkdir -p /var/log/openresty

Configure main OpenResty configuration

Create the main configuration file with load balancing and health check modules.

user www-data;
worker_processes auto;
error_log /var/log/openresty/error.log warn;
pid /var/run/openresty.pid;

events {
    worker_connections 1024;
    use epoll;
    multi_accept on;
}

http {
    include /usr/local/openresty/nginx/conf/mime.types;
    default_type application/octet-stream;

    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for" '
                    'upstream: $upstream_addr response_time: $upstream_response_time';

    access_log /var/log/openresty/access.log main;

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;

    # Shared memory for health checks
    lua_shared_dict healthcheck 1m;
    lua_shared_dict locks 1m;

    # Load health check module
    lua_package_path "/usr/local/openresty/lualib/?.lua;;";
    init_worker_by_lua_block {
        local hc = require "resty.upstream.healthcheck"
        
        local ok, err = hc.spawn_checker {
            shm = "healthcheck",
            upstream = "backend_servers",
            type = "http",
            http_req = "GET /health HTTP/1.0\r\nHost: backend\r\n\r\n",
            interval = 2000,  -- 2 seconds
            timeout = 1000,   -- 1 second
            fall = 3,         -- # successive failures before marking unhealthy
            rise = 2,         -- # successive successes before marking healthy
            valid_statuses = {200, 302},
            concurrency = 10,
        }
        if not ok then
            ngx.log(ngx.ERR, "failed to spawn health checker: ", err)
            return
        end
    }

    # Upstream backend servers
    upstream backend_servers {
        server 203.0.113.10:8080 max_fails=0 fail_timeout=0;
        server 203.0.113.11:8080 max_fails=0 fail_timeout=0;
        server 203.0.113.12:8080 max_fails=0 fail_timeout=0;
        
        # Load balancing method
        least_conn;
        
        # Connection keepalive
        keepalive 32;
    }

    # Health check status endpoint
    server {
        listen 8090;
        location /status {
            access_log off;
            allow 127.0.0.1;
            allow 203.0.113.0/24;
            deny all;
            
            content_by_lua_block {
                local hc = require "resty.upstream.healthcheck"
                ngx.say("Nginx health check status page")
                hc.status_page()
            }
        }
    }

    include /usr/local/openresty/nginx/conf/conf.d/*.conf;
}

Configure load balancer virtual host

Create the main load balancer configuration with failover logic.

server {
    listen 80;
    server_name example.com;

    # Real IP configuration for proxy
    set_real_ip_from 203.0.113.0/24;
    real_ip_header X-Forwarded-For;
    real_ip_recursive on;

    # Proxy settings
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    
    # Timeouts
    proxy_connect_timeout 5s;
    proxy_send_timeout 10s;
    proxy_read_timeout 30s;
    
    # Buffer settings
    proxy_buffering on;
    proxy_buffer_size 4k;
    proxy_buffers 8 4k;
    proxy_busy_buffers_size 8k;
    
    # Main location with health-aware load balancing
    location / {
        proxy_pass http://backend_servers;
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
        proxy_next_upstream_tries 3;
        proxy_next_upstream_timeout 10s;
        
        # Add response headers for debugging
        add_header X-Upstream-Server $upstream_addr always;
        add_header X-Response-Time $upstream_response_time always;
    }
    
    # Health check endpoint for backends
    location /health {
        access_log off;
        return 200 "healthy";
        add_header Content-Type text/plain;
    }
    
    # Load balancer status page
    location /lb-status {
        access_log off;
        allow 127.0.0.1;
        allow 203.0.113.0/24;
        deny all;
        
        content_by_lua_block {
            local upstream = require "ngx.upstream"
            local hc = require "resty.upstream.healthcheck"
            
            ngx.say("Backend Server Status:")
            ngx.say("========================")
            
            local ups = upstream.get_servers("backend_servers")
            if not ups then
                ngx.say("No upstream servers found")
                return
            end
            
            for _, server in ipairs(ups) do
                local status = "unknown"
                if server.backup then
                    status = "backup"
                elseif server.down then
                    status = "down"
                elseif server.fail_timeout and server.max_fails then
                    status = "active"
                end
                
                ngx.say(string.format("%s:%s - %s (weight: %d)", 
                    server.name or server.addr, 
                    server.port, 
                    status, 
                    server.weight or 1))
            end
        }
    }
}

Create SSL configuration for production

Add HTTPS support with automatic HTTP to HTTPS redirection.

server {
    listen 80;
    server_name example.com;
    return 301 https://$server_name$request_uri;
}

server {
    listen 443 ssl http2;
    server_name example.com;

    ssl_certificate /etc/ssl/certs/example.com.pem;
    ssl_certificate_key /etc/ssl/private/example.com.key;
    
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers off;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;
    ssl_session_tickets off;
    
    # HSTS
    add_header Strict-Transport-Security "max-age=31536000; includeSubdomains; preload" always;
    
    # Security headers
    add_header X-Frame-Options DENY always;
    add_header X-Content-Type-Options nosniff always;
    add_header Referrer-Policy strict-origin-when-cross-origin always;

    # Real IP configuration
    set_real_ip_from 203.0.113.0/24;
    real_ip_header X-Forwarded-For;
    real_ip_recursive on;

    # Proxy settings
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    
    # Main location
    location / {
        proxy_pass http://backend_servers;
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
        proxy_next_upstream_tries 3;
        proxy_next_upstream_timeout 10s;
        
        add_header X-Upstream-Server $upstream_addr always;
        add_header X-Response-Time $upstream_response_time always;
    }
}

Create systemd service file

Set up OpenResty to run as a system service with proper permissions.

[Unit]
Description=OpenResty (nginx)
Documentation=https://openresty.org/
After=network.target
Wants=network.target

[Service]
Type=forking
PIDFile=/var/run/openresty.pid
ExecStartPre=/usr/local/openresty/bin/openresty -t
ExecStart=/usr/local/openresty/bin/openresty
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true

[Install]
WantedBy=multi-user.target

Set proper permissions

Configure file ownership and permissions for OpenResty directories.

sudo chown -R www-data:www-data /var/log/openresty
sudo chmod 755 /var/log/openresty
sudo chown www-data:www-data /var/run/openresty.pid 2>/dev/null || true
sudo chmod 644 /usr/local/openresty/nginx/conf/nginx.conf
sudo chmod 644 /usr/local/openresty/nginx/conf/conf.d/*.conf

Configure log rotation

Set up automatic log rotation to prevent disk space issues.

/var/log/openresty/*.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    copytruncate
    postrotate
        if [ -f /var/run/openresty.pid ]; then
            kill -USR1 `cat /var/run/openresty.pid`
        fi
    endscript
}

Start and enable OpenResty

Enable the service to start automatically and verify it's running.

sudo systemctl daemon-reload
sudo systemctl enable openresty
sudo systemctl start openresty
sudo systemctl status openresty

Configure upstream backend monitoring

Create advanced health check configuration

Implement custom health check logic with detailed monitoring.

# Advanced health check monitoring
server {
    listen 8091;
    server_name localhost;
    
    location /health-check {
        access_log off;
        allow 127.0.0.1;
        allow 203.0.113.0/24;
        deny all;
        
        content_by_lua_block {
            local json = require "cjson"
            local hc = require "resty.upstream.healthcheck"
            local upstream = require "ngx.upstream"
            
            local function check_backend(host, port)
                local sock = ngx.socket.tcp()
                sock:settimeout(1000)
                
                local ok, err = sock:connect(host, port)
                if not ok then
                    return false, err
                end
                
                local req = "GET /health HTTP/1.0\r\nHost: " .. host .. "\r\n\r\n"
                local bytes, err = sock:send(req)
                if not bytes then
                    sock:close()
                    return false, err
                end
                
                local line, err = sock:receive("*l")
                sock:close()
                
                if line and string.find(line, "200") then
                    return true, "healthy"
                else
                    return false, "unhealthy response"
                end
            end
            
            local backends = {
                {host = "203.0.113.10", port = 8080},
                {host = "203.0.113.11", port = 8080},
                {host = "203.0.113.12", port = 8080}
            }
            
            local status = {}
            local healthy_count = 0
            local total_count = #backends
            
            for _, backend in ipairs(backends) do
                local is_healthy, msg = check_backend(backend.host, backend.port)
                status[backend.host .. ":" .. backend.port] = {
                    healthy = is_healthy,
                    message = msg,
                    checked_at = ngx.now()
                }
                if is_healthy then
                    healthy_count = healthy_count + 1
                end
            end
            
            local response = {
                timestamp = ngx.now(),
                total_backends = total_count,
                healthy_backends = healthy_count,
                status = status,
                load_balancer_healthy = healthy_count > 0
            }
            
            ngx.header.content_type = "application/json"
            ngx.say(json.encode(response))
        }
    }
}

Add performance monitoring

Configure metrics collection for load balancing performance.

server {
    listen 8092;
    server_name localhost;
    
    location /metrics {
        access_log off;
        allow 127.0.0.1;
        allow 203.0.113.0/24;
        deny all;
        
        content_by_lua_block {
            local json = require "cjson"
            local upstream = require "ngx.upstream"
            
            -- Get basic upstream statistics
            local ups = upstream.get_servers("backend_servers")
            local metrics = {
                timestamp = ngx.now(),
                upstream_servers = {},
                requests_total = 0,
                active_connections = 0
            }
            
            if ups then
                for _, server in ipairs(ups) do
                    local server_info = {
                        address = server.name or (server.addr .. ":" .. server.port),
                        weight = server.weight or 1,
                        max_fails = server.max_fails or 1,
                        fail_timeout = server.fail_timeout or 10,
                        backup = server.backup or false,
                        down = server.down or false
                    }
                    table.insert(metrics.upstream_servers, server_info)
                end
            end
            
            ngx.header.content_type = "application/json"
            ngx.say(json.encode(metrics))
        }
    }
    
    location /prometheus-metrics {
        access_log off;
        allow 127.0.0.1;
        allow 203.0.113.0/24;
        deny all;
        
        content_by_lua_block {
            local upstream = require "ngx.upstream"
            
            ngx.say("# HELP openresty_upstream_servers_total Total number of upstream servers")
            ngx.say("# TYPE openresty_upstream_servers_total gauge")
            
            local ups = upstream.get_servers("backend_servers")
            local total_servers = 0
            local active_servers = 0
            
            if ups then
                for _, server in ipairs(ups) do
                    total_servers = total_servers + 1
                    if not server.down then
                        active_servers = active_servers + 1
                    end
                end
            end
            
            ngx.say(string.format("openresty_upstream_servers_total{upstream=\"backend_servers\"} %d", total_servers))
            ngx.say(string.format("openresty_upstream_servers_active{upstream=\"backend_servers\"} %d", active_servers))
        }
    }
}

Implement automatic failover mechanisms

Configure backup server

Add a backup server that activates when all primary servers fail.

# Enhanced upstream with backup server
upstream backend_servers_with_backup {
    # Primary servers
    server 203.0.113.10:8080 max_fails=3 fail_timeout=30s;
    server 203.0.113.11:8080 max_fails=3 fail_timeout=30s;
    server 203.0.113.12:8080 max_fails=3 fail_timeout=30s;
    
    # Backup server - only used when all primary servers are down
    server 203.0.113.20:8080 backup;
    
    # Load balancing settings
    least_conn;
    keepalive 32;
    keepalive_requests 100;
    keepalive_timeout 60s;
}

server {
    listen 81;
    server_name example.com;
    
    # Enhanced error handling
    error_page 502 503 504 /maintenance.html;
    
    location = /maintenance.html {
        root /usr/local/openresty/nginx/html;
        internal;
    }
    
    location / {
        # Try backup upstream if primary fails
        proxy_pass http://backend_servers_with_backup;
        
        # Aggressive failover settings
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_next_upstream_tries 5;
        proxy_next_upstream_timeout 15s;
        
        # Shorter timeouts for faster failover
        proxy_connect_timeout 3s;
        proxy_send_timeout 5s;
        proxy_read_timeout 15s;
        
        # Headers
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        
        # Failover debugging
        add_header X-Upstream-Server $upstream_addr always;
        add_header X-Response-Time $upstream_response_time always;
        add_header X-Upstream-Status $upstream_status always;
    }
}

Create maintenance page

Design a user-friendly maintenance page for complete outages.




    
    
    
    

Automated install script

Run this to automate the entire setup

Nie chcesz zarządzać tym samodzielnie?

Zarządzamy infrastrukturą firm, które zależą od dostępności. W pełni zarządzana, z jednym stałym kontaktem, który zna Twoje środowisko.

Macie jednego stałego opiekuna, który zna Waszą konfigurację

Przy biurku w Rotterdamie 09:23 · dostępny w wiadomości, bez formularza zgłoszeń