Configure OpenResty with upstream backend servers, implement health monitoring, and set up automatic failover for high availability load balancing.
Prerequisites
- Root or sudo access
- Multiple backend servers
- Basic understanding of HTTP load balancing
What this solves
OpenResty provides advanced load balancing capabilities with built-in health checks and automatic failover. This setup distributes traffic across multiple backend servers while continuously monitoring their availability and removing unhealthy servers from rotation.
Step-by-step installation
Install OpenResty
OpenResty extends Nginx with Lua scripting capabilities for advanced load balancing features.
wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add -
echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/openresty.list
sudo apt update
sudo apt install -y openresty
sudo yum install -y yum-utils
sudo yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo
sudo yum install -y openresty
Install lua-resty-upstream-healthcheck
This module provides active health checking capabilities for upstream servers.
sudo apt install -y lua-resty-upstream-healthcheck
sudo apt install -y lua-resty-lock
sudo yum install -y lua-resty-upstream-healthcheck
sudo yum install -y lua-resty-lock
Create OpenResty configuration directory
Set up the directory structure for OpenResty configuration files.
sudo mkdir -p /usr/local/openresty/nginx/conf/conf.d
sudo mkdir -p /usr/local/openresty/nginx/logs
sudo mkdir -p /var/log/openresty
Configure main OpenResty configuration
Create the main configuration file with load balancing and health check modules.
user www-data;
worker_processes auto;
error_log /var/log/openresty/error.log warn;
pid /var/run/openresty.pid;
events {
worker_connections 1024;
use epoll;
multi_accept on;
}
http {
include /usr/local/openresty/nginx/conf/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" '
'upstream: $upstream_addr response_time: $upstream_response_time';
access_log /var/log/openresty/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# Shared memory for health checks
lua_shared_dict healthcheck 1m;
lua_shared_dict locks 1m;
# Load health check module
lua_package_path "/usr/local/openresty/lualib/?.lua;;";
init_worker_by_lua_block {
local hc = require "resty.upstream.healthcheck"
local ok, err = hc.spawn_checker {
shm = "healthcheck",
upstream = "backend_servers",
type = "http",
http_req = "GET /health HTTP/1.0\r\nHost: backend\r\n\r\n",
interval = 2000, -- 2 seconds
timeout = 1000, -- 1 second
fall = 3, -- # successive failures before marking unhealthy
rise = 2, -- # successive successes before marking healthy
valid_statuses = {200, 302},
concurrency = 10,
}
if not ok then
ngx.log(ngx.ERR, "failed to spawn health checker: ", err)
return
end
}
# Upstream backend servers
upstream backend_servers {
server 203.0.113.10:8080 max_fails=0 fail_timeout=0;
server 203.0.113.11:8080 max_fails=0 fail_timeout=0;
server 203.0.113.12:8080 max_fails=0 fail_timeout=0;
# Load balancing method
least_conn;
# Connection keepalive
keepalive 32;
}
# Health check status endpoint
server {
listen 8090;
location /status {
access_log off;
allow 127.0.0.1;
allow 203.0.113.0/24;
deny all;
content_by_lua_block {
local hc = require "resty.upstream.healthcheck"
ngx.say("Nginx health check status page")
hc.status_page()
}
}
}
include /usr/local/openresty/nginx/conf/conf.d/*.conf;
}
Configure load balancer virtual host
Create the main load balancer configuration with failover logic.
server {
listen 80;
server_name example.com;
# Real IP configuration for proxy
set_real_ip_from 203.0.113.0/24;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
# Proxy settings
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Timeouts
proxy_connect_timeout 5s;
proxy_send_timeout 10s;
proxy_read_timeout 30s;
# Buffer settings
proxy_buffering on;
proxy_buffer_size 4k;
proxy_buffers 8 4k;
proxy_busy_buffers_size 8k;
# Main location with health-aware load balancing
location / {
proxy_pass http://backend_servers;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
proxy_next_upstream_tries 3;
proxy_next_upstream_timeout 10s;
# Add response headers for debugging
add_header X-Upstream-Server $upstream_addr always;
add_header X-Response-Time $upstream_response_time always;
}
# Health check endpoint for backends
location /health {
access_log off;
return 200 "healthy";
add_header Content-Type text/plain;
}
# Load balancer status page
location /lb-status {
access_log off;
allow 127.0.0.1;
allow 203.0.113.0/24;
deny all;
content_by_lua_block {
local upstream = require "ngx.upstream"
local hc = require "resty.upstream.healthcheck"
ngx.say("Backend Server Status:")
ngx.say("========================")
local ups = upstream.get_servers("backend_servers")
if not ups then
ngx.say("No upstream servers found")
return
end
for _, server in ipairs(ups) do
local status = "unknown"
if server.backup then
status = "backup"
elseif server.down then
status = "down"
elseif server.fail_timeout and server.max_fails then
status = "active"
end
ngx.say(string.format("%s:%s - %s (weight: %d)",
server.name or server.addr,
server.port,
status,
server.weight or 1))
end
}
}
}
Create SSL configuration for production
Add HTTPS support with automatic HTTP to HTTPS redirection.
server {
listen 80;
server_name example.com;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl http2;
server_name example.com;
ssl_certificate /etc/ssl/certs/example.com.pem;
ssl_certificate_key /etc/ssl/private/example.com.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384;
ssl_prefer_server_ciphers off;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 10m;
ssl_session_tickets off;
# HSTS
add_header Strict-Transport-Security "max-age=31536000; includeSubdomains; preload" always;
# Security headers
add_header X-Frame-Options DENY always;
add_header X-Content-Type-Options nosniff always;
add_header Referrer-Policy strict-origin-when-cross-origin always;
# Real IP configuration
set_real_ip_from 203.0.113.0/24;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
# Proxy settings
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Main location
location / {
proxy_pass http://backend_servers;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
proxy_next_upstream_tries 3;
proxy_next_upstream_timeout 10s;
add_header X-Upstream-Server $upstream_addr always;
add_header X-Response-Time $upstream_response_time always;
}
}
Create systemd service file
Set up OpenResty to run as a system service with proper permissions.
[Unit]
Description=OpenResty (nginx)
Documentation=https://openresty.org/
After=network.target
Wants=network.target
[Service]
Type=forking
PIDFile=/var/run/openresty.pid
ExecStartPre=/usr/local/openresty/bin/openresty -t
ExecStart=/usr/local/openresty/bin/openresty
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true
[Install]
WantedBy=multi-user.target
Set proper permissions
Configure file ownership and permissions for OpenResty directories.
sudo chown -R www-data:www-data /var/log/openresty
sudo chmod 755 /var/log/openresty
sudo chown www-data:www-data /var/run/openresty.pid 2>/dev/null || true
sudo chmod 644 /usr/local/openresty/nginx/conf/nginx.conf
sudo chmod 644 /usr/local/openresty/nginx/conf/conf.d/*.conf
Configure log rotation
Set up automatic log rotation to prevent disk space issues.
/var/log/openresty/*.log {
daily
missingok
rotate 30
compress
delaycompress
notifempty
copytruncate
postrotate
if [ -f /var/run/openresty.pid ]; then
kill -USR1 `cat /var/run/openresty.pid`
fi
endscript
}
Start and enable OpenResty
Enable the service to start automatically and verify it's running.
sudo systemctl daemon-reload
sudo systemctl enable openresty
sudo systemctl start openresty
sudo systemctl status openresty
Configure upstream backend monitoring
Create advanced health check configuration
Implement custom health check logic with detailed monitoring.
# Advanced health check monitoring
server {
listen 8091;
server_name localhost;
location /health-check {
access_log off;
allow 127.0.0.1;
allow 203.0.113.0/24;
deny all;
content_by_lua_block {
local json = require "cjson"
local hc = require "resty.upstream.healthcheck"
local upstream = require "ngx.upstream"
local function check_backend(host, port)
local sock = ngx.socket.tcp()
sock:settimeout(1000)
local ok, err = sock:connect(host, port)
if not ok then
return false, err
end
local req = "GET /health HTTP/1.0\r\nHost: " .. host .. "\r\n\r\n"
local bytes, err = sock:send(req)
if not bytes then
sock:close()
return false, err
end
local line, err = sock:receive("*l")
sock:close()
if line and string.find(line, "200") then
return true, "healthy"
else
return false, "unhealthy response"
end
end
local backends = {
{host = "203.0.113.10", port = 8080},
{host = "203.0.113.11", port = 8080},
{host = "203.0.113.12", port = 8080}
}
local status = {}
local healthy_count = 0
local total_count = #backends
for _, backend in ipairs(backends) do
local is_healthy, msg = check_backend(backend.host, backend.port)
status[backend.host .. ":" .. backend.port] = {
healthy = is_healthy,
message = msg,
checked_at = ngx.now()
}
if is_healthy then
healthy_count = healthy_count + 1
end
end
local response = {
timestamp = ngx.now(),
total_backends = total_count,
healthy_backends = healthy_count,
status = status,
load_balancer_healthy = healthy_count > 0
}
ngx.header.content_type = "application/json"
ngx.say(json.encode(response))
}
}
}
Add performance monitoring
Configure metrics collection for load balancing performance.
server {
listen 8092;
server_name localhost;
location /metrics {
access_log off;
allow 127.0.0.1;
allow 203.0.113.0/24;
deny all;
content_by_lua_block {
local json = require "cjson"
local upstream = require "ngx.upstream"
-- Get basic upstream statistics
local ups = upstream.get_servers("backend_servers")
local metrics = {
timestamp = ngx.now(),
upstream_servers = {},
requests_total = 0,
active_connections = 0
}
if ups then
for _, server in ipairs(ups) do
local server_info = {
address = server.name or (server.addr .. ":" .. server.port),
weight = server.weight or 1,
max_fails = server.max_fails or 1,
fail_timeout = server.fail_timeout or 10,
backup = server.backup or false,
down = server.down or false
}
table.insert(metrics.upstream_servers, server_info)
end
end
ngx.header.content_type = "application/json"
ngx.say(json.encode(metrics))
}
}
location /prometheus-metrics {
access_log off;
allow 127.0.0.1;
allow 203.0.113.0/24;
deny all;
content_by_lua_block {
local upstream = require "ngx.upstream"
ngx.say("# HELP openresty_upstream_servers_total Total number of upstream servers")
ngx.say("# TYPE openresty_upstream_servers_total gauge")
local ups = upstream.get_servers("backend_servers")
local total_servers = 0
local active_servers = 0
if ups then
for _, server in ipairs(ups) do
total_servers = total_servers + 1
if not server.down then
active_servers = active_servers + 1
end
end
end
ngx.say(string.format("openresty_upstream_servers_total{upstream=\"backend_servers\"} %d", total_servers))
ngx.say(string.format("openresty_upstream_servers_active{upstream=\"backend_servers\"} %d", active_servers))
}
}
}
Implement automatic failover mechanisms
Configure backup server
Add a backup server that activates when all primary servers fail.
# Enhanced upstream with backup server
upstream backend_servers_with_backup {
# Primary servers
server 203.0.113.10:8080 max_fails=3 fail_timeout=30s;
server 203.0.113.11:8080 max_fails=3 fail_timeout=30s;
server 203.0.113.12:8080 max_fails=3 fail_timeout=30s;
# Backup server - only used when all primary servers are down
server 203.0.113.20:8080 backup;
# Load balancing settings
least_conn;
keepalive 32;
keepalive_requests 100;
keepalive_timeout 60s;
}
server {
listen 81;
server_name example.com;
# Enhanced error handling
error_page 502 503 504 /maintenance.html;
location = /maintenance.html {
root /usr/local/openresty/nginx/html;
internal;
}
location / {
# Try backup upstream if primary fails
proxy_pass http://backend_servers_with_backup;
# Aggressive failover settings
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_next_upstream_tries 5;
proxy_next_upstream_timeout 15s;
# Shorter timeouts for faster failover
proxy_connect_timeout 3s;
proxy_send_timeout 5s;
proxy_read_timeout 15s;
# Headers
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Failover debugging
add_header X-Upstream-Server $upstream_addr always;
add_header X-Response-Time $upstream_response_time always;
add_header X-Upstream-Status $upstream_status always;
}
}
Create maintenance page
Design a user-friendly maintenance page for complete outages.
Automated install script
Run this to automate the entire setup
#!/usr/bin/env bash
set -euo pipefail
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Default backend servers (can be overridden with arguments)
BACKEND_SERVERS=("203.0.113.10:8080" "203.0.113.11:8080" "203.0.113.12:8080")
# Usage function
usage() {
echo "Usage: $0 [backend_server1:port] [backend_server2:port] [backend_server3:port]"
echo "Example: $0 192.168.1.10:8080 192.168.1.11:8080 192.168.1.12:8080"
exit 1
}
# Error handling with cleanup
cleanup() {
echo -e "${RED}[ERROR] Installation failed. Cleaning up...${NC}"
systemctl stop openresty 2>/dev/null || true
exit 1
}
trap cleanup ERR
# Check if running as root
check_root() {
if [[ $EUID -ne 0 ]]; then
echo -e "${RED}This script must be run as root${NC}"
exit 1
fi
}
# Auto-detect distribution
detect_distro() {
if [ -f /etc/os-release ]; then
. /etc/os-release
case "$ID" in
ubuntu|debian)
PKG_MGR="apt"
PKG_INSTALL="apt install -y"
PKG_UPDATE="apt update"
WEB_USER="www-data"
;;
almalinux|rocky|centos|rhel|ol|fedora)
PKG_MGR="dnf"
PKG_INSTALL="dnf install -y"
PKG_UPDATE="dnf update -y"
WEB_USER="nginx"
;;
amzn)
PKG_MGR="yum"
PKG_INSTALL="yum install -y"
PKG_UPDATE="yum update -y"
WEB_USER="nginx"
;;
*)
echo -e "${RED}Unsupported distribution: $ID${NC}"
exit 1
;;
esac
else
echo -e "${RED}Cannot detect distribution. /etc/os-release not found${NC}"
exit 1
fi
}
# Parse command line arguments
parse_args() {
if [[ $# -gt 0 ]]; then
BACKEND_SERVERS=("$@")
fi
# Validate backend server format
for server in "${BACKEND_SERVERS[@]}"; do
if [[ ! $server =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+$ ]]; then
echo -e "${RED}Invalid backend server format: $server${NC}"
usage
fi
done
}
# Install OpenResty based on distribution
install_openresty() {
echo -e "${GREEN}[1/6] Installing OpenResty...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
# Ubuntu/Debian
wget -qO - https://openresty.org/package/pubkey.gpg | apt-key add -
echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/openresty.list
$PKG_UPDATE
$PKG_INSTALL openresty
else
# RHEL-based
$PKG_INSTALL yum-utils
if [[ "$PKG_MGR" == "dnf" ]]; then
dnf config-manager --add-repo https://openresty.org/package/centos/openresty.repo
else
yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo
fi
$PKG_INSTALL openresty
fi
}
# Install health check modules
install_healthcheck_modules() {
echo -e "${GREEN}[2/6] Installing health check modules...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
$PKG_INSTALL lua-resty-upstream-healthcheck lua-resty-lock
else
# For RHEL-based systems, we need to install from source or use opm
/usr/local/openresty/bin/opm get spacewander/lua-resty-upstream-healthcheck
/usr/local/openresty/bin/opm get openresty/lua-resty-lock
fi
}
# Create directory structure
create_directories() {
echo -e "${GREEN}[3/6] Creating directory structure...${NC}"
mkdir -p /usr/local/openresty/nginx/conf/conf.d
mkdir -p /usr/local/openresty/nginx/logs
mkdir -p /var/log/openresty
# Set proper ownership and permissions
chown -R $WEB_USER:$WEB_USER /var/log/openresty
chmod 755 /var/log/openresty
chmod 755 /usr/local/openresty/nginx/conf/conf.d
}
# Configure OpenResty
configure_openresty() {
echo -e "${GREEN}[4/6] Configuring OpenResty...${NC}"
# Generate upstream servers configuration
local upstream_servers=""
for server in "${BACKEND_SERVERS[@]}"; do
upstream_servers="$upstream_servers server $server max_fails=0 fail_timeout=0;\n"
done
# Create main configuration file
cat > /usr/local/openresty/nginx/conf/nginx.conf << EOF
user $WEB_USER;
worker_processes auto;
error_log /var/log/openresty/error.log warn;
pid /var/run/openresty.pid;
events {
worker_connections 1024;
use epoll;
multi_accept on;
}
http {
include /usr/local/openresty/nginx/conf/mime.types;
default_type application/octet-stream;
log_format main '\$remote_addr - \$remote_user [\$time_local] "\$request" '
'\$status \$body_bytes_sent "\$http_referer" '
'"\$http_user_agent" "\$http_x_forwarded_for" '
'upstream: \$upstream_addr response_time: \$upstream_response_time';
access_log /var/log/openresty/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# Shared memory for health checks
lua_shared_dict healthcheck 1m;
lua_shared_dict locks 1m;
# Load health check module
lua_package_path "/usr/local/openresty/lualib/?.lua;;";
init_worker_by_lua_block {
local hc = require "resty.upstream.healthcheck"
local ok, err = hc.spawn_checker {
shm = "healthcheck",
upstream = "backend_servers",
type = "http",
http_req = "GET /health HTTP/1.0\\r\\nHost: backend\\r\\n\\r\\n",
interval = 2000,
timeout = 1000,
fall = 3,
rise = 2,
valid_statuses = {200, 302},
concurrency = 10,
}
if not ok then
ngx.log(ngx.ERR, "failed to spawn health checker: ", err)
return
end
}
# Upstream backend servers
upstream backend_servers {
$(echo -e "$upstream_servers")
least_conn;
keepalive 32;
}
# Health check status endpoint
server {
listen 8090;
location /status {
access_log off;
allow 127.0.0.1;
deny all;
content_by_lua_block {
local hc = require "resty.upstream.healthcheck"
ngx.say("Upstream backend_servers:")
ngx.say(hc.status_page())
}
}
}
# Main load balancer server
server {
listen 80;
location / {
proxy_pass http://backend_servers;
proxy_set_header Host \$host;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
proxy_connect_timeout 5s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
}
}
}
EOF
# Set proper permissions
chmod 644 /usr/local/openresty/nginx/conf/nginx.conf
chown root:root /usr/local/openresty/nginx/conf/nginx.conf
}
# Create systemd service
create_systemd_service() {
echo -e "${GREEN}[5/6] Creating systemd service...${NC}"
cat > /etc/systemd/system/openresty.service << EOF
[Unit]
Description=OpenResty
After=syslog.target network-online.target remote-fs.target nss-lookup.target
Wants=network-online.target
[Service]
Type=forking
PIDFile=/var/run/openresty.pid
ExecStartPre=/usr/local/openresty/bin/openresty -t
ExecStart=/usr/local/openresty/bin/openresty
ExecReload=/bin/kill -s HUP \$MAINPID
KillMode=mixed
KillSignal=SIGQUIT
PrivateTmp=true
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable openresty
systemctl start openresty
}
# Configure firewall
configure_firewall() {
echo -e "${GREEN}[6/6] Configuring firewall...${NC}"
if command -v firewall-cmd >/dev/null 2>&1; then
# RHEL-based firewall
firewall-cmd --permanent --add-service=http
firewall-cmd --permanent --add-port=8090/tcp
firewall-cmd --reload
elif command -v ufw >/dev/null 2>&1; then
# Ubuntu/Debian firewall
ufw allow 80/tcp
ufw allow 8090/tcp
fi
}
# Verify installation
verify_installation() {
echo -e "${YELLOW}Verifying installation...${NC}"
# Check if OpenResty is running
if systemctl is-active --quiet openresty; then
echo -e "${GREEN}✓ OpenResty service is running${NC}"
else
echo -e "${RED}✗ OpenResty service is not running${NC}"
exit 1
fi
# Test configuration
if /usr/local/openresty/bin/openresty -t >/dev/null 2>&1; then
echo -e "${GREEN}✓ OpenResty configuration is valid${NC}"
else
echo -e "${RED}✗ OpenResty configuration is invalid${NC}"
exit 1
fi
echo -e "${GREEN}Installation completed successfully!${NC}"
echo -e "${YELLOW}Load balancer is available at: http://$(hostname -I | awk '{print $1}')${NC}"
echo -e "${YELLOW}Health check status at: http://$(hostname -I | awk '{print $1}'):8090/status${NC}"
echo -e "${YELLOW}Backend servers configured: ${BACKEND_SERVERS[*]}${NC}"
}
# Main execution
main() {
check_root
detect_distro
parse_args "$@"
install_openresty
install_healthcheck_modules
create_directories
configure_openresty
create_systemd_service
configure_firewall
verify_installation
}
main "$@"
Review the script before running. Execute with: bash install.sh