Implement OpenResty rate limiting and API protection with Lua middleware

Install OpenResty with required modules

First, add the OpenResty repository and install the core package with Lua modules.

wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add -
echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/openresty.list
sudo apt update
sudo apt install -y openresty openresty-resty openresty-opm

sudo yum install -y yum-utils
sudo yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo
sudo yum install -y openresty openresty-resty openresty-opm

Install Redis for rate limiting storage

Redis serves as the high-performance backend for storing rate limit counters and API protection data.

sudo apt install -y redis-server redis-tools

sudo dnf install -y redis

Install Lua Redis module

Install the lua-resty-redis module that allows OpenResty to communicate with Redis efficiently.

sudo opm get ledgetech/lua-resty-redis
sudo opm get pintsized/lua-resty-redis-connector
sudo opm get openresty/lua-resty-limit-traffic

Configure Redis for rate limiting

Configure Redis with optimized settings for high-throughput rate limiting operations.

# Memory optimization for rate limiting
maxmemory 256mb
maxmemory-policy allkeys-lru

# Performance tuning
tcp-keepalive 60
timeout 300
tcp-backlog 511

# Persistence settings for rate limit data
save 900 1
save 300 10
save 60 10000

# Security
bind 127.0.0.1
requirepass your_redis_password_here
protected-mode yes

Create rate limiting Lua module

Create a comprehensive Lua module that implements multiple rate limiting algorithms.

local redis = require "resty.redis"
local limit_req = require "resty.limit.req"
local limit_conn = require "resty.limit.conn"
local cjson = require "cjson"

local _M = {}

-- Redis connection configuration
local redis_config = {
    host = "127.0.0.1",
    port = 6379,
    password = "your_redis_password_here",
    timeout = 1000,
    pool_size = 100
}

-- Rate limiting policies
local policies = {
    api_strict = { requests = 100, window = 60, burst = 10 },
    api_normal = { requests = 500, window = 60, burst = 50 },
    api_premium = { requests = 2000, window = 60, burst = 200 },
    default = { requests = 50, window = 60, burst = 5 }
}

-- Initialize Redis connection
function _M.init_redis()
    local red = redis:new()
    red:set_timeout(redis_config.timeout)
    
    local ok, err = red:connect(redis_config.host, redis_config.port)
    if not ok then
        ngx.log(ngx.ERR, "Redis connection failed: ", err)
        return nil, err
    end
    
    if redis_config.password then
        local res, err = red:auth(redis_config.password)
        if not res then
            ngx.log(ngx.ERR, "Redis auth failed: ", err)
            return nil, err
        end
    end
    
    return red
end

-- Token bucket rate limiting
function _M.token_bucket_limit(key, policy_name, identifier)
    local policy = policies[policy_name] or policies.default
    local red, err = _M.init_redis()
    
    if not red then
        return false, "Redis connection failed"
    end
    
    local bucket_key = "bucket:" .. key .. ":" .. identifier
    local now = ngx.now()
    
    -- Lua script for atomic token bucket operations
    local script = [[
        local key = KEYS[1]
        local capacity = tonumber(ARGV[1])
        local tokens = tonumber(ARGV[2])
        local window = tonumber(ARGV[3])
        local now = tonumber(ARGV[4])
        
        local bucket = redis.call('HMGET', key, 'tokens', 'last_refill')
        local current_tokens = tonumber(bucket[1]) or capacity
        local last_refill = tonumber(bucket[2]) or now
        
        -- Calculate tokens to add based on elapsed time
        local elapsed = now - last_refill
        local tokens_to_add = math.floor(elapsed * (capacity / window))
        current_tokens = math.min(capacity, current_tokens + tokens_to_add)
        
        if current_tokens >= tokens then
            current_tokens = current_tokens - tokens
            redis.call('HMSET', key, 'tokens', current_tokens, 'last_refill', now)
            redis.call('EXPIRE', key, window * 2)
            return {1, current_tokens, capacity}
        else
            redis.call('HMSET', key, 'tokens', current_tokens, 'last_refill', now)
            redis.call('EXPIRE', key, window * 2)
            return {0, current_tokens, capacity}
        end
    ]]
    
    local res, err = red:eval(script, 1, bucket_key, policy.requests, 1, policy.window, now)
    
    if not res then
        ngx.log(ngx.ERR, "Token bucket script failed: ", err)
        return false, "Rate limit check failed"
    end
    
    red:set_keepalive(10000, redis_config.pool_size)
    
    local allowed = res[1] == 1
    local remaining = res[2]
    local total = res[3]
    
    return allowed, remaining, total
end

-- Sliding window rate limiting
function _M.sliding_window_limit(key, policy_name, identifier)
    local policy = policies[policy_name] or policies.default
    local red, err = _M.init_redis()
    
    if not red then
        return false, "Redis connection failed"
    end
    
    local window_key = "window:" .. key .. ":" .. identifier
    local now = ngx.now()
    local window_start = now - policy.window
    
    -- Lua script for sliding window rate limiting
    local script = [[
        local key = KEYS[1]
        local window_start = tonumber(ARGV[1])
        local now = tonumber(ARGV[2])
        local limit = tonumber(ARGV[3])
        local window_size = tonumber(ARGV[4])
        
        -- Remove old entries
        redis.call('ZREMRANGEBYSCORE', key, 0, window_start)
        
        -- Count current requests in window
        local current = redis.call('ZCARD', key)
        
        if current < limit then
            -- Add current request
            redis.call('ZADD', key, now, now .. ':' .. math.random(1000000))
            redis.call('EXPIRE', key, window_size * 2)
            return {1, limit - current - 1, limit}
        else
            return {0, 0, limit}
        end
    ]]
    
    local res, err = red:eval(script, 1, window_key, window_start, now, policy.requests, policy.window)
    
    if not res then
        ngx.log(ngx.ERR, "Sliding window script failed: ", err)
        return false, "Rate limit check failed"
    end
    
    red:set_keepalive(10000, redis_config.pool_size)
    
    local allowed = res[1] == 1
    local remaining = res[2]
    local total = res[3]
    
    return allowed, remaining, total
end

-- API key validation and rate limiting
function _M.validate_api_key(api_key)
    if not api_key then
        return false, "Missing API key", "default"
    end
    
    local red, err = _M.init_redis()
    if not red then
        return false, "Service unavailable", "default"
    end
    
    local key_info = red:hmget("api_key:" .. api_key, "valid", "policy", "user_id")
    red:set_keepalive(10000, redis_config.pool_size)
    
    if not key_info[1] or key_info[1] == ngx.null then
        return false, "Invalid API key", "default"
    end
    
    local policy = key_info[2] or "default"
    local user_id = key_info[3] or api_key
    
    return true, user_id, policy
end

-- Geolocation-based blocking
function _M.check_geo_blocking()
    local ip = ngx.var.remote_addr
    local country = ngx.var.geoip_country_code or "XX"
    
    local blocked_countries = {"CN", "RU", "KP"}  -- Example blocked countries
    
    for _, blocked in ipairs(blocked_countries) do
        if country == blocked then
            ngx.log(ngx.WARN, "Blocked request from country: ", country, " IP: ", ip)
            return false, "Access denied from your location"
        end
    end
    
    return true, "OK"
end

-- Bot detection based on User-Agent patterns
function _M.detect_bots()
    local user_agent = ngx.var.http_user_agent or ""
    
    local bot_patterns = {
        "bot", "crawler", "spider", "scraper", "curl", "wget",
        "python-requests", "postman", "insomnia"
    }
    
    local ua_lower = string.lower(user_agent)
    
    for _, pattern in ipairs(bot_patterns) do
        if string.find(ua_lower, pattern) then
            ngx.log(ngx.INFO, "Bot detected: ", user_agent)
            return true, "Bot traffic"
        end
    end
    
    return false, "Human traffic"
end

-- Request size limiting
function _M.check_request_size(max_size)
    local content_length = ngx.var.content_length
    
    if content_length then
        local size = tonumber(content_length)
        if size and size > max_size then
            return false, "Request too large"
        end
    end
    
    return true, "OK"
end

return _M

Configure OpenResty main configuration

Set up the main OpenResty configuration with Lua path and shared dictionaries for caching.

user www-data;
worker_processes auto;
worker_rlimit_nofile 65535;

events {
    worker_connections 8192;
    use epoll;
    multi_accept on;
}

http {
    include       mime.types;
    default_type  application/octet-stream;
    
    # Lua configuration
    lua_package_path "/usr/local/openresty/lualib/?.lua;/usr/local/openresty/lualib/?/init.lua;;";
    lua_code_cache on;
    
    # Shared dictionaries for caching
    lua_shared_dict rate_limit_cache 10m;
    lua_shared_dict api_keys_cache 5m;
    lua_shared_dict geo_cache 2m;
    
    # Initialize rate limiting module
    init_by_lua_block {
        ratelimit = require "ratelimit"
    }
    
    # Logging format with rate limiting info
    log_format rate_limit '$remote_addr - $remote_user [$time_local] '
                         '"$request" $status $body_bytes_sent '
                         '"$http_referer" "$http_user_agent" '
                         'api_key="$api_key" policy="$rate_limit_policy" '
                         'remaining="$rate_limit_remaining"';
    
    # Basic security headers
    add_header X-Frame-Options DENY always;
    add_header X-Content-Type-Options nosniff always;
    add_header X-XSS-Protection "1; mode=block" always;
    
    # Rate limiting server block
    server {
        listen 80;
        server_name api.example.com;
        
        access_log /var/log/openresty/api_access.log rate_limit;
        error_log /var/log/openresty/api_error.log;
        
        # Variables for rate limiting context
        set $api_key "";
        set $rate_limit_policy "";
        set $rate_limit_remaining "";
        
        # Rate limiting and API protection
        access_by_lua_block {
            local method = ngx.var.request_method
            local uri = ngx.var.uri
            local ip = ngx.var.remote_addr
            
            -- Check request size (max 1MB for API requests)
            local size_ok, size_msg = ratelimit.check_request_size(1048576)
            if not size_ok then
                ngx.status = 413
                ngx.header["Content-Type"] = "application/json"
                ngx.say('{"error": "' .. size_msg .. '"}')
                ngx.exit(413)
            end
            
            -- Geolocation blocking
            local geo_ok, geo_msg = ratelimit.check_geo_blocking()
            if not geo_ok then
                ngx.status = 403
                ngx.header["Content-Type"] = "application/json"
                ngx.say('{"error": "' .. geo_msg .. '"}')
                ngx.exit(403)
            end
            
            -- Bot detection for API endpoints
            if string.match(uri, "^/api/") then
                local is_bot, bot_msg = ratelimit.detect_bots()
                if is_bot then
                    ngx.status = 429
                    ngx.header["Content-Type"] = "application/json"
                    ngx.say('{"error": "Bot traffic not allowed on API endpoints"}')
                    ngx.exit(429)
                end
            end
            
            -- API key validation
            local api_key = ngx.var.http_x_api_key or ngx.var.arg_api_key
            ngx.var.api_key = api_key or "none"
            
            local valid, user_id, policy = ratelimit.validate_api_key(api_key)
            if not valid then
                ngx.status = 401
                ngx.header["Content-Type"] = "application/json"
                ngx.say('{"error": "' .. user_id .. '"}')
                ngx.exit(401)
            end
            
            ngx.var.rate_limit_policy = policy
            
            -- Apply rate limiting based on endpoint
            local rate_limit_key
            local algorithm = "sliding_window"  -- or "token_bucket"
            
            if string.match(uri, "^/api/v1/") then
                rate_limit_key = "api_v1:" .. method
            elseif string.match(uri, "^/api/upload/") then
                rate_limit_key = "upload"
                algorithm = "token_bucket"  -- Better for upload endpoints
            else
                rate_limit_key = "general"
            end
            
            -- Execute rate limiting
            local allowed, remaining, total
            
            if algorithm == "token_bucket" then
                allowed, remaining, total = ratelimit.token_bucket_limit(rate_limit_key, policy, user_id)
            else
                allowed, remaining, total = ratelimit.sliding_window_limit(rate_limit_key, policy, user_id)
            end
            
            if not allowed then
                ngx.var.rate_limit_remaining = "0"
                ngx.status = 429
                ngx.header["Content-Type"] = "application/json"
                ngx.header["X-RateLimit-Limit"] = total
                ngx.header["X-RateLimit-Remaining"] = "0"
                ngx.header["X-RateLimit-Reset"] = ngx.time() + 60
                ngx.header["Retry-After"] = "60"
                ngx.say('{"error": "Rate limit exceeded", "limit": ' .. total .. ', "remaining": 0}')
                ngx.exit(429)
            end
            
            -- Set rate limiting headers for successful requests
            ngx.var.rate_limit_remaining = remaining
            ngx.header["X-RateLimit-Limit"] = total
            ngx.header["X-RateLimit-Remaining"] = remaining
        }
        
        # API endpoints
        location /api/ {
            proxy_pass http://backend;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-API-Key $api_key;
        }
        
        # Health check endpoint (no rate limiting)
        location /health {
            access_by_lua_block {
                -- Skip rate limiting for health checks
            }
            
            content_by_lua_block {
                ngx.header["Content-Type"] = "application/json"
                ngx.say('{"status": "healthy", "timestamp": ' .. ngx.time() .. '}')
            }
        }
        
        # Rate limiting status endpoint
        location /rate-limit-status {
            content_by_lua_block {
                local api_key = ngx.var.http_x_api_key or ngx.var.arg_api_key
                
                if not api_key then
                    ngx.status = 400
                    ngx.header["Content-Type"] = "application/json"
                    ngx.say('{"error": "API key required"}')
                    return
                end
                
                local valid, user_id, policy = ratelimit.validate_api_key(api_key)
                if not valid then
                    ngx.status = 401
                    ngx.header["Content-Type"] = "application/json"
                    ngx.say('{"error": "Invalid API key"}')
                    return
                end
                
                local red = ratelimit.init_redis()
                if red then
                    local bucket_key = "bucket:api_v1:GET:" .. user_id
                    local bucket = red:hmget(bucket_key, "tokens", "last_refill")
                    local tokens = bucket[1] or "N/A"
                    
                    red:set_keepalive(10000, 100)
                    
                    ngx.header["Content-Type"] = "application/json"
                    ngx.say('{"user_id": "' .. user_id .. '", "policy": "' .. policy .. '", "remaining_tokens": "' .. tokens .. '"}')
                else
                    ngx.status = 500
                    ngx.header["Content-Type"] = "application/json"
                    ngx.say('{"error": "Unable to check status"}')
                end
            }
        }
    }
    
    # Upstream backend servers
    upstream backend {
        server 127.0.0.1:8080 weight=1 max_fails=2 fail_timeout=5s;
        server 127.0.0.1:8081 weight=1 max_fails=2 fail_timeout=5s;
        keepalive 32;
    }
}