Configure Nginx Redis cluster caching for high availability and performance optimization

Install Redis cluster dependencies

Install Redis server and cluster tools on all designated Redis nodes. We'll create a 6-node cluster with 3 masters and 3 replicas for optimal redundancy.

sudo apt update
sudo apt install -y redis-server redis-tools build-essential

sudo dnf update -y
sudo dnf install -y redis redis-tools gcc make

Configure Redis cluster nodes

Configure each Redis instance for cluster mode. Repeat this configuration on all 6 nodes, changing the port for each instance.

port 7001
cluster-enabled yes
cluster-config-file nodes-7001.conf
cluster-node-timeout 15000
appendonly yes
bind 0.0.0.0
protected-mode no
dir /var/lib/redis/7001
logfile /var/log/redis/redis-7001.log
pidfile /var/run/redis/redis-7001.pid
daemonize yes
maxmemory 2gb
maxmemory-policy allkeys-lru
tcp-keepalive 60
save 900 1
save 300 10
save 60 10000

Create Redis data directories

Create the required directories for each Redis instance and set proper ownership.

sudo mkdir -p /var/lib/redis/7001 /var/lib/redis/7002 /var/lib/redis/7003
sudo mkdir -p /var/log/redis /var/run/redis
sudo chown -R redis:redis /var/lib/redis /var/log/redis /var/run/redis
sudo chmod 755 /var/lib/redis/700{1,2,3}

Start Redis cluster instances

Start all Redis instances across your cluster nodes. Run these commands on each server with the appropriate configuration files.

sudo redis-server /etc/redis/redis-7001.conf
sudo redis-server /etc/redis/redis-7002.conf
sudo redis-server /etc/redis/redis-7003.conf

Verify instances are running
sudo netstat -tulpn | grep redis

Initialize Redis cluster

Create the Redis cluster with 3 master nodes and 3 replica nodes. Replace the IP addresses with your actual server IPs.

redis-cli --cluster create \
  203.0.113.10:7001 203.0.113.11:7001 203.0.113.12:7001 \
  203.0.113.10:7002 203.0.113.11:7002 203.0.113.12:7002 \
  --cluster-replicas 1

Type 'yes' when prompted to accept the cluster configuration

Install Nginx Redis module

Install Nginx with the Redis module for caching capabilities. The redis2-nginx-module provides Redis cluster support.

sudo apt install -y nginx-module-redis2 nginx-module-set-misc nginx-module-echo
sudo nginx -t

sudo dnf install -y nginx nginx-mod-redis2 nginx-mod-set-misc
sudo nginx -t

Configure Nginx Redis caching

Configure Nginx to use Redis cluster for caching with automatic failover and load balancing across cluster nodes.

load_module modules/ngx_http_redis2_module.so;
load_module modules/ngx_http_set_misc_module.so;
load_module modules/ngx_http_echo_module.so;

events {
    worker_connections 1024;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;
    
    # Redis cluster upstream
    upstream redis_cluster {
        server 203.0.113.10:7001 max_fails=3 fail_timeout=30s;
        server 203.0.113.11:7001 max_fails=3 fail_timeout=30s;
        server 203.0.113.12:7001 max_fails=3 fail_timeout=30s;
        keepalive 32;
    }
    
    # Backend application servers
    upstream app_backend {
        server 203.0.113.20:8080;
        server 203.0.113.21:8080;
        keepalive 16;
    }
    
    # Cache configuration
    redis2_query_timeout 5s;
    redis2_connect_timeout 3s;
    redis2_read_timeout 5s;
    redis2_send_timeout 3s;
    
    # Logging
    log_format cache_log '$remote_addr - $remote_user [$time_local] '
                        '"$request" $status $body_bytes_sent '
                        '"$http_referer" "$http_user_agent" '
                        'cache_status=$upstream_cache_status '
                        'response_time=$request_time';
    
    server {
        listen 80;
        server_name example.com;
        access_log /var/log/nginx/cache.log cache_log;
        
        # Cache key generation
        set $cache_key "$scheme$request_method$host$request_uri";
        
        location / {
            # Try cache first
            access_by_lua_block {
                local cache_key = ngx.var.cache_key
                local res = ngx.location.capture("/redis_get", {
                    args = { key = cache_key }
                })
                
                if res.status == 200 and res.body ~= "" and res.body ~= "$"-1" then
                    ngx.header.content_type = "application/json"
                    ngx.header["X-Cache-Status"] = "HIT"
                    ngx.say(res.body)
                    ngx.exit(200)
                end
            }
            
            # Cache miss - proxy to backend
            proxy_pass http://app_backend;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            
            # Cache the response
            access_by_lua_block {
                local cache_key = ngx.var.cache_key
                local cache_ttl = 300  -- 5 minutes
                
                -- Store in Redis cluster
                ngx.location.capture("/redis_set", {
                    args = {
                        key = cache_key,
                        value = ngx.var.response_body,
                        expire = cache_ttl
                    }
                })
            }
            
            add_header X-Cache-Status "MISS";
        }
        
        # Redis GET operation
        location /redis_get {
            internal;
            redis2_query get $arg_key;
            redis2_pass redis_cluster;
        }
        
        # Redis SET operation with expiration
        location /redis_set {
            internal;
            redis2_query setex $arg_key $arg_expire $arg_value;
            redis2_pass redis_cluster;
        }
        
        # Cache invalidation endpoint
        location /cache/invalidate {
            allow 203.0.113.0/24;
            deny all;
            
            redis2_query del $arg_key;
            redis2_pass redis_cluster;
            add_header Content-Type text/plain;
            return 200 "Cache invalidated";
        }
        
        # Health check endpoint
        location /health {
            access_log off;
            return 200 "OK";
            add_header Content-Type text/plain;
        }
    }
}

Configure cache optimization settings

Create an optimized caching configuration with intelligent TTL management and cache warming strategies.

# Cache zones for different content types
proxy_cache_path /var/cache/nginx/api levels=1:2 keys_zone=api_cache:10m max_size=1g inactive=60m use_temp_path=off;
proxy_cache_path /var/cache/nginx/static levels=1:2 keys_zone=static_cache:10m max_size=2g inactive=24h use_temp_path=off;

Cache key optimization
map $request_uri $cache_ttl {
    ~*/api/users    300;   # 5 minutes for user data
    ~*/api/products 1800;  # 30 minutes for product data
    ~*/api/config   3600;  # 1 hour for configuration
    default         600;   # 10 minutes default
}

map $http_accept $content_type_suffix {
    ~*json  ".json";
    ~*xml   ".xml";
    default "";
}

Intelligent cache bypass
map $http_cache_control $bypass_cache {
    ~*no-cache     1;
    ~*no-store     1;
    ~*private      1;
    default        0;
}

Cache warming for popular endpoints
map $request_uri $should_warm {
    ~/api/popular/.  1;
    ~/api/trending/. 1;
    default            0;
}

Set up cache monitoring and metrics

Configure monitoring to track cache performance, hit rates, and Redis cluster health.

# Metrics collection for cache performance
location /nginx_status {
    stub_status on;
    access_log off;
    allow 127.0.0.1;
    allow 203.0.113.0/24;
    deny all;
}

Redis cluster health check
location /redis_health {
    access_log off;
    allow 127.0.0.1;
    allow 203.0.113.0/24;
    deny all;
    
    content_by_lua_block {
        local redis = require "resty.redis"
        local red = redis:new()
        red:set_timeouts(1000, 1000, 1000)
        
        local nodes = {
            {"203.0.113.10", 7001},
            {"203.0.113.11", 7001},
            {"203.0.113.12", 7001}
        }
        
        local healthy_nodes = 0
        for _, node in ipairs(nodes) do
            local ok, err = red:connect(node[1], node[2])
            if ok then
                local res, err = red:ping()
                if res == "PONG" then
                    healthy_nodes = healthy_nodes + 1
                end
                red:close()
            end
        end
        
        ngx.header.content_type = "application/json"
        ngx.say('{"healthy_nodes":' .. healthy_nodes .. ',"total_nodes":' .. #nodes .. '}')
    }
}

Create cache directories and set permissions

Create the necessary cache directories with proper permissions for Nginx worker processes.

sudo mkdir -p /var/cache/nginx/{api,static}
sudo chown -R nginx:nginx /var/cache/nginx
sudo chmod 755 /var/cache/nginx /var/cache/nginx/{api,static}

Configure systemd services

Set up systemd services for Redis cluster instances to ensure automatic startup and proper management.

[Unit]
Description=Redis Cluster Instance %i
After=network.target

[Service]
Type=forking
User=redis
Group=redis
ExecStart=/usr/bin/redis-server /etc/redis/redis-%i.conf
ExecStop=/bin/kill -s QUIT $MAINPID
TimeoutStopSec=0
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

Enable and start all services

Enable the Redis cluster instances and Nginx to start automatically on boot.

sudo systemctl daemon-reload
sudo systemctl enable redis-cluster@7001 redis-cluster@7002 redis-cluster@7003
sudo systemctl start redis-cluster@7001 redis-cluster@7002 redis-cluster@7003
sudo systemctl enable nginx
sudo systemctl restart nginx

Configure Redis memory optimization

Optimize Redis memory settings for high-performance caching with appropriate eviction policies.

# Add to each Redis configuration file
echo "tcp-backlog 511" | sudo tee -a /etc/redis/redis-7001.conf
echo "timeout 300" | sudo tee -a /etc/redis/redis-7001.conf
echo "tcp-keepalive 60" | sudo tee -a /etc/redis/redis-7001.conf
echo "maxclients 10000" | sudo tee -a /etc/redis/redis-7001.conf

Restart Redis instances
sudo systemctl restart redis-cluster@7001

Optimize Nginx worker configuration

Configure Nginx workers and connections for optimal performance with Redis cluster caching.

worker_processes auto;
worker_rlimit_nofile 65535;

events {
    worker_connections 4096;
    use epoll;
    multi_accept on;
}

http {
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    keepalive_requests 1000;
    
    # Gzip compression
    gzip on;
    gzip_vary on;
    gzip_min_length 1024;
    gzip_types text/plain text/css application/json application/javascript text/xml application/xml;
    
    # Connection pooling to Redis
    upstream_keepalive_requests 1000;
    upstream_keepalive_timeout 60s;
}

Test automatic failover

Simulate node failures to verify automatic failover functionality in your Redis cluster.

# Check cluster status before testing
redis-cli -c -h 203.0.113.10 -p 7001 cluster nodes

Stop one master node to test failover
sudo systemctl stop redis-cluster@7001

Verify cluster rebalancing
redis-cli -c -h 203.0.113.11 -p 7001 cluster nodes

Test cache operations still work
curl -H "Cache-Control: no-cache" http://example.com/api/test

Restart the stopped node
sudo systemctl start redis-cluster@7001

Monitor cluster health

Set up continuous monitoring to track cluster health and cache performance metrics.

# Create monitoring script
sudo tee /usr/local/bin/monitor-redis-cluster.sh << 'EOF'
#!/bin/bash
LOG_FILE="/var/log/redis/cluster-health.log"
DATE=$(date '+%Y-%m-%d %H:%M:%S')

for port in 7001 7002 7003; do
    for host in 203.0.113.10 203.0.113.11 203.0.113.12; do
        if redis-cli -h $host -p $port ping > /dev/null 2>&1; then
            echo "$DATE - $host:$port OK" >> $LOG_FILE
        else
            echo "$DATE - $host:$port FAILED" >> $LOG_FILE
        fi
    done
done

Check cluster state
redis-cli -c -h 203.0.113.10 -p 7001 cluster info >> $LOG_FILE
EOF

sudo chmod +x /usr/local/bin/monitor-redis-cluster.sh

Add to crontab for every 2 minutes
echo "/2    * /usr/local/bin/monitor-redis-cluster.sh" | sudo crontab -