Set up Varnish 7 cluster with load balancing across multiple backends

Update system packages

Start by ensuring all system packages are current to get the latest security updates.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install Varnish Cache 7

Add the official Varnish repository and install the latest stable version with development tools.

curl -fsSL https://packagecloud.io/varnishcache/varnish70/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/varnish.gpg
echo "deb [signed-by=/usr/share/keyrings/varnish.gpg] https://packagecloud.io/varnishcache/varnish70/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/varnish.list
sudo apt update
sudo apt install -y varnish varnish-dev

sudo dnf install -y epel-release
sudo rpm --import https://packagecloud.io/varnishcache/varnish70/gpgkey
echo "[varnish70]
name=Varnish Cache 7.0
baseurl=https://packagecloud.io/varnishcache/varnish70/el/\$releasever/\$basearch
enabled=1
gpgcheck=1
gpgkey=https://packagecloud.io/varnishcache/varnish70/gpgkey" | sudo tee /etc/yum.repos.d/varnish.repo
sudo dnf install -y varnish varnish-devel

Configure systemd service parameters

Create a systemd override file to configure Varnish with proper memory allocation and network binding.

sudo mkdir -p /etc/systemd/system/varnish.service.d
sudo tee /etc/systemd/system/varnish.service.d/override.conf > /dev/null << 'EOF'
[Service]
ExecStart=
ExecStart=/usr/sbin/varnishd -a :80 -a :443,PROXY -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s malloc,4G -p thread_pool_min=100 -p thread_pool_max=5000 -p listen_depth=2048
EOF

Create cluster configuration with multiple backends

Configure the main VCL file with backend definitions, health checks, and load balancing logic.

vcl 4.1;

import std;
import directors;

# Backend web servers
backend web1 {
    .host = "10.0.1.10";
    .port = "8080";
    .probe = {
        .url = "/health";
        .timeout = 2s;
        .interval = 10s;
        .window = 5;
        .threshold = 3;
        .initial = 3;
    };
    .connect_timeout = 2s;
    .first_byte_timeout = 10s;
    .between_bytes_timeout = 2s;
    .max_connections = 300;
}

backend web2 {
    .host = "10.0.1.11";
    .port = "8080";
    .probe = {
        .url = "/health";
        .timeout = 2s;
        .interval = 10s;
        .window = 5;
        .threshold = 3;
        .initial = 3;
    };
    .connect_timeout = 2s;
    .first_byte_timeout = 10s;
    .between_bytes_timeout = 2s;
    .max_connections = 300;
}

backend web3 {
    .host = "10.0.1.12";
    .port = "8080";
    .probe = {
        .url = "/health";
        .timeout = 2s;
        .interval = 10s;
        .window = 5;
        .threshold = 3;
        .initial = 3;
    };
    .connect_timeout = 2s;
    .first_byte_timeout = 10s;
    .between_bytes_timeout = 2s;
    .max_connections = 300;
}

# Initialize load balancer with round-robin algorithm
sub vcl_init {
    new vdir = directors.round_robin();
    vdir.add_backend(web1);
    vdir.add_backend(web2);
    vdir.add_backend(web3);
}

sub vcl_recv {
    # Set backend director
    set req.backend_hint = vdir.backend();
    
    # Handle SSL termination
    if (std.port(server.ip) == 443) {
        set req.http.X-Forwarded-Proto = "https";
        set req.http.X-Forwarded-Port = "443";
    } else {
        set req.http.X-Forwarded-Proto = "http";
        set req.http.X-Forwarded-Port = "80";
    }
    
    # Set real client IP
    if (req.http.X-Real-IP) {
        set req.http.X-Forwarded-For = req.http.X-Real-IP;
    } else {
        set req.http.X-Forwarded-For = client.ip;
    }
    
    # Remove cookies for static assets
    if (req.url ~ "\.(css|js|png|gif|jp(e)?g|swf|ico|pdf|mov|fla|zip|rar)$") {
        unset req.http.cookie;
        return(hash);
    }
    
    # Handle purge requests from trusted IPs
    if (req.method == "PURGE") {
        if (client.ip !~ purge_acl) {
            return(synth(405, "Purging not allowed for " + client.ip));
        }
        return(purge);
    }
    
    # Normalize Accept-Encoding
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
            unset req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            unset req.http.Accept-Encoding;
        }
    }
    
    return(hash);
}

# ACL for purge requests
acl purge_acl {
    "127.0.0.1";
    "10.0.1.0/24";
}

sub vcl_backend_response {
    # Set cache TTL based on content type
    if (beresp.http.content-type ~ "^(text/css|application/javascript|text/javascript)") {
        set beresp.ttl = 1w;
        set beresp.http.Cache-Control = "public, max-age=604800";
    }
    
    if (beresp.http.content-type ~ "^image/") {
        set beresp.ttl = 1d;
        set beresp.http.Cache-Control = "public, max-age=86400";
    }
    
    # Handle backend errors
    if (beresp.status >= 400) {
        set beresp.ttl = 0s;
        set beresp.grace = 15s;
        return(deliver);
    }
    
    # Enable grace mode for stale content
    set beresp.grace = 1h;
    
    # Remove backend server headers
    unset beresp.http.Server;
    unset beresp.http.X-Powered-By;
    
    return(deliver);
}

sub vcl_deliver {
    # Add cache status headers
    if (obj.hits > 0) {
        set resp.http.X-Cache = "HIT";
        set resp.http.X-Cache-Hits = obj.hits;
    } else {
        set resp.http.X-Cache = "MISS";
    }
    
    # Add backend information for debugging
    set resp.http.X-Served-By = server.hostname;
    
    # Security headers
    set resp.http.X-Frame-Options = "SAMEORIGIN";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-XSS-Protection = "1; mode=block";
    
    # Remove internal headers
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Age;
    
    return(deliver);
}

sub vcl_backend_error {
    # Serve stale content if backend is down
    if (beresp.status >= 500 && beresp.status < 600) {
        return(abandon);
    }
    
    return(deliver);
}

Configure SSL termination with HAProxy

Install HAProxy to handle SSL termination and proxy to Varnish cluster nodes.

sudo apt install -y haproxy

sudo dnf install -y haproxy

Create HAProxy configuration for SSL and clustering

Configure HAProxy to terminate SSL and distribute traffic across multiple Varnish nodes.

global
    log 127.0.0.1:514 local0
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon
    
    # SSL Configuration
    ssl-default-bind-ciphers ECDHE+AESGCM:ECDHE+CHACHA20:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
    ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
    
    # Performance tuning
    tune.ssl.default-dh-param 2048
    tune.bufsize 32768
    tune.maxrewrite 8192

defaults
    mode http
    log global
    option httplog
    option dontlognull
    option log-health-checks
    option forwardfor
    option http-server-close
    
    timeout connect 5000
    timeout client 50000
    timeout server 50000
    
    errorfile 400 /etc/haproxy/errors/400.http
    errorfile 403 /etc/haproxy/errors/403.http
    errorfile 408 /etc/haproxy/errors/408.http
    errorfile 500 /etc/haproxy/errors/500.http
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    errorfile 504 /etc/haproxy/errors/504.http

frontend varnish_cluster
    bind *:80
    bind *:443 ssl crt /etc/ssl/certs/example.com.pem
    
    # Security headers
    http-response set-header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"
    http-response set-header X-Frame-Options "SAMEORIGIN"
    http-response set-header X-Content-Type-Options "nosniff"
    
    # Redirect HTTP to HTTPS
    redirect scheme https if !{ ssl_fc }
    
    # Health check endpoint
    acl health_check path_beg /haproxy-health
    http-request return status 200 content-type text/plain string "OK\n" if health_check
    
    # Rate limiting
    stick-table type ip size 100k expire 30s store http_req_rate(10s),http_err_rate(10s)
    http-request track-sc0 src
    http-request reject if { sc_http_req_rate(0) gt 50 }
    
    default_backend varnish_nodes

backend varnish_nodes
    balance roundrobin
    option httpchk GET /health HTTP/1.1\r\nHost:\ localhost
    
    # Varnish cluster nodes
    server varnish1 10.0.2.10:80 check inter 2000 rise 2 fall 3 weight 100
    server varnish2 10.0.2.11:80 check inter 2000 rise 2 fall 3 weight 100
    server varnish3 10.0.2.12:80 check inter 2000 rise 2 fall 3 weight 100
    
    # Health check configuration
    http-check expect status 200
    http-check connect

listen stats
    bind *:8404
    stats enable
    stats uri /
    stats refresh 10s
    stats admin if TRUE

Configure shared cache storage with Redis

Install Redis for cache invalidation coordination between Varnish nodes.

sudo apt install -y redis-server

sudo dnf install -y redis

Create Redis configuration for cache coordination

Configure Redis to handle cache invalidation messages between cluster nodes with persistence and security.

# Network
bind 127.0.0.1 10.0.2.10
port 6379
tcp-backlog 511
tcp-keepalive 300

# Memory and persistence
maxmemory 512mb
maxmemory-policy allkeys-lru
save 900 1
save 300 10
save 60 10000
rdbcompression yes
rdbchecksum yes
dbfilename varnish-cache.rdb
dir /var/lib/redis

# Logging
loglevel notice
logfile /var/log/redis/redis-server.log

# Security
requirepass varnish_cluster_redis_2024
rename-command FLUSHDB ""
rename-command FLUSHALL ""
rename-command DEBUG ""
rename-command CONFIG "CONFIG_b835f3d4c2a7"

# Performance
timeout 300
tcp-keepalive 60
databases 16

Install and configure cache purge scripts

Create scripts to handle coordinated cache purging across all Varnish nodes in the cluster.

#!/bin/bash

# Varnish Cluster Cache Purge Script
# Usage: ./varnish-purge-cluster.sh [URL_PATTERN]

set -euo pipefail

# Configuration
VARNISH_NODES=("10.0.2.10:80" "10.0.2.11:80" "10.0.2.12:80")
REDIS_HOST="10.0.2.10"
REDIS_PORT="6379"
REDIS_PASS="varnish_cluster_redis_2024"
LOG_FILE="/var/log/varnish-cluster-purge.log"

# Functions
log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

purge_varnish_node() {
    local node="$1"
    local url_pattern="$2"
    local host="${node%:*}"
    local port="${node#*:}"
    
    if curl -sf -X PURGE "http://${host}:${port}${url_pattern}" > /dev/null 2>&1; then
        log_message "Successfully purged ${url_pattern} on ${node}"
        return 0
    else
        log_message "Failed to purge ${url_pattern} on ${node}"
        return 1
    fi
}

notify_redis() {
    local url_pattern="$1"
    local message="{\"action\":\"purge\",\"pattern\":\"${url_pattern}\",\"timestamp\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}"
    
    redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" -a "$REDIS_PASS" PUBLISH varnish:purge "$message" > /dev/null 2>&1
}

# Main execution
URL_PATTERN="${1:-/*}"

log_message "Starting cluster purge for pattern: $URL_PATTERN"

# Purge all nodes
success_count=0
for node in "${VARNISH_NODES[@]}"; do
    if purge_varnish_node "$node" "$URL_PATTERN"; then
        ((success_count++))
    fi
done

# Notify via Redis
notify_redis "$URL_PATTERN"

log_message "Purge completed. Success: $success_count/${#VARNISH_NODES[@]} nodes"

if [ "$success_count" -eq "${#VARNISH_NODES[@]}" ]; then
    exit 0
else
    exit 1
fi

Set script permissions and create monitoring

Configure proper permissions for the purge script and set up basic monitoring.

sudo chmod 755 /usr/local/bin/varnish-purge-cluster.sh
sudo mkdir -p /var/log
sudo touch /var/log/varnish-cluster-purge.log
sudo chown varnish:varnish /var/log/varnish-cluster-purge.log
sudo chmod 644 /var/log/varnish-cluster-purge.log

Configure Prometheus monitoring integration

Install varnish_exporter to provide metrics for monitoring and alerting.

wget https://github.com/jonnenauha/prometheus_varnish_exporter/releases/download/1.6.1/prometheus_varnish_exporter-1.6.1.linux-amd64.tar.gz
tar xzf prometheus_varnish_exporter-1.6.1.linux-amd64.tar.gz
sudo mv prometheus_varnish_exporter-1.6.1.linux-amd64/prometheus_varnish_exporter /usr/local/bin/
sudo chmod 755 /usr/local/bin/prometheus_varnish_exporter

Create systemd service for metrics exporter

Configure the Varnish Prometheus exporter to run as a system service with proper monitoring configuration.

[Unit]
Description=Prometheus Varnish Exporter
After=network.target varnish.service
Requires=varnish.service

[Service]
Type=simple
User=varnish
Group=varnish
ExecStart=/usr/local/bin/prometheus_varnish_exporter \
    -web.listen-address=:9131 \
    -varnish.instance= \
    -web.telemetry-path=/metrics \
    -verbose
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

Configure firewall rules

Open required ports for the Varnish cluster, HAProxy, and monitoring services.

# Allow HTTP/HTTPS traffic
sudo ufw allow 80/tcp
sudo ufw allow 443/tcp

# Allow Varnish admin interface (restrict to management network)
sudo ufw allow from 10.0.0.0/8 to any port 6082

# Allow HAProxy stats (restrict to management network)
sudo ufw allow from 10.0.0.0/8 to any port 8404

# Allow Redis (cluster communication)
sudo ufw allow from 10.0.2.0/24 to any port 6379

# Allow Prometheus metrics
sudo ufw allow from 10.0.0.0/8 to any port 9131

# Allow inter-cluster communication
sudo ufw allow from 10.0.2.0/24 to any port 80

Start and enable all services

Enable and start all components of the Varnish cluster in the correct order.

# Reload systemd and start Redis
sudo systemctl daemon-reload
sudo systemctl enable --now redis-server

# Start Varnish with new configuration
sudo systemctl enable --now varnish

# Start HAProxy for SSL termination
sudo systemctl enable --now haproxy

# Start metrics exporter
sudo systemctl enable --now varnish-exporter

# Verify all services are running
sudo systemctl status redis-server varnish haproxy varnish-exporter

Implement weighted round-robin with sticky sessions

Create an advanced VCL configuration that supports session persistence and weighted backend selection.

vcl 4.1;

import std;
import directors;
import cookie;

# Include main backends from default.vcl
include "backends.vcl";

# Advanced director with weighted round-robin
sub vcl_init {
    # Main director with weights
    new vdir = directors.round_robin();
    vdir.add_backend(web1, 3.0);  # 3x weight
    vdir.add_backend(web2, 2.0);  # 2x weight  
    vdir.add_backend(web3, 1.0);  # 1x weight
    
    # Fallback director for when backends fail
    new fallback = directors.fallback();
    fallback.add_backend(web1);
    fallback.add_backend(web2);
    fallback.add_backend(web3);
}

sub vcl_recv {
    # Session persistence based on cookie
    if (req.http.Cookie ~ "JSESSIONID=([^;]+)") {
        set req.http.X-Session-ID = regsub(req.http.Cookie, ".*JSESSIONID=([^;]+).*", "\1");
        
        # Hash session ID to determine backend
        set req.backend_hint = vdir.backend(std.fnv1a(req.http.X-Session-ID));
    } else {
        # Use weighted round-robin for new sessions
        set req.backend_hint = vdir.backend();
    }
    
    # Fallback if primary backend is unhealthy
    if (!std.healthy(req.backend_hint)) {
        set req.backend_hint = fallback.backend();
    }
    
    # Geographic routing based on client IP
    if (client.ip ~ geo_eu) {
        set req.http.X-Geo-Region = "EU";
    } elsif (client.ip ~ geo_us) {
        set req.http.X-Geo-Region = "US";
    } else {
        set req.http.X-Geo-Region = "OTHER";
    }
    
    return(hash);
}

# Geographic ACLs
acl geo_eu {
    "185.0.0.0/8";     # European IP range example
    "78.0.0.0/8";
}

acl geo_us {
    "192.0.0.0/8";     # US IP range example
    "198.0.0.0/8";
}

Create separate backend definitions file

Organize backend configurations in a separate file for easier management.