Implement Varnish cache warming with automated content preloading for high-performance websites

Update system packages

Start by updating your package manager to ensure you get the latest versions of required tools.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install cache warming dependencies

Install curl for making HTTP requests, jq for JSON parsing, and Python for advanced warming scripts.

sudo apt install -y curl jq python3 python3-pip python3-venv bc

sudo dnf install -y curl jq python3 python3-pip bc

Create cache warming directory structure

Set up organized directories for warming scripts, URL lists, logs, and configuration files.

sudo mkdir -p /opt/varnish-warming/{scripts,config,logs,urls}
sudo mkdir -p /var/log/varnish-warming

Create the main cache warming script

This script handles URL loading with priority levels, parallel processing, and detailed logging.

#!/bin/bash

# Varnish Cache Warming Script
# Usage: ./warm-cache.sh [priority_level] [max_parallel]

set -euo pipefail

# Configuration
CONFIG_DIR="/opt/varnish-warming/config"
LOG_DIR="/var/log/varnish-warming"
URL_DIR="/opt/varnish-warming/urls"
VARNISH_HOST="127.0.0.1"
VARNISH_PORT="80"
DEFAULT_TIMEOUT="30"
DEFAULT_PARALLEL="10"
USER_AGENT="VarnishWarming/1.0"

# Load configuration if exists
if [[ -f "$CONFIG_DIR/warming.conf" ]]; then
    source "$CONFIG_DIR/warming.conf"
fi

# Parameters
PRIORITY_LEVEL=${1:-"high"}
MAX_PARALLEL=${2:-$DEFAULT_PARALLEL}
LOG_FILE="$LOG_DIR/warming-$(date +%Y%m%d-%H%M%S).log"
STATS_FILE="$LOG_DIR/warming-stats.json"

# Logging function
log() {
    local level=$1
    shift
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] [$level] $*" | tee -a "$LOG_FILE"
}

# Warm single URL function
warm_url() {
    local url=$1
    local priority=${2:-"medium"}
    local start_time=$(date +%s.%N)
    
    # Add cache-busting headers for first request
    local response=$(curl -s -w "%{http_code},%{time_total},%{size_download}" \
        -H "Cache-Control: no-cache" \
        -H "Pragma: no-cache" \
        -H "User-Agent: $USER_AGENT" \
        -H "X-Cache-Warming: true" \
        --max-time "$DEFAULT_TIMEOUT" \
        -o /dev/null \
        "$url" 2>/dev/null || echo "000,0,0")
    
    local http_code=$(echo "$response" | cut -d',' -f1)
    local response_time=$(echo "$response" | cut -d',' -f2)
    local content_size=$(echo "$response" | cut -d',' -f3)
    
    # Make second request to populate cache
    if [[ "$http_code" == "200" ]]; then
        curl -s \
            -H "User-Agent: $USER_AGENT" \
            --max-time "$DEFAULT_TIMEOUT" \
            -o /dev/null \
            "$url" 2>/dev/null || true
    fi
    
    local end_time=$(date +%s.%N)
    local total_time=$(echo "$end_time - $start_time" | bc)
    
    # Log result
    if [[ "$http_code" == "200" ]]; then
        log "INFO" "SUCCESS: $url ($priority) - ${response_time}s, ${content_size} bytes"
        echo "success,$url,$priority,$http_code,$response_time,$content_size,$total_time"
    else
        log "WARN" "FAILED: $url ($priority) - HTTP $http_code"
        echo "failed,$url,$priority,$http_code,$response_time,$content_size,$total_time"
    fi
}

export -f warm_url
export -f log
export VARNISH_HOST VARNISH_PORT DEFAULT_TIMEOUT USER_AGENT LOG_FILE

# Main warming function
warm_cache() {
    local priority=$1
    local url_file="$URL_DIR/urls-$priority.txt"
    
    if [[ ! -f "$url_file" ]]; then
        log "WARN" "URL file not found: $url_file"
        return 1
    fi
    
    local url_count=$(wc -l < "$url_file")
    log "INFO" "Starting cache warming for $priority priority ($url_count URLs)"
    
    # Process URLs in parallel
    cat "$url_file" | grep -v '^#' | grep -v '^$' | \
    parallel -j "$MAX_PARALLEL" --will-cite \
        "warm_url {} $priority" > "/tmp/warming-results-$priority.csv"
    
    # Calculate statistics
    local success_count=$(grep -c '^success,' "/tmp/warming-results-$priority.csv" || echo 0)
    local failed_count=$(grep -c '^failed,' "/tmp/warming-results-$priority.csv" || echo 0)
    local success_rate=0
    
    if [[ $url_count -gt 0 ]]; then
        success_rate=$(echo "scale=2; $success_count * 100 / $url_count" | bc)
    fi
    
    log "INFO" "Completed $priority priority: $success_count/$url_count successful (${success_rate}%)"
    
    # Update stats file
    local stats=$(cat <

Create configuration file

Set up the main configuration file with customizable parameters for your environment.

# Varnish Cache Warming Configuration

# Varnish connection settings
VARNISH_HOST="127.0.0.1"
VARNISH_PORT="80"

# Request settings
DEFAULT_TIMEOUT="30"
DEFAULT_PARALLEL="15"
USER_AGENT="VarnishWarming/1.0"

# Retry settings
MAX_RETRIES="2"
RETRY_DELAY="1"

# Logging
LOG_RETENTION_DAYS="7"
DEBUG_MODE="false"

# Performance settings
RATE_LIMIT_DELAY="0.1"  # Seconds between requests to same host
MAX_CONCURRENT_HOSTS="5"  # Max parallel hosts to warm simultaneously

Create priority URL lists

Set up URL lists organized by priority levels for efficient cache warming scheduling.

# High priority URLs - Critical pages that need immediate caching
# Homepage and main landing pages
http://example.com/
http://example.com/index.html
http://example.com/about
http://example.com/contact

# Critical API endpoints
http://example.com/api/health
http://example.com/api/config

# Essential assets
http://example.com/css/main.css
http://example.com/js/app.js
http://example.com/images/logo.png

Create medium and low priority URL lists

Add additional URL lists for medium and low priority content to optimize cache warming coverage.

# Medium priority URLs - Important but not critical
# Product categories and popular content
http://example.com/products
http://example.com/services
http://example.com/blog
http://example.com/news

# User-facing pages
http://example.com/login
http://example.com/register
http://example.com/dashboard

# Common resources
http://example.com/css/theme.css
http://example.com/js/utils.js

# Low priority URLs - Nice to have cached
# Secondary pages and content
http://example.com/privacy
http://example.com/terms
http://example.com/help
http://example.com/faq

# Archive and historical content
http://example.com/archive
http://example.com/old-blog-posts

# Non-critical assets
http://example.com/images/background.jpg
http://example.com/fonts/custom.woff2

Create URL discovery script

This script automatically discovers URLs from sitemaps and access logs to maintain current URL lists.

#!/usr/bin/env python3

import requests
import xml.etree.ElementTree as ET
import json
import re
import sys
import argparse
from urllib.parse import urljoin, urlparse
from collections import defaultdict
import logging

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

class URLDiscovery:
    def __init__(self, base_url, output_dir='/opt/varnish-warming/urls'):
        self.base_url = base_url.rstrip('/')
        self.output_dir = output_dir
        self.discovered_urls = defaultdict(list)
        
    def discover_from_sitemap(self, sitemap_url=None):
        """Discover URLs from XML sitemap"""
        if not sitemap_url:
            sitemap_url = urljoin(self.base_url, '/sitemap.xml')
            
        try:
            response = requests.get(sitemap_url, timeout=30)
            response.raise_for_status()
            
            root = ET.fromstring(response.content)
            
            # Handle sitemap index
            if 'sitemapindex' in root.tag:
                for sitemap in root.findall('.//{http://www.sitemaps.org/schemas/sitemap/0.9}sitemap'):
                    loc = sitemap.find('{http://www.sitemaps.org/schemas/sitemap/0.9}loc')
                    if loc is not None:
                        self.discover_from_sitemap(loc.text)
            else:
                # Handle URL set
                for url in root.findall('.//{http://www.sitemaps.org/schemas/sitemap/0.9}url'):
                    loc = url.find('{http://www.sitemaps.org/schemas/sitemap/0.9}loc')
                    priority = url.find('{http://www.sitemaps.org/schemas/sitemap/0.9}priority')
                    
                    if loc is not None:
                        url_text = loc.text
                        priority_value = float(priority.text) if priority is not None else 0.5
                        
                        # Categorize by priority
                        if priority_value >= 0.8:
                            self.discovered_urls['high'].append(url_text)
                        elif priority_value >= 0.5:
                            self.discovered_urls['medium'].append(url_text)
                        else:
                            self.discovered_urls['low'].append(url_text)
                            
            logger.info(f"Discovered {sum(len(urls) for urls in self.discovered_urls.values())} URLs from sitemap")
            
        except Exception as e:
            logger.error(f"Error parsing sitemap {sitemap_url}: {e}")
    
    def discover_from_access_log(self, log_file, min_requests=5):
        """Discover popular URLs from access logs"""
        url_counts = defaultdict(int)
        
        try:
            with open(log_file, 'r') as f:
                for line in f:
                    # Parse common log format
                    match = re.search(r'"[A-Z]+ (\S+)', line)
                    if match:
                        path = match.group(1)
                        if not path.startswith('/'):
                            continue
                            
                        # Skip common non-cacheable paths
                        if any(skip in path for skip in ['.php', '.cgi', '/admin/', '/api/auth']):
                            continue
                            
                        url = urljoin(self.base_url, path)
                        url_counts[url] += 1
            
            # Categorize by popularity
            sorted_urls = sorted(url_counts.items(), key=lambda x: x[1], reverse=True)
            
            for url, count in sorted_urls:
                if count >= min_requests * 5:
                    self.discovered_urls['high'].append(url)
                elif count >= min_requests * 2:
                    self.discovered_urls['medium'].append(url)
                elif count >= min_requests:
                    self.discovered_urls['low'].append(url)
                    
            logger.info(f"Discovered {len(sorted_urls)} URLs from access logs")
            
        except Exception as e:
            logger.error(f"Error parsing access log {log_file}: {e}")
    
    def save_url_lists(self):
        """Save discovered URLs to priority files"""
        for priority, urls in self.discovered_urls.items():
            # Remove duplicates and sort
            unique_urls = list(set(urls))
            unique_urls.sort()
            
            filename = f"{self.output_dir}/urls-{priority}.txt"
            
            try:
                with open(filename, 'w') as f:
                    f.write(f"# {priority.title()} priority URLs - Auto-generated\n")
                    f.write(f"# Total URLs: {len(unique_urls)}\n\n")
                    
                    for url in unique_urls:
                        f.write(f"{url}\n")
                        
                logger.info(f"Saved {len(unique_urls)} {priority} priority URLs to {filename}")
                
            except Exception as e:
                logger.error(f"Error saving URL list {filename}: {e}")

def main():
    parser = argparse.ArgumentParser(description='Discover URLs for Varnish cache warming')
    parser.add_argument('base_url', help='Base URL of the website')
    parser.add_argument('--sitemap', help='Sitemap URL (default: /sitemap.xml)')
    parser.add_argument('--access-log', help='Path to access log file')
    parser.add_argument('--output-dir', default='/opt/varnish-warming/urls', help='Output directory')
    parser.add_argument('--min-requests', type=int, default=5, help='Minimum requests for log-based discovery')
    
    args = parser.parse_args()
    
    discovery = URLDiscovery(args.base_url, args.output_dir)
    
    # Discover from sitemap
    discovery.discover_from_sitemap(args.sitemap)
    
    # Discover from access logs if provided
    if args.access_log:
        discovery.discover_from_access_log(args.access_log, args.min_requests)
    
    # Save results
    discovery.save_url_lists()
    
    total_urls = sum(len(urls) for urls in discovery.discovered_urls.values())
    logger.info(f"URL discovery completed. Total URLs: {total_urls}")

if __name__ == '__main__':
    main()

Set proper permissions

Configure ownership and permissions for the warming scripts and directories. The scripts need execute permissions while config files should be readable by the service user.

sudo chmod +x /opt/varnish-warming/scripts/warm-cache.sh
sudo chmod +x /opt/varnish-warming/scripts/discover-urls.py
sudo chown -R root:root /opt/varnish-warming
sudo chmod 755 /opt/varnish-warming/scripts/
sudo chmod 644 /opt/varnish-warming/config/*
sudo chmod 644 /opt/varnish-warming/urls/*
sudo chown -R syslog:adm /var/log/varnish-warming
sudo chmod 755 /var/log/varnish-warming

Create systemd service for cache warming

Set up a systemd service to manage cache warming operations with proper resource limits and logging.

[Unit]
Description=Varnish Cache Warming Service
After=network.target varnish.service
Requires=varnish.service
Wants=network-online.target

[Service]
Type=oneshot
User=www-data
Group=www-data
ExecStart=/opt/varnish-warming/scripts/warm-cache.sh all 15
WorkingDirectory=/opt/varnish-warming
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
StandardOutput=journal
StandardError=journal

# Resource limits
MemoryMax=512M
CPUQuota=50%
TasksMax=50

# Security settings
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=yes
ReadWritePaths=/var/log/varnish-warming /tmp

# Timeout settings
TimeoutStartSec=300
TimeoutStopSec=30

[Install]
WantedBy=multi-user.target

Create systemd timer for scheduled warming

Configure automatic cache warming with systemd timers for consistent performance optimization.

[Unit]
Description=Varnish Cache Warming Timer
Requires=varnish-warming.service

[Timer]
# Run high priority warming every 30 minutes
OnCalendar=*:0/30
RandomizedDelaySec=60
Persistent=true

[Install]
WantedBy=timers.target

Create additional timer for full warming

Set up a separate timer for complete cache warming including all priority levels during off-peak hours.

[Unit]
Description=Varnish Full Cache Warming Service
After=network.target varnish.service
Requires=varnish.service

[Service]
Type=oneshot
User=www-data
Group=www-data
ExecStartPre=/opt/varnish-warming/scripts/discover-urls.py http://example.com
ExecStart=/opt/varnish-warming/scripts/warm-cache.sh all 20
WorkingDirectory=/opt/varnish-warming
StandardOutput=journal
StandardError=journal

# Resource limits for full warming
MemoryMax=1G
CPUQuota=75%
TasksMax=100

# Security settings
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=yes
ReadWritePaths=/var/log/varnish-warming /tmp /opt/varnish-warming/urls

TimeoutStartSec=1800
TimeoutStopSec=60

[Unit]
Description=Varnish Full Cache Warming Timer
Requires=varnish-warming-full.service

[Timer]
# Run full warming twice daily during off-peak hours
OnCale