Set up HAProxy with Consul integration for automatic service discovery, health checking, and dynamic backend updates. This tutorial covers consul-template configuration for zero-downtime scaling and failover in microservices architectures.

Prerequisites

Root access to server
Basic understanding of load balancing concepts
Familiarity with systemd services

What this solves

Manual HAProxy backend management becomes complex in dynamic environments where services frequently scale up or down. This tutorial integrates HAProxy with Consul for automatic service discovery, allowing your load balancer to dynamically update backends based on service registration and health checks. You'll eliminate manual configuration changes and achieve zero-downtime scaling for microservices deployments.

Step-by-step configuration

Update system packages and install dependencies

Start by updating your system and installing required packages for HAProxy and Consul integration.

sudo apt update && sudo apt upgrade -y
sudo apt install -y haproxy consul unzip curl wget

sudo dnf update -y
sudo dnf install -y haproxy consul unzip curl wget

Install consul-template

Download and install consul-template, which will automatically update HAProxy configuration based on Consul service changes.

CONSUL_TEMPLATE_VERSION="0.34.0"
wget https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip
unzip consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip
sudo mv consul-template /usr/local/bin/
sudo chmod +x /usr/local/bin/consul-template
consul-template --version

Configure Consul server

Set up Consul with a basic configuration for service discovery. This creates a single-node setup suitable for development or small environments.

datacenter = "dc1"
data_dir = "/opt/consul"
log_level = "INFO"
server = true
bootstrap_expect = 1
bind_addr = "0.0.0.0"
client_addr = "0.0.0.0"
retry_join = ["127.0.0.1"]
ui_config {
  enabled = true
}
connect {
  enabled = true
}
ports {
  grpc = 8502
}

Create Consul data directory and set permissions

Create the required directories and set proper ownership for the Consul service.

sudo mkdir -p /opt/consul
sudo useradd --system --home /etc/consul --shell /bin/false consul
sudo chown -R consul:consul /etc/consul /opt/consul
sudo chmod 755 /etc/consul /opt/consul

Create Consul systemd service

Configure Consul to run as a systemd service with proper security settings.

[Unit]
Description=Consul
Documentation=https://www.consul.io/
Requires=network-online.target
After=network-online.target
ConditionFileNotEmpty=/etc/consul/consul.hcl

[Service]
Type=notify
User=consul
Group=consul
ExecStart=/usr/bin/consul agent -config-dir=/etc/consul
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

Start and enable Consul

Start the Consul service and verify it's running correctly.

sudo systemctl daemon-reload
sudo systemctl enable --now consul
sudo systemctl status consul
consul members

Configure HAProxy base configuration

Create the base HAProxy configuration that will be extended by consul-template.

global
    log stdout local0
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    mode http
    log global
    option httplog
    option dontlognull
    option http-server-close
    option forwardfor except 127.0.0.0/8
    option redispatch
    retries 3
    timeout http-request 10s
    timeout queue 1m
    timeout connect 10s
    timeout client 1m
    timeout server 1m
    timeout http-keep-alive 10s
    timeout check 10s
    maxconn 3000

frontend stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s

frontend web_frontend
    bind *:80
    default_backend web_servers

Backend will be dynamically generated by consul-template

Create consul-template configuration

Configure consul-template to monitor Consul services and update HAProxy backend configuration automatically.

consul {
  address = "127.0.0.1:8500"
  retry {
    enabled = true
    attempts = 12
    backoff = "250ms"
    max_backoff = "1m"
  }
}

template {
  source = "/etc/consul-template/haproxy.ctmpl"
  destination = "/etc/haproxy/haproxy.cfg"
  create_dest_dirs = true
  command = "systemctl reload haproxy"
  command_timeout = "60s"
  error_on_missing_key = false
  perms = 0644
  backup = true
  left_delimiter  = "{{"
  right_delimiter = "}}"
  wait {
    min = "5s"
    max = "10s"
  }
}

Create HAProxy template for dynamic backends

Create the template file that consul-template will use to generate HAProxy configuration with dynamic backends.

global
    log stdout local0
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    mode http
    log global
    option httplog
    option dontlognull
    option http-server-close
    option forwardfor except 127.0.0.0/8
    option redispatch
    retries 3
    timeout http-request 10s
    timeout queue 1m
    timeout connect 10s
    timeout client 1m
    timeout server 1m
    timeout http-keep-alive 10s
    timeout check 10s
    maxconn 3000

frontend stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s

frontend web_frontend
    bind *:80
    default_backend web_servers

backend web_servers
    balance roundrobin
    option httpchk GET /health
{{range service "web"}}
    server {{.Name}}-{{.ID}} {{.Address}}:{{.Port}} check inter 5s fall 3 rise 2{{end}}

Create consul-template directories and set permissions

Create necessary directories and set proper permissions for consul-template operation.

sudo mkdir -p /etc/consul-template
sudo chown -R haproxy:haproxy /etc/consul-template
sudo chmod 755 /etc/consul-template
sudo chmod 644 /etc/consul-template/haproxy.hcl /etc/consul-template/haproxy.ctmpl

Create consul-template systemd service

Configure consul-template to run as a systemd service that monitors Consul for service changes.

[Unit]
Description=Consul Template
Documentation=https://github.com/hashicorp/consul-template
Requires=network-online.target consul.service
After=network-online.target consul.service
ConditionFileNotEmpty=/etc/consul-template/haproxy.hcl

[Service]
Type=notify
User=haproxy
Group=haproxy
ExecStart=/usr/local/bin/consul-template -config=/etc/consul-template/haproxy.hcl
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

Enable and start services

Start HAProxy and consul-template services, ensuring they start automatically on boot.

sudo systemctl daemon-reload
sudo systemctl enable --now haproxy consul-template
sudo systemctl status haproxy consul-template

Register a test service in Consul

Create a sample service registration to test the integration. This simulates a web service that HAProxy will load balance.

{
  "ID": "web-1",
  "Name": "web",
  "Tags": ["primary", "v1"],
  "Address": "203.0.113.10",
  "Port": 8080,
  "Check": {
    "HTTP": "http://203.0.113.10:8080/health",
    "Interval": "10s",
    "Timeout": "3s"
  }
}

consul services register /tmp/web-service.json

Configure firewall rules

Open necessary ports for HAProxy, Consul, and service communication.

sudo ufw allow 80/tcp comment "HAProxy HTTP"
sudo ufw allow 8404/tcp comment "HAProxy Stats"
sudo ufw allow 8500/tcp comment "Consul HTTP API"
sudo ufw allow 8301/tcp comment "Consul Serf LAN"
sudo ufw allow 8301/udp comment "Consul Serf LAN"
sudo ufw reload

sudo firewall-cmd --permanent --add-port=80/tcp --add-port=8404/tcp --add-port=8500/tcp --add-port=8301/tcp --add-port=8301/udp
sudo firewall-cmd --reload

Verify your setup

Check that all services are running and the integration is working correctly.

# Verify Consul is running and services are registered
consul catalog services
consul catalog nodes

Check HAProxy configuration was updated
sudo systemctl status haproxy consul-template
grep -A 10 "backend web_servers" /etc/haproxy/haproxy.cfg

Test HAProxy stats page
curl -s http://localhost:8404/stats

Verify Consul web UI access
curl -s http://localhost:8500/ui/

Note: The HAProxy configuration file should now contain dynamically generated backend servers based on services registered in Consul. Any changes to service registration will trigger automatic updates.

Advanced configuration options

Configure service health check integration

Enhance the template to use Consul health check status for HAProxy backend availability.

backend web_servers
    balance roundrobin
    option httpchk GET /health
{{range service "web"}}
    {{if .Tags | contains "primary"}}
    server {{.Name}}-{{.ID}} {{.Address}}:{{.Port}} check inter 5s fall 3 rise 2 weight 100{{else}}
    server {{.Name}}-{{.ID}} {{.Address}}:{{.Port}} check inter 5s fall 3 rise 2 weight 50 backup{{end}}{{end}}

{{if service "api"}}
backend api_servers
    balance leastconn
    option httpchk GET /api/health
{{range service "api"}}
    server {{.Name}}-{{.ID}} {{.Address}}:{{.Port}} check inter 10s fall 2 rise 3{{end}}
{{end}}

Add service tagging for routing

Configure advanced routing based on Consul service tags for blue-green deployments.

{
  "ID": "web-2",
  "Name": "web",
  "Tags": ["secondary", "v2"],
  "Address": "203.0.113.11",
  "Port": 8080,
  "Check": {
    "HTTP": "http://203.0.113.11:8080/health",
    "Interval": "10s",
    "Timeout": "3s"
  }
}

consul services register /tmp/web-service-v2.json

Common issues

Symptom	Cause	Fix
consul-template not updating HAProxy	Permission denied on config file	`sudo chown haproxy:haproxy /etc/haproxy/haproxy.cfg`
HAProxy reload fails	Invalid configuration syntax	Check template with `haproxy -f /etc/haproxy/haproxy.cfg -c`
Services not appearing in HAProxy	Consul service name mismatch	Verify service name in template matches registration
Health checks failing	Consul can't reach service endpoints	Check network connectivity and health check URLs
consul-template service fails to start	Missing Consul connection	Verify Consul is running: `consul members`

Never use chmod 777. It gives every user on the system full access to your files. Instead, fix ownership with chown and use minimal permissions like 644 for config files.

Next steps

Automated install script

Run this to automate the entire setup

install.sh

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Global variables
CONSUL_TEMPLATE_VERSION="0.34.0"
CONSUL_USER="consul"
SCRIPT_DIR="/tmp/haproxy-consul-install"

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -h, --help     Show this help message"
    echo "  --consul-dc    Consul datacenter name (default: dc1)"
    exit 1
}

# Parse arguments
CONSUL_DC="dc1"
while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            usage
            ;;
        --consul-dc)
            CONSUL_DC="$2"
            shift 2
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Cleanup function
cleanup() {
    echo -e "${RED}Installation failed. Cleaning up...${NC}"
    rm -rf "$SCRIPT_DIR"
    exit 1
}

trap cleanup ERR

# Check if running as root or with sudo
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root or with sudo${NC}"
    exit 1
fi

# Detect distribution
echo -e "${YELLOW}[1/12] Detecting distribution...${NC}"
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_UPDATE="apt update && apt upgrade -y"
            PKG_INSTALL="apt install -y"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora) 
            PKG_MGR="dnf"
            PKG_UPDATE="dnf update -y"
            PKG_INSTALL="dnf install -y"
            ;;
        amzn) 
            PKG_MGR="yum"
            PKG_UPDATE="yum update -y"
            PKG_INSTALL="yum install -y"
            ;;
        *) 
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}Cannot detect distribution${NC}"
    exit 1
fi

echo -e "${GREEN}Detected: $PRETTY_NAME${NC}"

# Update system packages
echo -e "${YELLOW}[2/12] Updating system packages...${NC}"
$PKG_UPDATE

# Install dependencies
echo -e "${YELLOW}[3/12] Installing dependencies...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL haproxy unzip curl wget
    # Install consul from HashiCorp repository
    curl -fsSL https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
    echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/hashicorp.list
    apt update
    $PKG_INSTALL consul
else
    $PKG_INSTALL haproxy unzip curl wget
    # Install consul from HashiCorp repository
    dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
    $PKG_INSTALL consul
fi

# Create script directory
mkdir -p "$SCRIPT_DIR"
cd "$SCRIPT_DIR"

# Install consul-template
echo -e "${YELLOW}[4/12] Installing consul-template...${NC}"
wget -q "https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip"
unzip -q "consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip"
mv consul-template /usr/local/bin/
chmod 755 /usr/local/bin/consul-template

# Create consul user and directories
echo -e "${YELLOW}[5/12] Creating consul user and directories...${NC}"
if ! id "$CONSUL_USER" &>/dev/null; then
    useradd --system --home /etc/consul --shell /bin/false --create-home "$CONSUL_USER"
fi
mkdir -p /opt/consul /etc/consul /etc/consul-template /run/haproxy
chown -R consul:consul /etc/consul /opt/consul
chmod 755 /etc/consul /opt/consul

# Configure Consul
echo -e "${YELLOW}[6/12] Configuring Consul...${NC}"
cat > /etc/consul/consul.hcl << EOF
datacenter = "${CONSUL_DC}"
data_dir = "/opt/consul"
log_level = "INFO"
server = true
bootstrap_expect = 1
bind_addr = "0.0.0.0"
client_addr = "0.0.0.0"
retry_join = ["127.0.0.1"]
ui_config {
  enabled = true
}
connect {
  enabled = true
}
ports {
  grpc = 8502
}
EOF

chown consul:consul /etc/consul/consul.hcl
chmod 640 /etc/consul/consul.hcl

# Create Consul systemd service
echo -e "${YELLOW}[7/12] Creating Consul systemd service...${NC}"
cat > /etc/systemd/system/consul.service << 'EOF'
[Unit]
Description=Consul
Documentation=https://www.consul.io/
Requires=network-online.target
After=network-online.target
ConditionFileNotEmpty=/etc/consul/consul.hcl

[Service]
Type=notify
User=consul
Group=consul
ExecStart=/usr/bin/consul agent -config-dir=/etc/consul
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

# Configure HAProxy base configuration
echo -e "${YELLOW}[8/12] Configuring HAProxy...${NC}"
cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

cat > /etc/haproxy/haproxy.cfg << 'EOF'
global
    log stdout local0
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    mode http
    log global
    option httplog
    option dontlognull
    option http-server-close
    option forwardfor except 127.0.0.0/8
    option redispatch
    retries 3
    timeout http-request 10s
    timeout queue 1m
    timeout connect 10s
    timeout client 1m
    timeout server 1m
    timeout http-keep-alive 10s
    timeout check 10s
    maxconn 3000

frontend stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s

frontend web_frontend
    bind *:80
    default_backend web_servers

# Backend will be dynamically generated by consul-template
backend web_servers
    balance roundrobin
EOF

# Create consul-template configuration
echo -e "${YELLOW}[9/12] Configuring consul-template...${NC}"
cat > /etc/consul-template/haproxy.conf << 'EOF'
consul {
  address = "127.0.0.1:8500"
  retry {
    enabled = true
    attempts = 12
    backoff = "250ms"
    max_backoff = "1m"
  }
}

template {
  source = "/etc/consul-template/haproxy.ctmpl"
  destination = "/etc/haproxy/haproxy.cfg"
  command = "systemctl reload haproxy"
  command_timeout = "60s"
  perms = 0644
}
EOF

# Create HAProxy template
cat > /etc/consul-template/haproxy.ctmpl << 'EOF'
global
    log stdout local0
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    mode http
    log global
    option httplog
    option dontlognull
    option http-server-close
    option forwardfor except 127.0.0.0/8
    option redispatch
    retries 3
    timeout http-request 10s
    timeout queue 1m
    timeout connect 10s
    timeout client 1m
    timeout server 1m
    timeout http-keep-alive 10s
    timeout check 10s
    maxconn 3000

frontend stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s

frontend web_frontend
    bind *:80
    default_backend web_servers

backend web_servers
    balance roundrobin
    {{range service "web"}}
    server {{.Node}}-{{.ID}} {{.Address}}:{{.Port}} check{{end}}
EOF

chmod 644 /etc/consul-template/haproxy.conf /etc/consul-template/haproxy.ctmpl

# Create consul-template systemd service
echo -e "${YELLOW}[10/12] Creating consul-template systemd service...${NC}"
cat > /etc/systemd/system/consul-template.service << 'EOF'
[Unit]
Description=consul-template
Requires=network-online.target
After=network-online.target consul.service
Wants=consul.service

[Service]
Type=notify
ExecStart=/usr/local/bin/consul-template -config=/etc/consul-template/haproxy.conf
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
KillSignal=SIGINT

[Install]
WantedBy=multi-user.target
EOF

# Start and enable services
echo -e "${YELLOW}[11/12] Starting and enabling services...${NC}"
systemctl daemon-reload
systemctl enable --now consul
systemctl enable --now haproxy
systemctl enable --now consul-template

# Wait for consul to be ready
sleep 5

# Verification
echo -e "${YELLOW}[12/12] Verifying installation...${NC}"
if systemctl is-active --quiet consul; then
    echo -e "${GREEN}✓ Consul is running${NC}"
else
    echo -e "${RED}✗ Consul failed to start${NC}"
    exit 1
fi

if systemctl is-active --quiet haproxy; then
    echo -e "${GREEN}✓ HAProxy is running${NC}"
else
    echo -e "${RED}✗ HAProxy failed to start${NC}"
    exit 1
fi

if systemctl is-active --quiet consul-template; then
    echo -e "${GREEN}✓ Consul-template is running${NC}"
else
    echo -e "${RED}✗ Consul-template failed to start${NC}"
    exit 1
fi

# Cleanup
rm -rf "$SCRIPT_DIR"

echo -e "${GREEN}Installation completed successfully!${NC}"
echo -e "${GREEN}Access points:${NC}"
echo "  - Consul UI: http://$(hostname -I | awk '{print $1}'):8500"
echo "  - HAProxy Stats: http://$(hostname -I | awk '{print $1}'):8404/stats"
echo ""
echo -e "${YELLOW}To register a service with Consul:${NC}"
echo "curl -X PUT http://127.0.0.1:8500/v1/agent/service/register -d '{\"ID\":\"web1\",\"Name\":\"web\",\"Address\":\"192.168.1.100\",\"Port\":8080}'"

Review the script before running. Execute with: bash install.sh

#haproxy #consul #service-discovery #load-balancing #microservices

Configure HAProxy with Consul for dynamic service discovery and automatic backend updates