Monitor HAProxy and Consul with Prometheus and Grafana dashboards

Update system packages

Start by updating your package manager to ensure you get the latest versions of all components.

sudo apt update && sudo apt upgrade -y
sudo apt install -y wget curl gnupg

sudo dnf update -y
sudo dnf install -y wget curl gnupg

Configure HAProxy metrics endpoint

Enable HAProxy's built-in statistics and metrics endpoint by modifying the main configuration file.

global
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s

defaults
    stats enable
    stats uri /stats
    stats refresh 30s
    stats admin if TRUE

frontend stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 5s
    stats admin if TRUE
    http-request use-service prometheus-exporter if { path /metrics }

frontend web_frontend
    bind *:80
    default_backend web_servers

backend web_servers
    balance roundrobin
    server web1 192.168.1.10:80 check
    server web2 192.168.1.11:80 check

Install HAProxy Exporter

Download and install the official HAProxy Exporter to collect detailed metrics from HAProxy's stats socket.

cd /tmp
wget https://github.com/prometheus/haproxy_exporter/releases/download/v0.15.0/haproxy_exporter-0.15.0.linux-amd64.tar.gz
tar xzf haproxy_exporter-0.15.0.linux-amd64.tar.gz
sudo mv haproxy_exporter-0.15.0.linux-amd64/haproxy_exporter /usr/local/bin/
sudo chown root:root /usr/local/bin/haproxy_exporter
sudo chmod 755 /usr/local/bin/haproxy_exporter

Create HAProxy Exporter service

Configure HAProxy Exporter as a systemd service to ensure it starts automatically and runs reliably.

[Unit]
Description=HAProxy Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=nobody
Group=nogroup
Type=simple
ExecStart=/usr/local/bin/haproxy_exporter \
  --haproxy.scrape-uri=unix:/run/haproxy/admin.sock \
  --web.listen-address=0.0.0.0:9101
SyslogIdentifier=haproxy_exporter
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

Configure Consul metrics

Enable Prometheus metrics in Consul by updating the configuration file with telemetry settings.

{
  "datacenter": "dc1",
  "data_dir": "/var/lib/consul",
  "log_level": "INFO",
  "server": true,
  "ui_config": {
    "enabled": true
  },
  "client_addr": "0.0.0.0",
  "bind_addr": "0.0.0.0",
  "telemetry": {
    "prometheus_retention_time": "30s",
    "disable_hostname": true
  },
  "ports": {
    "grpc": 8502
  },
  "connect": {
    "enabled": true
  }
}

Install Consul Exporter

Download and configure the Consul Exporter to collect additional service discovery and health check metrics.

cd /tmp
wget https://github.com/prometheus/consul_exporter/releases/download/v0.11.0/consul_exporter-0.11.0.linux-amd64.tar.gz
tar xzf consul_exporter-0.11.0.linux-amd64.tar.gz
sudo mv consul_exporter-0.11.0.linux-amd64/consul_exporter /usr/local/bin/
sudo chown root:root /usr/local/bin/consul_exporter
sudo chmod 755 /usr/local/bin/consul_exporter

Create Consul Exporter service

Set up the Consul Exporter as a systemd service with appropriate configuration for your environment.

[Unit]
Description=Consul Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=nobody
Group=nogroup
Type=simple
ExecStart=/usr/local/bin/consul_exporter \
  --consul.server=localhost:8500 \
  --web.listen-address=0.0.0.0:9107
SyslogIdentifier=consul_exporter
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

Configure Prometheus scraping targets

Add HAProxy and Consul metrics endpoints to your Prometheus configuration to start collecting metrics.

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "/etc/prometheus/rules/*.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - localhost:9093

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'haproxy'
    static_configs:
      - targets: ['localhost:9101']
    scrape_interval: 5s
    metrics_path: /metrics

  - job_name: 'haproxy-stats'
    static_configs:
      - targets: ['localhost:8404']
    scrape_interval: 5s
    metrics_path: /metrics

  - job_name: 'consul'
    static_configs:
      - targets: ['localhost:8500']
    scrape_interval: 5s
    metrics_path: /v1/agent/metrics
    params:
      format: ['prometheus']

  - job_name: 'consul-exporter'
    static_configs:
      - targets: ['localhost:9107']
    scrape_interval: 15s

Create alerting rules

Define Prometheus alerting rules for critical HAProxy and Consul metrics to receive notifications about issues.

groups:
  - name: haproxy-alerts
    rules:
      - alert: HAProxyDown
        expr: up{job="haproxy"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "HAProxy instance is down"
          description: "HAProxy instance {{ $labels.instance }} has been down for more than 1 minute"

      - alert: HAProxyBackendDown
        expr: haproxy_server_up == 0
        for: 30s
        labels:
          severity: warning
        annotations:
          summary: "HAProxy backend server is down"
          description: "Backend server {{ $labels.server }} in {{ $labels.backend }} is down"

      - alert: HAProxyHighResponseTime
        expr: haproxy_server_response_time_average_seconds > 1
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "HAProxy high response time"
          description: "Average response time is {{ $value }}s on {{ $labels.server }}"

  - name: consul-alerts
    rules:
      - alert: ConsulDown
        expr: up{job="consul"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Consul instance is down"
          description: "Consul instance {{ $labels.instance }} has been down for more than 1 minute"

      - alert: ConsulServiceUnhealthy
        expr: consul_health_service_status{status!="passing"} > 0
        for: 30s
        labels:
          severity: warning
        annotations:
          summary: "Consul service is unhealthy"
          description: "Service {{ $labels.service_name }} is in {{ $labels.status }} state"

      - alert: ConsulLeaderElection
        expr: changes(consul_raft_leader[10m]) > 0
        for: 0s
        labels:
          severity: warning
        annotations:
          summary: "Consul leader election occurred"
          description: "Consul cluster had a leader election in the last 10 minutes"

Enable and start all services

Start and enable all the monitoring services to begin collecting metrics from HAProxy and Consul.

sudo systemctl daemon-reload
sudo systemctl enable --now haproxy-exporter
sudo systemctl enable --now consul-exporter
sudo systemctl restart haproxy
sudo systemctl restart consul
sudo systemctl restart prometheus

Import Grafana dashboards

Download and import pre-built dashboard templates for HAProxy and Consul monitoring visualization.

curl -X POST \
  http://admin:admin@localhost:3000/api/dashboards/db \
  -H 'Content-Type: application/json' \
  -d @/tmp/haproxy-dashboard.json

curl -X POST \
  http://admin:admin@localhost:3000/api/dashboards/db \
  -H 'Content-Type: application/json' \
  -d @/tmp/consul-dashboard.json

Create HAProxy dashboard configuration

Create a comprehensive Grafana dashboard for HAProxy metrics including frontend, backend, and server statistics.

{
  "dashboard": {
    "id": null,
    "title": "HAProxy Metrics",
    "tags": ["haproxy", "loadbalancer"],
    "timezone": "browser",
    "panels": [
      {
        "id": 1,
        "title": "Frontend Requests/sec",
        "type": "graph",
        "targets": [
          {
            "expr": "rate(haproxy_frontend_http_requests_total[1m])",
            "legendFormat": "{{ frontend }}"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
      },
      {
        "id": 2,
        "title": "Backend Response Time",
        "type": "graph",
        "targets": [
          {
            "expr": "haproxy_server_response_time_average_seconds",
            "legendFormat": "{{ server }}"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
      },
      {
        "id": 3,
        "title": "Server Status",
        "type": "stat",
        "targets": [
          {
            "expr": "haproxy_server_up",
            "legendFormat": "{{ server }}"
          }
        ],
        "gridPos": {"h": 8, "w": 24, "x": 0, "y": 8}
      }
    ],
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "refresh": "5s"
  }
}

Create Consul dashboard configuration

Set up a detailed Grafana dashboard for monitoring Consul cluster health, service discovery, and performance metrics.

{
  "dashboard": {
    "id": null,
    "title": "Consul Metrics",
    "tags": ["consul", "servicediscovery"],
    "timezone": "browser",
    "panels": [
      {
        "id": 1,
        "title": "Consul Nodes",
        "type": "stat",
        "targets": [
          {
            "expr": "consul_serf_lan_members",
            "legendFormat": "Cluster Members"
          }
        ],
        "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0}
      },
      {
        "id": 2,
        "title": "Registered Services",
        "type": "stat",
        "targets": [
          {
            "expr": "count(consul_catalog_service_node_healthy)",
            "legendFormat": "Services"
          }
        ],
        "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0}
      },
      {
        "id": 3,
        "title": "Service Health Status",
        "type": "piechart",
        "targets": [
          {
            "expr": "sum by (status) (consul_health_service_status)",
            "legendFormat": "{{ status }}"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
      },
      {
        "id": 4,
        "title": "Raft Commit Rate",
        "type": "graph",
        "targets": [
          {
            "expr": "rate(consul_raft_commitTime_sum[1m])",
            "legendFormat": "Commit Rate"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4}
      }
    ],
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "refresh": "10s"
  }
}