Deploy Consul Template for dynamic configuration management with systemd integration

Update system packages

Start by updating your package manager to ensure you get the latest versions.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install required packages

Install curl, unzip and other dependencies needed for Consul Template installation.

sudo apt install -y curl unzip jq

sudo dnf install -y curl unzip jq

Download and install Consul Template

Download the latest Consul Template binary from HashiCorp's releases and install it to the system path.

export CONSUL_TEMPLATE_VERSION="0.37.5"
wget https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip
unzip consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip
sudo mv consul-template /usr/local/bin/
sudo chmod +x /usr/local/bin/consul-template

Create consul-template user

Create a dedicated system user for running Consul Template with minimal privileges.

sudo useradd --system --home /etc/consul-template --shell /bin/false consul-template

Create directory structure

Set up the required directories for configuration files, templates, and generated output.

sudo mkdir -p /etc/consul-template/templates
sudo mkdir -p /etc/consul-template/config
sudo mkdir -p /var/lib/consul-template
sudo mkdir -p /var/log/consul-template
sudo chown -R consul-template:consul-template /etc/consul-template
sudo chown -R consul-template:consul-template /var/lib/consul-template
sudo chown -R consul-template:consul-template /var/log/consul-template

Configure main Consul Template settings

Create the main configuration file that defines how Consul Template connects to Consul and processes templates.

consul {
  address = "127.0.0.1:8500"
  retry {
    enabled = true
    attempts = 12
    backoff = "250ms"
    max_backoff = "1m"
  }
}

reload_signal = "SIGHUP"
kill_signal = "SIGINT"
max_stale = "10m"
log_level = "INFO"
log_file = "/var/log/consul-template/consul-template.log"
pid_file = "/var/lib/consul-template/consul-template.pid"

wait {
  min = "5s"
  max = "10s"
}

Create your first template

Create a simple template that generates a configuration file based on Consul services. This example creates an upstream configuration for NGINX.

{{- range services }}
{{- if .Tags | contains "web" }}
upstream {{ .Name }} {
{{- range service .Name }}
  server {{ .Address }}:{{ .Port }};
{{- end }}
}
{{- end }}
{{- end }}

Generated at {{ timestamp }}
Services found: {{ services | len }}

Configure template processing

Add template configuration to the main config file to specify input template, output file, and commands to run when the template changes.

template {
  source = "/etc/consul-template/templates/nginx-upstream.conf.tpl"
  destination = "/etc/nginx/conf.d/upstream.conf"
  create_dest_dirs = true
  command = "nginx -t && systemctl reload nginx"
  command_timeout = "30s"
  error_on_missing_key = false
  perms = 0644
  backup = true
  left_delimiter  = "{{"
  right_delimiter = "}}"
  wait {
    min = "2s"
    max = "10s"
  }
}

Create systemd service file

Set up systemd service management for Consul Template with proper restart policies and security settings.

[Unit]
Description=Consul Template
Documentation=https://github.com/hashicorp/consul-template
Requires=network-online.target
After=network-online.target
ConditionFileNotEmpty=/etc/consul-template/config/consul-template.hcl

[Service]
Type=notify
User=consul-template
Group=consul-template
ExecStart=/usr/local/bin/consul-template -config-dir=/etc/consul-template/config
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
TimeoutStopSec=30

Security settings
NoNewPrivileges=yes
PrivateTmp=yes
PrivateDevices=yes
ProtectHome=yes
ProtectSystem=strict
ReadWritePaths=/var/lib/consul-template /var/log/consul-template /etc/nginx/conf.d

[Install]
WantedBy=multi-user.target

Set correct permissions

Ensure all files have the correct ownership and permissions for the consul-template user.

sudo chown -R consul-template:consul-template /etc/consul-template
sudo chmod 755 /etc/consul-template
sudo chmod 644 /etc/consul-template/config/*
sudo chmod 644 /etc/consul-template/templates/*

Configure log rotation

Set up log rotation to prevent Consul Template logs from filling up disk space.

/var/log/consul-template/*.log {
  daily
  missingok
  rotate 7
  compress
  delaycompress
  notifempty
  copytruncate
  create 644 consul-template consul-template
}

Enable and start the service

Enable Consul Template to start automatically on boot and start the service now.

sudo systemctl daemon-reload
sudo systemctl enable consul-template
sudo systemctl start consul-template

Create HAProxy backend template

This template generates HAProxy backend configurations dynamically based on Consul service registrations.

global
  daemon
  maxconn 4096

defaults
  mode http
  timeout connect 5000ms
  timeout client 50000ms
  timeout server 50000ms

{{- range services }}
{{- if .Tags | contains "backend" }}
backend {{ .Name }}
  balance roundrobin
{{- range service .Name }}
  server {{ .Name }}-{{ .ID }} {{ .Address }}:{{ .Port }} check
{{- end }}
{{- end }}
{{- end }}

frontend web
  bind *:80
{{- range services }}
{{- if .Tags | contains "backend" }}
  use_backend {{ .Name }} if { hdr(host) -i {{ .Name }}.example.com }
{{- end }}
{{- end }}

Add HAProxy template to configuration

Configure Consul Template to process the HAProxy template and reload HAProxy when changes occur.

template {
  source = "/etc/consul-template/templates/haproxy-backends.cfg.tpl"
  destination = "/etc/haproxy/haproxy.cfg"
  create_dest_dirs = true
  command = "haproxy -f /etc/haproxy/haproxy.cfg -c && systemctl reload haproxy"
  command_timeout = "30s"
  perms = 0644
  backup = true
}

Create application configuration template

Generate application configuration files using Consul key-value store data.

{
  "database": {
    "host": "{{ keyOrDefault "config/database/host" "localhost" }}",
    "port": {{ keyOrDefault "config/database/port" "5432" }},
    "username": "{{ keyOrDefault "config/database/username" "app" }}",
    "password": "{{ keyOrDefault "config/database/password" "changeme" }}",
    "name": "{{ keyOrDefault "config/database/name" "myapp" }}"
  },
  "redis": {
    "host": "{{ keyOrDefault "config/redis/host" "localhost" }}",
    "port": {{ keyOrDefault "config/redis/port" "6379" }}
  },
  "services": {
{{- range $index, $service := services }}
{{- if $index }},{{- end }}
    "{{ .Name }}": [
{{- range $sIndex, $instance := service .Name }}
{{- if $sIndex }},{{- end }}
      {
        "address": "{{ .Address }}",
        "port": {{ .Port }},
        "tags": [{{ range $tIndex, $tag := .Tags }}{{- if $tIndex }},{{- end }}"{{ . }}"{{ end }}]
      }
{{- end }}
    ]
{{- end }}
  },
  "generated_at": "{{ timestamp }}"
}

Configure service dependencies

Ensure Consul Template starts after Consul and required services are available. This setup provides coordination between Consul service discovery and template rendering.

[Unit]
After=consul.service
Wants=consul.service

[Service]
Environment=CONSUL_HTTP_ADDR=127.0.0.1:8500
Restart=always
RestartSec=10

sudo mkdir -p /etc/systemd/system/consul-template.service.d

Create health check script

Set up a health check script to monitor Consul Template status and template rendering.

#!/bin/bash

Check if Consul Template is running
if ! systemctl is-active --quiet consul-template; then
  echo "ERROR: Consul Template service is not running"
  exit 1
fi

Check if PID file exists and process is running
PID_FILE="/var/lib/consul-template/consul-template.pid"
if [[ -f "$PID_FILE" ]]; then
  PID=$(cat "$PID_FILE")
  if ! ps -p "$PID" > /dev/null 2>&1; then
    echo "ERROR: Consul Template process not found (PID: $PID)"
    exit 1
  fi
else
  echo "ERROR: PID file not found"
  exit 1
fi

Check if log file has recent entries (within 5 minutes)
LOG_FILE="/var/log/consul-template/consul-template.log"
if [[ -f "$LOG_FILE" ]]; then
  LAST_LOG=$(stat -c %Y "$LOG_FILE")
  CURRENT_TIME=$(date +%s)
  TIME_DIFF=$((CURRENT_TIME - LAST_LOG))
  
  if [[ $TIME_DIFF -gt 300 ]]; then
    echo "WARNING: No recent log entries (last update: ${TIME_DIFF}s ago)"
  fi
fi

echo "OK: Consul Template is healthy"
exit 0

sudo chmod +x /usr/local/bin/consul-template-health-check

Configure systemd timer for health checks

Set up automated health monitoring using systemd timers.

[Unit]
Description=Consul Template Health Check
After=consul-template.service

[Service]
Type=oneshot
ExecStart=/usr/local/bin/consul-template-health-check
User=consul-template
Group=consul-template

[Unit]
Description=Run Consul Template health check every 2 minutes
Requires=consul-template-health.service

[Timer]
OnBootSec=5min
OnUnitActiveSec=2min

[Install]
WantedBy=timers.target

sudo systemctl daemon-reload
sudo systemctl enable consul-template-health.timer
sudo systemctl start consul-template-health.timer

Set up Prometheus metrics collection

Configure Consul Template to expose metrics for monitoring and create a metrics endpoint.

telemetry {
  prometheus_retention_time = "30s"
  disable_hostname = true
}

Create monitoring template

Generate a monitoring configuration that tracks template rendering and service health.

{
  "consul_template": {
    "version": "0.37.5",
    "status": "running",
    "last_update": "{{ timestamp }}",
    "services_count": {{ services | len }},
    "services": {
{{- range $index, $service := services }}
{{- if $index }},{{- end }}
      "{{ .Name }}": {
        "instances": {{ service .Name | len }},
        "healthy_instances": {{ service .Name "passing" | len }},
        "tags": [{{ range $tIndex, $tag := .Tags }}{{- if $tIndex }},{{- end }}"{{ . }}"{{ end }}]
      }
{{- end }}
    }
  }
}

Configure alerting rules

Set up alerting based on template rendering failures and service availability. This integrates well with existing Prometheus alerting setups.

template {
  source = "/etc/consul-template/templates/monitoring.json.tpl"
  destination = "/var/lib/consul-template/monitoring.json"
  create_dest_dirs = true
  command = "curl -X POST http://localhost:9090/api/v1/admin/tsdb/delete_series?match[]=consul_template_info"
  command_timeout = "10s"
  perms = 0644
  wait {
    min = "10s"
    max = "30s"
  }
}

Create alert notification script

Set up a script that sends notifications when template rendering fails or services become unhealthy.

#!/bin/bash

ALERT_TYPE="${1:-unknown}"
SERVICE_NAME="${2:-unknown}"
MESSAGE="${3:-No message provided}"
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')

Log alert
echo "[$TIMESTAMP] ALERT: $ALERT_TYPE - $SERVICE_NAME - $MESSAGE" >> /var/log/consul-template/alerts.log

Send to syslog
logger -t consul-template "ALERT: $ALERT_TYPE - $SERVICE_NAME - $MESSAGE"

Optional: Send to external monitoring system
curl -X POST "https://hooks.slack.com/services/YOUR/WEBHOOK/URL" \
  -H 'Content-type: application/json' \
  --data "{\"text\":\"Consul Template Alert: $MESSAGE\"}"

exit 0

sudo chmod +x /usr/local/bin/consul-template-alert

Test template rendering

Verify that templates are being rendered correctly by checking generated files.

sudo cat /etc/nginx/conf.d/upstream.conf
sudo /usr/local/bin/consul-template-health-check
sudo systemctl status consul-template-health.timer