Set up HashiCorp Vault with Consul backend for production-grade secrets management. Configure clustering, authentication, policies, and security hardening for enterprise environments.

Prerequisites

Root or sudo access
At least 2GB RAM per server
Network connectivity between cluster nodes
Basic understanding of TLS certificates

What this solves

HashiCorp Vault provides centralized secrets management for applications and infrastructure. This tutorial sets up a production-ready Vault cluster with high availability using Consul as the storage backend, complete with authentication methods, policies, and security hardening.

Step-by-step installation

Add HashiCorp repository

Install the official HashiCorp repository to get the latest stable versions of Vault and Consul.

wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo apt update

sudo dnf install -y dnf-plugins-core
sudo dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo

Install Vault and Consul

Install both Vault for secrets management and Consul for the highly available storage backend.

sudo apt install -y vault consul

sudo dnf install -y vault consul

Create system users and directories

Create dedicated users for Vault and Consul with minimal privileges and proper directory structure.

sudo useradd --system --home /etc/consul.d --shell /bin/false consul
sudo useradd --system --home /etc/vault.d --shell /bin/false vault
sudo mkdir -p /opt/consul/data /opt/vault/data /etc/consul.d /etc/vault.d
sudo chown -R consul:consul /opt/consul /etc/consul.d
sudo chown -R vault:vault /opt/vault /etc/vault.d
sudo chmod 750 /opt/consul /opt/vault /etc/consul.d /etc/vault.d

Generate TLS certificates

Create self-signed certificates for internal communication. Replace with proper CA certificates in production.

sudo mkdir -p /etc/vault.d/tls /etc/consul.d/tls
cd /tmp
openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=US/ST=CA/L=SF/O=Example/CN=vault.example.com" -keyout vault-key.pem -out vault-cert.pem
openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=US/ST=CA/L=SF/O=Example/CN=consul.example.com" -keyout consul-key.pem -out consul-cert.pem
sudo mv vault-*.pem /etc/vault.d/tls/
sudo mv consul-*.pem /etc/consul.d/tls/
sudo chown -R vault:vault /etc/vault.d/tls
sudo chown -R consul:consul /etc/consul.d/tls
sudo chmod 600 /etc/vault.d/tls/.pem /etc/consul.d/tls/.pem

Configure Consul cluster

Set up Consul as the storage backend with encryption and ACLs enabled for security.

datacenter = "dc1"
data_dir = "/opt/consul/data"
log_level = "INFO"
server = true
bootstrap_expect = 3
bind_addr = "0.0.0.0"
client_addr = "0.0.0.0"
retry_join = ["203.0.113.10", "203.0.113.11", "203.0.113.12"]
ui_config {
  enabled = true
}
connect {
  enabled = true
}
ports {
  grpc = 8502
  https = 8501
}
tls {
  defaults {
    ca_file = "/etc/consul.d/tls/consul-cert.pem"
    cert_file = "/etc/consul.d/tls/consul-cert.pem"
    key_file = "/etc/consul.d/tls/consul-key.pem"
    verify_incoming = false
    verify_outgoing = true
  }
  https {
    verify_incoming = false
  }
}
acl = {
  enabled = true
  default_policy = "deny"
  enable_token_persistence = true
}

Configure Vault with Consul backend

Create the main Vault configuration using Consul for storage and enabling the UI.

storage "consul" {
  address = "127.0.0.1:8500"
  path    = "vault/"
  scheme  = "https"
  tls_cert_file = "/etc/vault.d/tls/vault-cert.pem"
  tls_key_file  = "/etc/vault.d/tls/vault-key.pem"
  tls_ca_file   = "/etc/consul.d/tls/consul-cert.pem"
}

listener "tcp" {
  address     = "0.0.0.0:8200"
  tls_cert_file = "/etc/vault.d/tls/vault-cert.pem"
  tls_key_file  = "/etc/vault.d/tls/vault-key.pem"
  tls_min_version = "tls12"
}

api_addr = "https://203.0.113.10:8200"
cluster_addr = "https://203.0.113.10:8201"
ui = true
max_lease_ttl = "8760h"
default_lease_ttl = "8760h"
disable_mlock = false

telemetry {
  prometheus_retention_time = "24h"
  disable_hostname = true
}

Create systemd service files

Set up proper systemd services with security restrictions and dependency management.

[Unit]
Description=Consul
Documentation=https://www.consul.io/
Requires=network-online.target
After=network-online.target
ConditionFileNotEmpty=/etc/consul.d/consul.hcl

[Service]
Type=notify
User=consul
Group=consul
ExecStart=/usr/bin/consul agent -config-dir=/etc/consul.d/
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
LimitNOFILE=65536
LimitNPROC=32768
NoNewPrivileges=yes
PrivateTmp=yes
ProtectHome=yes
ProtectSystem=strict
ReadWritePaths=/opt/consul/data

[Install]
WantedBy=multi-user.target

[Unit]
Description=HashiCorp Vault
Documentation=https://www.vaultproject.io/docs/
Requires=network-online.target
After=network-online.target consul.service
ConditionFileNotEmpty=/etc/vault.d/vault.hcl
StartLimitIntervalSec=60
StartLimitBurst=3

[Service]
Type=notify
User=vault
Group=vault
ProtectSystem=full
ProtectHome=read-only
PrivateTmp=yes
PrivateDevices=yes
SecureBits=keep-caps
AmbientCapabilities=CAP_IPC_LOCK
Capabilities=CAP_IPC_LOCK+ep
CapabilityBoundingSet=CAP_SYSLOG CAP_IPC_LOCK
NoNewPrivileges=yes
ExecStart=/usr/bin/vault server -config=/etc/vault.d/vault.hcl
ExecReload=/bin/kill --signal HUP $MAINPID
KillMode=process
KillSignal=SIGINT
Restart=on-failure
RestartSec=5
TimeoutStopSec=30
LimitNOFILE=65536
LimitMEMLOCK=infinity

[Install]
WantedBy=multi-user.target

Configure firewall rules

Open the necessary ports for Vault and Consul communication while maintaining security.

sudo ufw allow 8200/tcp comment "Vault API"
sudo ufw allow 8201/tcp comment "Vault cluster"
sudo ufw allow 8300/tcp comment "Consul server RPC"
sudo ufw allow 8301/tcp comment "Consul LAN serf"
sudo ufw allow 8302/tcp comment "Consul WAN serf"
sudo ufw allow 8500/tcp comment "Consul HTTP API"
sudo ufw allow 8501/tcp comment "Consul HTTPS API"
sudo ufw allow 8502/tcp comment "Consul gRPC"

sudo firewall-cmd --permanent --add-port=8200/tcp --add-port=8201/tcp
sudo firewall-cmd --permanent --add-port=8300/tcp --add-port=8301/tcp
sudo firewall-cmd --permanent --add-port=8302/tcp --add-port=8500/tcp
sudo firewall-cmd --permanent --add-port=8501/tcp --add-port=8502/tcp
sudo firewall-cmd --reload

Start and enable services

Start Consul first, then Vault, and enable both services to start automatically on boot.

sudo systemctl daemon-reload
sudo systemctl enable --now consul
sudo systemctl status consul
sleep 10
sudo systemctl enable --now vault
sudo systemctl status vault

Initialize Vault cluster

Initialize the Vault cluster and securely store the unseal keys and root token.

export VAULT_ADDR='https://127.0.0.1:8200'
export VAULT_CACERT='/etc/vault.d/tls/vault-cert.pem'
vault operator init -key-shares=5 -key-threshold=3 > /tmp/vault-init.txt
sudo mv /tmp/vault-init.txt /etc/vault.d/vault-init.txt
sudo chown vault:vault /etc/vault.d/vault-init.txt
sudo chmod 600 /etc/vault.d/vault-init.txt

Warning: Store the unseal keys and root token in a secure location immediately. Without these, you cannot access your Vault cluster.

Unseal Vault

Use three of the five unseal keys to unlock the Vault cluster for operation.

vault operator unseal $(grep 'Unseal Key 1:' /etc/vault.d/vault-init.txt | awk '{print $NF}')
vault operator unseal $(grep 'Unseal Key 2:' /etc/vault.d/vault-init.txt | awk '{print $NF}')
vault operator unseal $(grep 'Unseal Key 3:' /etc/vault.d/vault-init.txt | awk '{print $NF}')
vault status

Configure authentication and policies

Set up userpass authentication and create a basic policy for application access.

export VAULT_TOKEN=$(grep 'Initial Root Token:' /etc/vault.d/vault-init.txt | awk '{print $NF}')
vault auth enable userpass
vault policy write app-policy - << EOF
path "secret/data/myapp/*" {
  capabilities = ["create", "read", "update", "delete", "list"]
}
path "secret/metadata/myapp/*" {
  capabilities = ["list"]
}
EOF
vault write auth/userpass/users/appuser password="SecureP@ssw0rd123" policies="app-policy"

Enable secrets engines and audit logging

Enable the KV secrets engine and configure audit logging to a file for security compliance.

vault secrets enable -path=secret kv-v2
vault audit enable file file_path=/var/log/vault-audit.log
sudo touch /var/log/vault-audit.log
sudo chown vault:vault /var/log/vault-audit.log
sudo chmod 640 /var/log/vault-audit.log

Verify your setup

Test the Vault installation by checking cluster status, storing a secret, and retrieving it.

vault status
vault auth -method=userpass username=appuser password="SecureP@ssw0rd123"
vault kv put secret/myapp/database username="dbuser" password="dbpass123"
vault kv get secret/myapp/database
curl -k https://localhost:8200/v1/sys/health | jq

Note: The cluster is sealed by default after restart. You must unseal it with three keys each time the service restarts unless you configure auto-unsealing.

Security hardening

Configure log rotation

Set up log rotation for Vault audit logs to prevent disk space issues.

/var/log/vault-audit.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    postrotate
        /bin/systemctl reload vault
    endscript
    su vault vault
}

Enable monitoring integration

Configure Vault to export metrics for monitoring systems like Prometheus.

vault write sys/config/ui header_value="X-Custom-Header=vault-monitoring"
vault read sys/metrics?format=prometheus

Consider integrating with Grafana and Prometheus for comprehensive monitoring.

Backup and recovery procedures

Create backup script

Set up automated backups of Vault data through Consul snapshots.

#!/bin/bash
BACKUP_DIR="/opt/backups/vault"
DATE=$(date +%Y%m%d-%H%M%S)

mkdir -p $BACKUP_DIR
consul snapshot save $BACKUP_DIR/consul-backup-$DATE.snap

Keep only last 7 days of backups
find $BACKUP_DIR -name "consul-backup-*.snap" -mtime +7 -delete

echo "Vault backup completed: $BACKUP_DIR/consul-backup-$DATE.snap"

sudo chmod +x /usr/local/bin/vault-backup.sh
sudo mkdir -p /opt/backups/vault
sudo chown consul:consul /opt/backups/vault

Schedule automated backups

Add a cron job to run backups daily during low-usage hours.

echo "0 2   * consul /usr/local/bin/vault-backup.sh" | sudo tee -a /etc/crontab

For production environments, consider using MinIO object storage for backup retention and HAProxy for load balancing multiple Vault instances.

Common issues

Symptom	Cause	Fix
Vault sealed after restart	Normal behavior - Vault seals on restart	Run unseal process with 3 keys: `vault operator unseal KEY`
Permission denied errors	Incorrect file ownership or permissions	Check ownership: `sudo chown -R vault:vault /etc/vault.d /opt/vault`
TLS certificate errors	Self-signed certificates not trusted	Use `-tls-skip-verify` flag or add `VAULT_SKIP_VERIFY=true`
Consul connection refused	Consul not running or wrong port	Check Consul status: `sudo systemctl status consul`
High availability not working	Consul cluster not formed properly	Verify all nodes in cluster: `consul members`
Memory lock failures	Insufficient privileges for mlock	Check systemd capabilities and `LimitMEMLOCK=infinity`

Next steps

Automated install script

Run this to automate the entire setup

install.sh

#!/usr/bin/env bash
set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Default values
DOMAIN="${1:-vault.local}"
CONSUL_IPS="${2:-127.0.0.1}"
VAULT_IP="${3:-127.0.0.1}"

usage() {
    echo "Usage: $0 [DOMAIN] [CONSUL_IPS] [VAULT_IP]"
    echo "Example: $0 vault.example.com '203.0.113.10,203.0.113.11,203.0.113.12' 203.0.113.10"
    exit 1
}

log() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1"
    exit 1
}

cleanup() {
    warn "Installation failed. Cleaning up..."
    systemctl stop vault consul 2>/dev/null || true
    systemctl disable vault consul 2>/dev/null || true
}

trap cleanup ERR

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    error "This script must be run as root"
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) 
            PKG_MGR="apt"
            PKG_UPDATE="apt update"
            PKG_INSTALL="apt install -y"
            REPO_SETUP="debian"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_UPDATE="dnf check-update || true"
            PKG_INSTALL="dnf install -y"
            REPO_SETUP="rhel"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_UPDATE="yum check-update || true"
            PKG_INSTALL="yum install -y"
            REPO_SETUP="rhel"
            ;;
        *)
            error "Unsupported distribution: $ID"
            ;;
    esac
else
    error "Cannot detect distribution"
fi

echo "[1/10] Checking prerequisites..."
command -v wget >/dev/null 2>&1 || $PKG_INSTALL wget
command -v curl >/dev/null 2>&1 || $PKG_INSTALL curl
command -v openssl >/dev/null 2>&1 || $PKG_INSTALL openssl

echo "[2/10] Adding HashiCorp repository..."
if [[ "$REPO_SETUP" == "debian" ]]; then
    wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
    echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/hashicorp.list
    $PKG_UPDATE
else
    $PKG_INSTALL dnf-plugins-core 2>/dev/null || $PKG_INSTALL yum-utils
    if command -v dnf-config-manager >/dev/null 2>&1; then
        dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
    else
        yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
    fi
fi

echo "[3/10] Installing Vault and Consul..."
$PKG_INSTALL vault consul

echo "[4/10] Creating system users and directories..."
useradd --system --home /etc/consul.d --shell /bin/false consul 2>/dev/null || true
useradd --system --home /etc/vault.d --shell /bin/false vault 2>/dev/null || true

mkdir -p /opt/consul/data /opt/vault/data /etc/consul.d /etc/vault.d
chown -R consul:consul /opt/consul /etc/consul.d
chown -R vault:vault /opt/vault /etc/vault.d
chmod 750 /opt/consul /opt/vault /etc/consul.d /etc/vault.d

echo "[5/10] Generating TLS certificates..."
mkdir -p /etc/vault.d/tls /etc/consul.d/tls

openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
    -subj "/C=US/ST=CA/L=SF/O=Vault/CN=$DOMAIN" \
    -keyout /etc/vault.d/tls/vault-key.pem \
    -out /etc/vault.d/tls/vault-cert.pem

openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
    -subj "/C=US/ST=CA/L=SF/O=Consul/CN=consul.$DOMAIN" \
    -keyout /etc/consul.d/tls/consul-key.pem \
    -out /etc/consul.d/tls/consul-cert.pem

chown -R vault:vault /etc/vault.d/tls
chown -R consul:consul /etc/consul.d/tls
chmod 600 /etc/vault.d/tls/*.pem /etc/consul.d/tls/*.pem

echo "[6/10] Configuring Consul..."
IFS=',' read -ra CONSUL_ARRAY <<< "$CONSUL_IPS"
CONSUL_JOIN=""
for ip in "${CONSUL_ARRAY[@]}"; do
    CONSUL_JOIN="$CONSUL_JOIN\"$ip\", "
done
CONSUL_JOIN=${CONSUL_JOIN%, }

cat > /etc/consul.d/consul.hcl << EOF
datacenter = "dc1"
data_dir = "/opt/consul/data"
log_level = "INFO"
server = true
bootstrap_expect = ${#CONSUL_ARRAY[@]}
bind_addr = "0.0.0.0"
client_addr = "0.0.0.0"
retry_join = [$CONSUL_JOIN]
ui_config {
  enabled = true
}
connect {
  enabled = true
}
ports {
  grpc = 8502
  https = 8501
}
tls {
  defaults {
    ca_file = "/etc/consul.d/tls/consul-cert.pem"
    cert_file = "/etc/consul.d/tls/consul-cert.pem"
    key_file = "/etc/consul.d/tls/consul-key.pem"
    verify_incoming = false
    verify_outgoing = false
  }
  https {
    verify_incoming = false
  }
}
acl = {
  enabled = false
  default_policy = "allow"
}
EOF

chown consul:consul /etc/consul.d/consul.hcl
chmod 640 /etc/consul.d/consul.hcl

echo "[7/10] Configuring Vault..."
cat > /etc/vault.d/vault.hcl << EOF
storage "consul" {
  address = "127.0.0.1:8500"
  path    = "vault/"
  scheme  = "http"
}

listener "tcp" {
  address     = "0.0.0.0:8200"
  tls_cert_file = "/etc/vault.d/tls/vault-cert.pem"
  tls_key_file  = "/etc/vault.d/tls/vault-key.pem"
  tls_min_version = "tls12"
}

api_addr = "https://$VAULT_IP:8200"
cluster_addr = "https://$VAULT_IP:8201"
ui = true
max_lease_ttl = "8760h"
default_lease_ttl = "8760h"
disable_mlock = false

telemetry {
  prometheus_retention_time = "24h"
  disable_hostname = true
}
EOF

chown vault:vault /etc/vault.d/vault.hcl
chmod 640 /etc/vault.d/vault.hcl

echo "[8/10] Configuring systemd services..."
systemctl daemon-reload
systemctl enable consul vault
systemctl start consul

# Wait for consul to be ready
sleep 10

systemctl start vault

echo "[9/10] Configuring firewall..."
if command -v ufw >/dev/null 2>&1; then
    ufw allow 8200/tcp comment "Vault API"
    ufw allow 8500/tcp comment "Consul API"
    ufw allow 8600/tcp comment "Consul DNS"
elif command -v firewall-cmd >/dev/null 2>&1; then
    firewall-cmd --permanent --add-port=8200/tcp
    firewall-cmd --permanent --add-port=8500/tcp
    firewall-cmd --permanent --add-port=8600/tcp
    firewall-cmd --reload
fi

echo "[10/10] Verifying installation..."
sleep 5

if systemctl is-active --quiet consul; then
    log "Consul is running"
else
    error "Consul failed to start"
fi

if systemctl is-active --quiet vault; then
    log "Vault is running"
else
    error "Vault failed to start"
fi

if curl -k -s https://localhost:8200/v1/sys/health | grep -q "initialized"; then
    log "Vault API is responding"
else
    warn "Vault API not responding - may need initialization"
fi

log "Installation completed successfully!"
log "Consul UI: http://$VAULT_IP:8500"
log "Vault UI: https://$VAULT_IP:8200"
warn "Initialize Vault with: vault operator init"
warn "Remember to unseal Vault after initialization"

Review the script before running. Execute with: bash install.sh

#hashicorp vault #secrets management #vault clustering #consul backend #zero trust security

Install and configure Vault for secrets management with high availability