Set up Prometheus Alertmanager to send critical alerts via email and Slack channels with custom webhook integration. This tutorial covers installation, SMTP configuration, routing rules, and alert notification testing.
Prerequisites
- Prometheus server installed and running
- SMTP server access for email notifications
- Slack workspace with webhook permissions
- Basic familiarity with YAML configuration
What this solves
Prometheus Alertmanager centralizes alert routing and notification management for your monitoring infrastructure. Without Alertmanager, Prometheus can only store alert states but cannot notify your team when critical issues occur. This tutorial configures email notifications via SMTP, Slack integration with webhooks, and custom routing rules to ensure the right alerts reach the right people at the right time.
Step-by-step installation
Update system packages
Start by updating your package manager to ensure you get the latest versions of dependencies.
sudo apt update && sudo apt upgrade -y
sudo apt install -y wget curl
Create Alertmanager user
Create a dedicated system user to run Alertmanager securely without shell access.
sudo useradd --no-create-home --shell /bin/false alertmanager
Download and install Alertmanager
Download the latest Alertmanager binary from the official Prometheus releases.
cd /tmp
wget https://github.com/prometheus/alertmanager/releases/download/v0.27.0/alertmanager-0.27.0.linux-amd64.tar.gz
tar xzf alertmanager-0.27.0.linux-amd64.tar.gz
sudo cp alertmanager-0.27.0.linux-amd64/alertmanager /usr/local/bin/
sudo cp alertmanager-0.27.0.linux-amd64/amtool /usr/local/bin/
sudo chown alertmanager:alertmanager /usr/local/bin/alertmanager /usr/local/bin/amtool
Create directories and set permissions
Set up the required directories with proper ownership and permissions for Alertmanager.
sudo mkdir -p /etc/alertmanager /var/lib/alertmanager
sudo chown alertmanager:alertmanager /etc/alertmanager /var/lib/alertmanager
sudo chmod 755 /etc/alertmanager /var/lib/alertmanager
Create Alertmanager configuration
Configure Alertmanager with email SMTP settings, Slack webhook, and routing rules. Replace the webhook URL and email settings with your actual values.
global:
smtp_smarthost: 'smtp.gmail.com:587'
smtp_from: 'alerts@example.com'
smtp_auth_username: 'alerts@example.com'
smtp_auth_password: 'your-app-password'
smtp_require_tls: true
route:
group_by: ['alertname', 'cluster', 'service']
group_wait: 10s
group_interval: 10s
repeat_interval: 1h
receiver: 'default-receiver'
routes:
- match:
severity: critical
receiver: 'critical-alerts'
- match:
severity: warning
receiver: 'warning-alerts'
- match:
alertname: 'InstanceDown'
receiver: 'instance-down-alerts'
receivers:
- name: 'default-receiver'
email_configs:
- to: 'admin@example.com'
subject: 'Prometheus Alert: {{ .GroupLabels.alertname }}'
body: |
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Description: {{ .Annotations.description }}
Labels: {{ .Labels }}
{{ end }}
- name: 'critical-alerts'
email_configs:
- to: 'oncall@example.com'
subject: 'CRITICAL: {{ .GroupLabels.alertname }}'
body: |
CRITICAL ALERT TRIGGERED
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Description: {{ .Annotations.description }}
Severity: {{ .Labels.severity }}
Instance: {{ .Labels.instance }}
Started: {{ .StartsAt }}
{{ end }}
slack_configs:
- api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
channel: '#alerts'
title: 'Critical Alert: {{ .GroupLabels.alertname }}'
text: |
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Description: {{ .Annotations.description }}
Severity: {{ .Labels.severity }}
Instance: {{ .Labels.instance }}
{{ end }}
send_resolved: true
- name: 'warning-alerts'
email_configs:
- to: 'team@example.com'
subject: 'Warning: {{ .GroupLabels.alertname }}'
body: |
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Description: {{ .Annotations.description }}
Labels: {{ .Labels }}
{{ end }}
slack_configs:
- api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
channel: '#monitoring'
title: 'Warning: {{ .GroupLabels.alertname }}'
text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
send_resolved: true
- name: 'instance-down-alerts'
email_configs:
- to: 'infrastructure@example.com'
subject: 'Instance Down: {{ .GroupLabels.instance }}'
body: |
SERVER DOWN ALERT
{{ range .Alerts }}
Instance: {{ .Labels.instance }}
Job: {{ .Labels.job }}
Started: {{ .StartsAt }}
{{ end }}
slack_configs:
- api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
channel: '#infrastructure'
title: 'Server Down: {{ .GroupLabels.instance }}'
text: 'Instance {{ .GroupLabels.instance }} is unreachable'
send_resolved: true
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
equal: ['alertname', 'cluster', 'service']
Set configuration file permissions
Secure the configuration file since it contains sensitive SMTP credentials.
sudo chown alertmanager:alertmanager /etc/alertmanager/alertmanager.yml
sudo chmod 640 /etc/alertmanager/alertmanager.yml
Create systemd service file
Set up Alertmanager as a systemd service for automatic startup and management.
[Unit]
Description=Alertmanager
Wants=network-online.target
After=network-online.target
[Service]
User=alertmanager
Group=alertmanager
Type=simple
WorkingDirectory=/var/lib/alertmanager
ExecStart=/usr/local/bin/alertmanager \
--config.file=/etc/alertmanager/alertmanager.yml \
--storage.path=/var/lib/alertmanager \
--web.listen-address=0.0.0.0:9093 \
--cluster.listen-address=0.0.0.0:9094 \
--log.level=info
Restart=always
RestartSec=3
[Install]
WantedBy=multi-user.target
Enable and start Alertmanager
Enable the service to start on boot and launch Alertmanager.
sudo systemctl daemon-reload
sudo systemctl enable --now alertmanager
sudo systemctl status alertmanager
Configure firewall
Open the necessary ports for Alertmanager web UI and cluster communication.
sudo ufw allow 9093/tcp
sudo ufw allow 9094/tcp
sudo ufw reload
Configure Prometheus to use Alertmanager
Update your Prometheus configuration to send alerts to Alertmanager. Add this to your existing Prometheus config.
alerting:
alertmanagers:
- static_configs:
- targets:
- localhost:9093
rule_files:
- "/etc/prometheus/alert_rules.yml"
Create sample alerting rules
Create basic alerting rules for Prometheus to trigger notifications through Alertmanager.
groups:
- name: basic-alerts
rules:
- alert: InstanceDown
expr: up == 0
for: 1m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} is down"
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minute."
- alert: HighCpuUsage
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
for: 2m
labels:
severity: warning
annotations:
summary: "High CPU usage on {{ $labels.instance }}"
description: "CPU usage is above 80% on {{ $labels.instance }} for more than 2 minutes."
- alert: HighMemoryUsage
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
for: 2m
labels:
severity: critical
annotations:
summary: "High memory usage on {{ $labels.instance }}"
description: "Memory usage is above 90% on {{ $labels.instance }} for more than 2 minutes."
- alert: DiskSpaceLow
expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"})) * 100 > 85
for: 1m
labels:
severity: warning
annotations:
summary: "Low disk space on {{ $labels.instance }}"
description: "Disk usage is above 85% on {{ $labels.instance }} {{ $labels.mountpoint }}."
- alert: ServiceDown
expr: up{job="node"} == 0
for: 30s
labels:
severity: critical
annotations:
summary: "Service {{ $labels.job }} is down"
description: "Service {{ $labels.job }} on {{ $labels.instance }} is down for more than 30 seconds."
Restart Prometheus to load new configuration
Reload Prometheus configuration to activate the new alerting rules and Alertmanager integration.
sudo systemctl restart prometheus
sudo systemctl status prometheus
Configure Slack webhook integration
Create Slack webhook URL
Go to your Slack workspace settings and create an Incoming Webhook. Navigate to Apps > Incoming Webhooks > Add to Slack. Select the channel where you want alerts and copy the webhook URL.
Test Slack notification
Send a test alert to verify your Slack integration works correctly.
curl -X POST 'http://localhost:9093/api/v1/alerts' \
-H 'Content-Type: application/json' \
-d '[{
"labels": {
"alertname": "TestAlert",
"severity": "critical",
"instance": "test-server"
},
"annotations": {
"summary": "This is a test alert",
"description": "Testing Alertmanager Slack integration"
},
"generatorURL": "http://localhost:9090/graph"
}]'
Configure email notifications
Set up Gmail App Password
If using Gmail, create an App Password for SMTP authentication. Go to Google Account settings > Security > 2-Step Verification > App passwords. Generate a password and use it in the Alertmanager config.
Test email notification
Send a test email alert to verify SMTP configuration works.
curl -X POST 'http://localhost:9093/api/v1/alerts' \
-H 'Content-Type: application/json' \
-d '[{
"labels": {
"alertname": "EmailTest",
"severity": "warning",
"instance": "mail-test-server"
},
"annotations": {
"summary": "Email notification test",
"description": "Testing Alertmanager email integration via SMTP"
},
"generatorURL": "http://localhost:9090/graph"
}]'
Advanced webhook integration
Create custom webhook receiver
Add a custom webhook receiver to your Alertmanager configuration for integration with external systems.
- name: 'webhook-alerts'
webhook_configs:
- url: 'http://your-webhook-endpoint.com/alerts'
http_config:
basic_auth:
username: 'webhook-user'
password: 'webhook-password'
send_resolved: true
max_alerts: 10
title: 'Prometheus Alert'
text: |
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Status: {{ .Status }}
Labels: {{ .Labels }}
{{ end }}
Add webhook routing rule
Add a routing rule to send specific alerts to your webhook endpoint.
- match:
service: 'database'
receiver: 'webhook-alerts'
Reload Alertmanager configuration
Apply the updated configuration with webhook integration.
sudo systemctl reload alertmanager
sudo systemctl status alertmanager
Verify your setup
# Check Alertmanager status
sudo systemctl status alertmanager
Verify web UI is accessible
curl -I http://localhost:9093
Check current alerts
curl http://localhost:9093/api/v1/alerts
Validate configuration
/usr/local/bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yml --config.check
View logs for troubleshooting
sudo journalctl -u alertmanager -f
Check Prometheus integration
curl http://localhost:9090/api/v1/alertmanagers
Access the Alertmanager web UI at http://your-server-ip:9093 to view active alerts, silences, and configuration status.
Common issues
| Symptom | Cause | Fix |
|---|---|---|
| Email notifications not sent | SMTP authentication failed | Verify credentials and use App Password for Gmail |
| Slack notifications missing | Invalid webhook URL | Check webhook URL and channel permissions |
| Alertmanager won't start | Configuration syntax error | Run /usr/local/bin/alertmanager --config.check |
| No alerts showing | Prometheus not connected | Check Prometheus alerting config and restart service |
| Permission denied errors | Incorrect file ownership | Run sudo chown -R alertmanager:alertmanager /etc/alertmanager |
| Web UI not accessible | Firewall blocking port | Open port 9093 in firewall configuration |
Next steps
- Set up Prometheus Blackbox Exporter for uptime monitoring to add external service checks
- Configure advanced Grafana dashboards and alerting for visual monitoring
- Configure Alertmanager high availability clustering for production resilience
- Integrate Alertmanager with PagerDuty and OpsGenie for incident management
Running this in production?
Automated install script
Run this to automate the entire setup
#!/usr/bin/env bash
set -euo pipefail
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Script variables
ALERTMANAGER_VERSION="0.27.0"
ALERTMANAGER_USER="alertmanager"
CONFIG_FILE="/etc/alertmanager/alertmanager.yml"
SERVICE_FILE="/etc/systemd/system/alertmanager.service"
# Function to print colored output
print_status() {
echo -e "${GREEN}[INFO]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
print_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# Usage function
usage() {
echo "Usage: $0 [OPTIONS]"
echo "Options:"
echo " --smtp-host <host> SMTP server host (default: smtp.gmail.com:587)"
echo " --smtp-user <email> SMTP username/email"
echo " --smtp-pass <password> SMTP password"
echo " --admin-email <email> Admin email for notifications"
echo " --slack-webhook <url> Slack webhook URL"
echo " -h, --help Show this help message"
exit 1
}
# Parse arguments
SMTP_HOST="smtp.gmail.com:587"
SMTP_USER=""
SMTP_PASS=""
ADMIN_EMAIL=""
SLACK_WEBHOOK=""
while [[ $# -gt 0 ]]; do
case $1 in
--smtp-host)
SMTP_HOST="$2"
shift 2
;;
--smtp-user)
SMTP_USER="$2"
shift 2
;;
--smtp-pass)
SMTP_PASS="$2"
shift 2
;;
--admin-email)
ADMIN_EMAIL="$2"
shift 2
;;
--slack-webhook)
SLACK_WEBHOOK="$2"
shift 2
;;
-h|--help)
usage
;;
*)
echo "Unknown option: $1"
usage
;;
esac
done
# Cleanup function for rollback
cleanup() {
print_error "Installation failed. Cleaning up..."
systemctl stop alertmanager 2>/dev/null || true
systemctl disable alertmanager 2>/dev/null || true
rm -f "$SERVICE_FILE" 2>/dev/null || true
rm -rf /etc/alertmanager /var/lib/alertmanager 2>/dev/null || true
rm -f /usr/local/bin/alertmanager /usr/local/bin/amtool 2>/dev/null || true
userdel "$ALERTMANAGER_USER" 2>/dev/null || true
systemctl daemon-reload 2>/dev/null || true
print_error "Cleanup completed."
}
trap cleanup ERR
# Check if running as root
if [[ $EUID -ne 0 ]]; then
print_error "This script must be run as root or with sudo"
exit 1
fi
# Auto-detect distribution
if [ -f /etc/os-release ]; then
. /etc/os-release
case "$ID" in
ubuntu|debian)
PKG_MGR="apt"
PKG_UPDATE="apt update && apt upgrade -y"
PKG_INSTALL="apt install -y"
;;
almalinux|rocky|centos|rhel|ol)
PKG_MGR="dnf"
PKG_UPDATE="dnf update -y"
PKG_INSTALL="dnf install -y"
;;
fedora)
PKG_MGR="dnf"
PKG_UPDATE="dnf update -y"
PKG_INSTALL="dnf install -y"
;;
amzn)
PKG_MGR="yum"
PKG_UPDATE="yum update -y"
PKG_INSTALL="yum install -y"
;;
*)
print_error "Unsupported distribution: $ID"
exit 1
;;
esac
else
print_error "Cannot detect distribution. /etc/os-release not found."
exit 1
fi
print_status "Detected distribution: $PRETTY_NAME"
# [1/8] Update system packages
print_status "[1/8] Updating system packages..."
$PKG_UPDATE
$PKG_INSTALL wget curl tar systemd
# [2/8] Create Alertmanager user
print_status "[2/8] Creating Alertmanager system user..."
if ! id "$ALERTMANAGER_USER" &>/dev/null; then
useradd --no-create-home --shell /bin/false --system "$ALERTMANAGER_USER"
print_status "User $ALERTMANAGER_USER created successfully"
else
print_warning "User $ALERTMANAGER_USER already exists"
fi
# [3/8] Download and install Alertmanager
print_status "[3/8] Downloading and installing Alertmanager v$ALERTMANAGER_VERSION..."
cd /tmp
wget -q "https://github.com/prometheus/alertmanager/releases/download/v${ALERTMANAGER_VERSION}/alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz"
tar xzf "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz"
cp "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/alertmanager" /usr/local/bin/
cp "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/amtool" /usr/local/bin/
chown "$ALERTMANAGER_USER:$ALERTMANAGER_USER" /usr/local/bin/alertmanager /usr/local/bin/amtool
chmod 755 /usr/local/bin/alertmanager /usr/local/bin/amtool
rm -rf "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64" "alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz"
# [4/8] Create directories
print_status "[4/8] Creating directories and setting permissions..."
mkdir -p /etc/alertmanager /var/lib/alertmanager
chown "$ALERTMANAGER_USER:$ALERTMANAGER_USER" /etc/alertmanager /var/lib/alertmanager
chmod 755 /etc/alertmanager /var/lib/alertmanager
# [5/8] Create Alertmanager configuration
print_status "[5/8] Creating Alertmanager configuration..."
# Set default values if not provided
if [[ -z "$SMTP_USER" ]]; then
SMTP_USER="alerts@example.com"
print_warning "No SMTP user provided, using default: $SMTP_USER"
fi
if [[ -z "$SMTP_PASS" ]]; then
SMTP_PASS="your-app-password"
print_warning "No SMTP password provided, using placeholder"
fi
if [[ -z "$ADMIN_EMAIL" ]]; then
ADMIN_EMAIL="admin@example.com"
print_warning "No admin email provided, using default: $ADMIN_EMAIL"
fi
if [[ -z "$SLACK_WEBHOOK" ]]; then
SLACK_WEBHOOK="https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK"
print_warning "No Slack webhook provided, using placeholder"
fi
cat > "$CONFIG_FILE" << EOF
global:
smtp_smarthost: '$SMTP_HOST'
smtp_from: '$SMTP_USER'
smtp_auth_username: '$SMTP_USER'
smtp_auth_password: '$SMTP_PASS'
smtp_require_tls: true
route:
group_by: ['alertname', 'cluster', 'service']
group_wait: 10s
group_interval: 10s
repeat_interval: 1h
receiver: 'default-receiver'
routes:
- match:
severity: critical
receiver: 'critical-alerts'
- match:
severity: warning
receiver: 'warning-alerts'
receivers:
- name: 'default-receiver'
email_configs:
- to: '$ADMIN_EMAIL'
subject: 'Prometheus Alert: {{ .GroupLabels.alertname }}'
body: |
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Description: {{ .Annotations.description }}
Labels: {{ .Labels }}
{{ end }}
- name: 'critical-alerts'
email_configs:
- to: '$ADMIN_EMAIL'
subject: 'CRITICAL: {{ .GroupLabels.alertname }}'
body: |
CRITICAL ALERT TRIGGERED
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Description: {{ .Annotations.description }}
Severity: {{ .Labels.severity }}
Instance: {{ .Labels.instance }}
Started: {{ .StartsAt }}
{{ end }}
slack_configs:
- api_url: '$SLACK_WEBHOOK'
channel: '#alerts'
title: 'Critical Alert: {{ .GroupLabels.alertname }}'
text: |
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Description: {{ .Annotations.description }}
Severity: {{ .Labels.severity }}
Instance: {{ .Labels.instance }}
{{ end }}
send_resolved: true
- name: 'warning-alerts'
email_configs:
- to: '$ADMIN_EMAIL'
subject: 'Warning: {{ .GroupLabels.alertname }}'
body: |
{{ range .Alerts }}
Alert: {{ .Annotations.summary }}
Description: {{ .Annotations.description }}
Labels: {{ .Labels }}
{{ end }}
EOF
chown "$ALERTMANAGER_USER:$ALERTMANAGER_USER" "$CONFIG_FILE"
chmod 644 "$CONFIG_FILE"
# [6/8] Create systemd service
print_status "[6/8] Creating systemd service..."
cat > "$SERVICE_FILE" << EOF
[Unit]
Description=Prometheus Alertmanager
Wants=network-online.target
After=network-online.target
[Service]
User=$ALERTMANAGER_USER
Group=$ALERTMANAGER_USER
Type=simple
ExecStart=/usr/local/bin/alertmanager \\
--config.file=/etc/alertmanager/alertmanager.yml \\
--storage.path=/var/lib/alertmanager/ \\
--web.listen-address=0.0.0.0:9093 \\
--web.external-url=http://localhost:9093/ \\
--cluster.listen-address=""
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
chmod 644 "$SERVICE_FILE"
# [7/8] Configure firewall (if available)
print_status "[7/8] Configuring firewall..."
if command -v ufw >/dev/null 2>&1 && ufw status | grep -q "Status: active"; then
ufw allow 9093/tcp comment "Alertmanager"
elif command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active firewalld >/dev/null 2>&1; then
firewall-cmd --permanent --add-port=9093/tcp
firewall-cmd --reload
else
print_warning "No active firewall detected or unsupported firewall"
fi
# [8/8] Start and enable service
print_status "[8/8] Starting and enabling Alertmanager service..."
systemctl daemon-reload
systemctl enable alertmanager
systemctl start alertmanager
# Verification
print_status "Verifying installation..."
sleep 3
if systemctl is-active alertmanager >/dev/null 2>&1; then
print_status "✓ Alertmanager service is running"
else
print_error "✗ Alertmanager service is not running"
systemctl status alertmanager --no-pager
exit 1
fi
if curl -s http://localhost:9093 >/dev/null; then
print_status "✓ Alertmanager web interface is accessible"
else
print_warning "✗ Alertmanager web interface is not accessible on port 9093"
fi
print_status "Alertmanager installation completed successfully!"
echo ""
echo -e "${BLUE}Configuration Summary:${NC}"
echo " • Service: systemctl {start|stop|restart|status} alertmanager"
echo " • Config: $CONFIG_FILE"
echo " • Web UI: http://localhost:9093"
echo " • User: $ALERTMANAGER_USER"
echo " • Data: /var/lib/alertmanager"
echo ""
echo -e "${YELLOW}Next Steps:${NC}"
echo " 1. Update SMTP credentials in $CONFIG_FILE if needed"
echo " 2. Configure Slack webhook URL if needed"
echo " 3. Test notifications with: curl -XPOST http://localhost:9093/api/v1/alerts -d '[{\"labels\":{\"alertname\":\"test\"}}]'"
echo " 4. Configure Prometheus to send alerts to http://localhost:9093"
trap - ERR
Review the script before running. Execute with: bash install.sh