Set up Prometheus Blackbox Exporter for uptime monitoring with SSL and alerting

Update system packages

Start by updating your package manager and installing dependencies.

sudo apt update && sudo apt upgrade -y
sudo apt install -y curl wget tar

sudo dnf update -y
sudo dnf install -y curl wget tar

Download and install Blackbox Exporter

Download the latest Blackbox Exporter release and install it to the system.

cd /tmp
wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.24.0/blackbox_exporter-0.24.0.linux-amd64.tar.gz
tar -xzf blackbox_exporter-0.24.0.linux-amd64.tar.gz
sudo mv blackbox_exporter-0.24.0.linux-amd64/blackbox_exporter /usr/local/bin/
sudo chmod +x /usr/local/bin/blackbox_exporter

Create system user and directories

Create a dedicated user and the necessary directories for Blackbox Exporter.

sudo useradd --system --no-create-home --shell /bin/false blackbox_exporter
sudo mkdir -p /etc/blackbox_exporter
sudo chown blackbox_exporter:blackbox_exporter /etc/blackbox_exporter

Configure Blackbox Exporter

Create the main configuration file with HTTP, HTTPS, and SSL certificate monitoring modules.

modules:
  http_2xx:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200, 201, 202, 204]
      method: GET
      follow_redirects: true
      fail_if_ssl: false
      fail_if_not_ssl: false
      preferred_ip_protocol: "ip4"
      
  http_post_2xx:
    prober: http
    timeout: 5s
    http:
      method: POST
      headers:
        Content-Type: application/json
      body: '{}'
      valid_status_codes: [200, 201, 202]
      
  https_2xx:
    prober: http
    timeout: 10s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200, 201, 202, 204]
      method: GET
      follow_redirects: true
      fail_if_ssl: false
      fail_if_not_ssl: true
      preferred_ip_protocol: "ip4"
      tls_config:
        insecure_skip_verify: false
        
  ssl_expiry:
    prober: http
    timeout: 10s
    http:
      method: GET
      fail_if_not_ssl: true
      preferred_ip_protocol: "ip4"
      tls_config:
        insecure_skip_verify: false
        
  tcp_connect:
    prober: tcp
    timeout: 5s
    tcp:
      preferred_ip_protocol: "ip4"
      
  dns_query:
    prober: dns
    timeout: 5s
    dns:
      query_name: "example.com"
      query_type: "A"
      valid_rcodes:
        - NOERROR
      preferred_ip_protocol: "ip4"
      
  icmp_ping:
    prober: icmp
    timeout: 5s
    icmp:
      preferred_ip_protocol: "ip4"

Set configuration permissions

Set the correct ownership and permissions for the configuration file.

sudo chown blackbox_exporter:blackbox_exporter /etc/blackbox_exporter/blackbox.yml
sudo chmod 640 /etc/blackbox_exporter/blackbox.yml

Create systemd service

Create a systemd service file to manage the Blackbox Exporter daemon.

[Unit]
Description=Blackbox Exporter
After=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
Restart=always
RestartSec=1
User=blackbox_exporter
Group=blackbox_exporter
ExecStart=/usr/local/bin/blackbox_exporter \
  --config.file=/etc/blackbox_exporter/blackbox.yml \
  --web.listen-address=:9115
SyslogIdentifier=blackbox_exporter
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target

Enable and start Blackbox Exporter

Enable the service to start on boot and start it immediately.

sudo systemctl daemon-reload
sudo systemctl enable --now blackbox_exporter
sudo systemctl status blackbox_exporter

Configure firewall access

Allow access to the Blackbox Exporter port from your Prometheus server.

sudo ufw allow from 203.0.113.10 to any port 9115 comment 'Prometheus Blackbox Exporter'
sudo ufw reload

sudo firewall-cmd --permanent --add-rich-rule="rule family='ipv4' source address='203.0.113.10' port protocol='tcp' port='9115' accept"
sudo firewall-cmd --reload

Add Blackbox Exporter job to Prometheus

Configure Prometheus to scrape metrics from Blackbox Exporter and define your monitoring targets.

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "blackbox_alerts.yml"

scrape_configs:
  - job_name: 'blackbox_exporter'
    static_configs:
      - targets: ['localhost:9115']
      
  - job_name: 'blackbox_http'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - http://example.com
        - http://api.example.com/health
        - http://status.example.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115
        
  - job_name: 'blackbox_https'
    metrics_path: /probe
    params:
      module: [https_2xx]
    static_configs:
      - targets:
        - https://example.com
        - https://api.example.com
        - https://dashboard.example.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115
        
  - job_name: 'blackbox_ssl'
    metrics_path: /probe
    params:
      module: [ssl_expiry]
    static_configs:
      - targets:
        - https://example.com
        - https://api.example.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115
        
  - job_name: 'blackbox_tcp'
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets:
        - example.com:443
        - database.example.com:5432
        - redis.example.com:6379
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115

Create alerting rules

Define alert rules for website downtime, SSL certificate expiry, and slow response times.

groups:
  - name: blackbox_alerts
    rules:
    - alert: WebsiteDown
      expr: probe_success == 0
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: "Website {{ $labels.instance }} is down"
        description: "Website {{ $labels.instance }} has been down for more than 5 minutes."
        
    - alert: SSLCertExpiringSoon
      expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
      for: 1h
      labels:
        severity: warning
      annotations:
        summary: "SSL certificate for {{ $labels.instance }} expires soon"
        description: "SSL certificate for {{ $labels.instance }} will expire in {{ $value | humanizeDuration }}."
        
    - alert: SSLCertExpiring
      expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7
      for: 1h
      labels:
        severity: critical
      annotations:
        summary: "SSL certificate for {{ $labels.instance }} expires very soon"
        description: "SSL certificate for {{ $labels.instance }} will expire in {{ $value | humanizeDuration }}."
        
    - alert: HighResponseTime
      expr: probe_duration_seconds > 5
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "High response time for {{ $labels.instance }}"
        description: "{{ $labels.instance }} response time is {{ $value }}s which is above the 5s threshold."
        
    - alert: HTTPStatusError
      expr: probe_http_status_code >= 400
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: "HTTP error for {{ $labels.instance }}"
        description: "{{ $labels.instance }} returned HTTP status {{ $value }}."
        
    - alert: TCPConnectionFailed
      expr: probe_success{job="blackbox_tcp"} == 0
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: "TCP connection failed for {{ $labels.instance }}"
        description: "TCP connection to {{ $labels.instance }} has been failing for more than 2 minutes."

Restart Prometheus

Reload Prometheus configuration to apply the new scrape jobs and alert rules.

sudo systemctl restart prometheus
sudo systemctl status prometheus

Import Blackbox Exporter dashboard

Access your Grafana instance and import a comprehensive dashboard for Blackbox monitoring.

{
  "dashboard": {
    "id": null,
    "title": "Blackbox Exporter Overview",
    "tags": ["blackbox", "monitoring"],
    "timezone": "browser",
    "panels": [
      {
        "title": "Website Status",
        "type": "stat",
        "targets": [
          {
            "expr": "probe_success",
            "legendFormat": "{{ instance }}"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "mappings": [
              {
                "options": {
                  "0": { "text": "DOWN", "color": "red" },
                  "1": { "text": "UP", "color": "green" }
                },
                "type": "value"
              }
            ],
            "thresholds": {
              "steps": [
                { "color": "red", "value": 0 },
                { "color": "green", "value": 1 }
              ]
            }
          }
        },
        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }
      },
      {
        "title": "SSL Certificate Expiry (Days)",
        "type": "stat",
        "targets": [
          {
            "expr": "(probe_ssl_earliest_cert_expiry - time()) / 86400",
            "legendFormat": "{{ instance }}"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "d",
            "thresholds": {
              "steps": [
                { "color": "red", "value": 0 },
                { "color": "yellow", "value": 7 },
                { "color": "green", "value": 30 }
              ]
            }
          }
        },
        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }
      },
      {
        "title": "Response Time",
        "type": "timeseries",
        "targets": [
          {
            "expr": "probe_duration_seconds",
            "legendFormat": "{{ instance }}"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "s"
          }
        },
        "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 }
      }
    ],
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "refresh": "30s"
  }
}

Configure Grafana alerting

Set up alert rules in Grafana that complement your Prometheus alerts. In this tutorial, we'll focus on setting up Prometheus Alertmanager with email notifications for centralized alert management.

curl -X POST http://admin:admin@localhost:3000/api/dashboards/db \
  -H "Content-Type: application/json" \
  -d @dashboard.json

Add custom SSL monitoring

Create additional modules for specific SSL requirements and certificate chain validation.

modules:
  ssl_detailed:
    prober: http
    timeout: 10s
    http:
      method: GET
      fail_if_not_ssl: true
      preferred_ip_protocol: "ip4"
      tls_config:
        insecure_skip_verify: false
        ca_file: "/etc/ssl/certs/ca-certificates.crt"
        
  ssl_client_auth:
    prober: http
    timeout: 10s
    http:
      method: GET
      fail_if_not_ssl: true
      tls_config:
        cert_file: "/etc/blackbox_exporter/client.crt"
        key_file: "/etc/blackbox_exporter/client.key"
        ca_file: "/etc/ssl/certs/ca-certificates.crt"
        
  http_basic_auth:
    prober: http
    timeout: 5s
    http:
      valid_status_codes: [200, 201, 202]
      method: GET
      basic_auth:
        username: "monitor"
        password: "secure_password"
        
  http_custom_headers:
    prober: http
    timeout: 5s
    http:
      valid_status_codes: [200]
      method: GET
      headers:
        Authorization: "Bearer your_token_here"
        User-Agent: "Blackbox-Monitor/1.0"
        X-API-Key: "your_api_key_here"

Restart Blackbox Exporter

Apply the new configuration by restarting the service.

sudo systemctl restart blackbox_exporter
sudo systemctl status blackbox_exporter

Set up multi-location monitoring

For comprehensive uptime monitoring, you can deploy Blackbox Exporter on multiple servers. This approach provides redundancy and helps distinguish between local network issues and actual service problems.

scrape_configs:
  - job_name: 'blackbox_primary'
    metrics_path: /probe
    params:
      module: [https_2xx]
    static_configs:
      - targets:
        - https://example.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: primary-monitor.example.com:9115
      - target_label: location
        replacement: primary
        
  - job_name: 'blackbox_secondary'
    metrics_path: /probe
    params:
      module: [https_2xx]
    static_configs:
      - targets:
        - https://example.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: secondary-monitor.example.com:9115
      - target_label: location
        replacement: secondary