Set up Prometheus and Grafana monitoring stack with Docker Compose

Update system packages and install dependencies

Start by updating your system and installing Docker and Docker Compose if not already present.

sudo apt update && sudo apt upgrade -y
sudo apt install -y docker.io docker-compose-v2
sudo systemctl enable --now docker
sudo usermod -aG docker $USER

sudo dnf update -y
sudo dnf install -y docker docker-compose
sudo systemctl enable --now docker
sudo usermod -aG docker $USER

Log out and back in for the Docker group changes to take effect.

Create the project directory structure

Set up directories for configuration files, data persistence, and the Docker Compose setup.

mkdir -p ~/monitoring-stack/{prometheus,grafana,alertmanager}
cd ~/monitoring-stack

Configure Prometheus

Create the Prometheus configuration file with basic scraping targets and Alertmanager integration.

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alert_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - alertmanager:9093

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'cadvisor'
    static_configs:
      - targets: ['cadvisor:8080']

  - job_name: 'alertmanager'
    static_configs:
      - targets: ['alertmanager:9093']

Create Prometheus alert rules

Set up basic alerting rules for system monitoring and container health.

groups:
name: system_alerts
  rules:
  - alert: HighCPUUsage
    expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "High CPU usage detected on {{ $labels.instance }}"
      description: "CPU usage is above 80% for more than 2 minutes on {{ $labels.instance }}"

  - alert: HighMemoryUsage
    expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "High memory usage detected on {{ $labels.instance }}"
      description: "Memory usage is above 85% for more than 2 minutes on {{ $labels.instance }}"

  - alert: DiskSpaceLow
    expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"})) * 100 > 90
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Disk space low on {{ $labels.instance }}"
      description: "Disk usage is above 90% on filesystem {{ $labels.mountpoint }}"

  - alert: ContainerDown
    expr: up == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Container {{ $labels.job }} is down"
      description: "Container {{ $labels.job }} on {{ $labels.instance }} has been down for more than 1 minute"

Configure Alertmanager

Set up Alertmanager to handle alert routing and notifications via email and Slack.

global:
  smtp_smarthost: 'localhost:587'
  smtp_from: 'alerts@example.com'

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  receiver: 'web.hook'

receivers:
name: 'web.hook'
  email_configs:
  - to: 'admin@example.com'
    subject: 'Alert: {{ .GroupLabels.alertname }}'
    body: |
      {{ range .Alerts }}
      Alert: {{ .Annotations.summary }}
      Description: {{ .Annotations.description }}
      {{ end }}
  
  # Uncomment and configure for Slack notifications
  # slack_configs:
  # - api_url: 'YOUR_SLACK_WEBHOOK_URL'
  #   channel: '#alerts'
  #   title: 'Alert: {{ .GroupLabels.alertname }}'
  #   text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

Create Grafana configuration

Set up Grafana with automatic Prometheus datasource provisioning and default dashboards.

mkdir -p ~/monitoring-stack/grafana/{provisioning/datasources,provisioning/dashboards,dashboards}

apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true

apiVersion: 1

providers:
  - name: 'Default'
    orgId: 1
    folder: ''
    folderUid: ''
    type: file
    disableDeletion: false
    editable: true
    updateIntervalSeconds: 10
    allowUiUpdates: true
    options:
      path: /var/lib/grafana/dashboards

Download pre-built dashboards

Get popular Grafana dashboards for system monitoring and Docker container metrics.

cd ~/monitoring-stack/grafana/dashboards

Node Exporter Full dashboard
curl -o node-exporter-full.json "https://grafana.com/api/dashboards/1860/revisions/37/download"

Docker Container & Host Metrics dashboard  
curl -o docker-monitoring.json "https://grafana.com/api/dashboards/193/revisions/5/download"

Prometheus Stats dashboard
curl -o prometheus-stats.json "https://grafana.com/api/dashboards/2/revisions/2/download"

Create the Docker Compose configuration

Define all services including Prometheus, Grafana, Alertmanager, and monitoring exporters.

version: '3.8'

networks:
  monitoring:
    driver: bridge

volumes:
  prometheus_data:
  grafana_data:
  alertmanager_data:

services:
  prometheus:
    image: prom/prometheus:v2.48.0
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--web.enable-lifecycle'
      - '--web.enable-admin-api'
    networks:
      - monitoring

  grafana:
    image: grafana/grafana:10.2.2
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
      - ./grafana/dashboards:/var/lib/grafana/dashboards
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin123!
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_INSTALL_PLUGINS=grafana-piechart-panel
    networks:
      - monitoring

  alertmanager:
    image: prom/alertmanager:v0.26.0
    container_name: alertmanager
    restart: unless-stopped
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager:/etc/alertmanager
      - alertmanager_data:/alertmanager
    command:
      - '--config.file=/etc/alertmanager/alertmanager.yml'
      - '--storage.path=/alertmanager'
      - '--web.external-url=http://localhost:9093'
    networks:
      - monitoring

  node-exporter:
    image: prom/node-exporter:v1.7.0
    container_name: node-exporter
    restart: unless-stopped
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    networks:
      - monitoring

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.47.2
    container_name: cadvisor
    restart: unless-stopped
    ports:
      - "8080:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    privileged: true
    devices:
      - /dev/kmsg
    networks:
      - monitoring

Set proper file permissions

Ensure the monitoring stack can read configuration files and write to data directories.

cd ~/monitoring-stack

Set ownership for Grafana (runs as user ID 472)
sudo chown -R 472:472 grafana/

Set proper permissions for configuration files
chmod -R 644 prometheus/ alertmanager/
chmod 755 prometheus/ alertmanager/

Make sure Docker can read the compose file
chmod 644 docker-compose.yml

Launch the monitoring stack

Start all services and verify they come up healthy.

docker compose up -d

Check that all containers are running
docker compose ps

View logs to ensure everything started properly
docker compose logs -f

Access Grafana web interface

Open Grafana in your browser and set up the initial configuration.

# Open Grafana at http://localhost:3000
Default credentials: admin / admin123!

Navigate to Dashboards to see the pre-loaded system monitoring dashboards. The Node Exporter Full dashboard provides comprehensive system metrics, while the Docker monitoring dashboard shows container performance.

Create custom dashboard

Build a custom dashboard for application-specific metrics and alerts.

1. Click + Create Dashboard in Grafana
2. Add a new panel and select Prometheus as the data source
3. Use queries like up to show service availability or rate(prometheus_http_requests_total[5m]) for request rates
4. Configure alert rules by clicking the Alert tab in the panel editor
5. Set conditions like IS BELOW 1 for the up metric to alert when services go down

Configure notification channels

Set up Grafana to send alerts via email, Slack, or webhooks.

1. Go to Alerting → Notification channels
2. Click Add channel and choose your preferred method
3. For email: Configure SMTP settings in /etc/grafana/grafana.ini
4. For Slack: Add your webhook URL and choose the channel
5. Test the notification to ensure delivery works

Add external targets to Prometheus

Configure Prometheus to monitor additional services and applications.

# Add this section to your existing prometheus.yml
  - job_name: 'nginx'
    static_configs:
      - targets: ['203.0.113.10:9113']  # nginx-prometheus-exporter

  - job_name: 'mysql'
    static_configs:
      - targets: ['203.0.113.11:9104']  # mysqld_exporter

  - job_name: 'redis'
    static_configs:
      - targets: ['203.0.113.12:9121']  # redis_exporter

  - job_name: 'blackbox'
    metrics_path: /probe
    params:
      module: [http_2xx]  # Look for a HTTP 200 response.
    static_configs:
      - targets:
        - https://example.com    # Target to probe
        - https://api.example.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

Reload Prometheus configuration without restarting:

curl -X POST http://localhost:9090/-/reload

Add blackbox exporter for uptime monitoring

Monitor website and API availability with HTTP probes.

mkdir ~/monitoring-stack/blackbox-exporter

modules:
  http_2xx:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: []
      method: GET
      follow_redirects: true
      preferred_ip_protocol: "ip4"
      
  http_post_2xx:
    prober: http
    timeout: 5s
    http:
      method: POST
      headers:
        Content-Type: application/json
      body: '{"test": "data"}'
      
  tcp_connect:
    prober: tcp
    timeout: 5s

Add the blackbox exporter service to your docker-compose.yml:

# Add this service to your existing docker-compose.yml
  blackbox-exporter:
    image: prom/blackbox-exporter:v0.24.0
    container_name: blackbox-exporter
    restart: unless-stopped
    ports:
      - "9115:9115"
    volumes:
      - ./blackbox-exporter:/etc/blackbox_exporter
    command:
      - '--config.file=/etc/blackbox_exporter/blackbox.yml'
    networks:
      - monitoring