Set up Prometheus and Grafana monitoring stack with Docker Compose

Install Docker and Docker Compose

First, install Docker and Docker Compose on your system. These tools will manage our monitoring containers.

sudo apt update
sudo apt install -y docker.io docker-compose-plugin
sudo systemctl enable --now docker
sudo usermod -aG docker $USER

sudo dnf update -y
sudo dnf install -y docker docker-compose-plugin
sudo systemctl enable --now docker
sudo usermod -aG docker $USER

Log out and back in for the group membership to take effect, or run:

newgrp docker

Create project directory structure

Create a dedicated directory for your monitoring stack with subdirectories for configuration files and data persistence.

mkdir -p monitoring-stack/{prometheus,grafana/provisioning/{datasources,dashboards}}
cd monitoring-stack

Configure Prometheus

Create the main Prometheus configuration file that defines scrape targets and retention policies.

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alert_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - alertmanager:9093

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'cadvisor'
    static_configs:
      - targets: ['cadvisor:8080']

  - job_name: 'grafana'
    static_configs:
      - targets: ['grafana:3000']

Create Prometheus alert rules

Define alert conditions for common monitoring scenarios like high CPU usage and service downtime.

groups:
  - name: system_alerts
    rules:
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage detected on {{ $labels.instance }}"
          description: "CPU usage is above 80% for more than 5 minutes on {{ $labels.instance }}"

      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage detected on {{ $labels.instance }}"
          description: "Memory usage is above 85% for more than 5 minutes on {{ $labels.instance }}"

      - alert: ServiceDown
        expr: up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Service {{ $labels.job }} is down"
          description: "Service {{ $labels.job }} on {{ $labels.instance }} has been down for more than 2 minutes"

Configure Grafana datasource

Set up Prometheus as a datasource for Grafana with automatic provisioning.

apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true

Configure Grafana dashboard provisioning

Enable automatic dashboard loading from the dashboards directory.

apiVersion: 1

providers:
  - name: 'default'
    orgId: 1
    folder: ''
    folderUid: ''
    type: file
    disableDeletion: false
    editable: true
    updateIntervalSeconds: 10
    allowUiUpdates: true
    options:
      path: /etc/grafana/provisioning/dashboards

Create Alertmanager configuration

Configure Alertmanager to handle alert routing and notifications via email and Slack.

global:
  smtp_smarthost: 'localhost:587'
  smtp_from: 'alerts@example.com'

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  receiver: 'web.hook'

receivers:
  - name: 'web.hook'
    email_configs:
      - to: 'admin@example.com'
        subject: '[ALERT] {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        body: |
          {{ range .Alerts }}
          Alert: {{ .Annotations.summary }}
          Description: {{ .Annotations.description }}
          Instance: {{ .Labels.instance }}
          Severity: {{ .Labels.severity }}
          {{ end }}

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

Create Docker Compose configuration

Define all services including Prometheus, Grafana, Alertmanager, and system monitoring exporters.

version: '3.8'

services:
  prometheus:
    image: prom/prometheus:v2.47.2
    container_name: prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=200h'
      - '--web.enable-lifecycle'
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
    networks:
      - monitoring

  alertmanager:
    image: prom/alertmanager:v0.26.0
    container_name: alertmanager
    restart: unless-stopped
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager:/etc/alertmanager
    command:
      - '--config.file=/etc/alertmanager/alertmanager.yml'
      - '--storage.path=/alertmanager'
      - '--web.external-url=http://localhost:9093'
    networks:
      - monitoring

  grafana:
    image: grafana/grafana:10.2.0
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin123
      - GF_USERS_ALLOW_SIGN_UP=false
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
    networks:
      - monitoring

  node-exporter:
    image: prom/node-exporter:v1.6.1
    container_name: node-exporter
    restart: unless-stopped
    ports:
      - "9100:9100"
    command:
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    networks:
      - monitoring

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.47.2
    container_name: cadvisor
    restart: unless-stopped
    ports:
      - "8080:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:rw
      - /sys:/sys:ro
      - /var/lib/docker:/var/lib/docker:ro
      - /dev/disk:/dev/disk:ro
    devices:
      - /dev/kmsg:/dev/kmsg
    networks:
      - monitoring

volumes:
  prometheus_data:
  grafana_data:

networks:
  monitoring:
    driver: bridge

Set proper permissions

Ensure the monitoring containers can read configuration files and write data to mounted volumes.

sudo chown -R 472:472 grafana/
sudo chmod -R 755 prometheus/ alertmanager/

Start the monitoring stack

Launch all monitoring services with Docker Compose and verify they start successfully.

docker compose up -d
docker compose ps

Configure basic system dashboard

Create a comprehensive system monitoring dashboard for immediate insights into server performance.

{
  "dashboard": {
    "id": null,
    "title": "System Overview",
    "tags": ["system", "monitoring"],
    "style": "dark",
    "timezone": "browser",
    "panels": [
      {
        "id": 1,
        "title": "CPU Usage",
        "type": "stat",
        "targets": [
          {
            "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
            "refId": "A"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "percent",
            "min": 0,
            "max": 100
          }
        },
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
      },
      {
        "id": 2,
        "title": "Memory Usage",
        "type": "stat",
        "targets": [
          {
            "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
            "refId": "A"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "percent",
            "min": 0,
            "max": 100
          }
        },
        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
      }
    ],
    "time": {
      "from": "now-6h",
      "to": "now"
    },
    "refresh": "30s"
  }
}

Set up email notifications

Configure SMTP settings in the Alertmanager configuration for email alerts. Update the alertmanager.yml file with your email server details.

global:
  smtp_smarthost: 'smtp.gmail.com:587'
  smtp_from: 'monitoring@example.com'
  smtp_auth_username: 'monitoring@example.com'
  smtp_auth_password: 'your-app-password'

route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 12h
  receiver: 'email-notifications'
  routes:
    - match:
        severity: critical
      receiver: 'critical-alerts'
      group_wait: 5s
      repeat_interval: 5m

receivers:
  - name: 'email-notifications'
    email_configs:
      - to: 'admin@example.com'
        subject: '[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}'
        html: |
          
Alert Details
          {{ range .Alerts }}
          
Alert: {{ .Annotations.summary }}
          
Description: {{ .Annotations.description }}
          
Instance: {{ .Labels.instance }}
          
Severity: {{ .Labels.severity }}
          
Started: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
          

          {{ end }}

  - name: 'critical-alerts'
    email_configs:
      - to: 'oncall@example.com'
        subject: '[CRITICAL] {{ .GroupLabels.alertname }}'
        html: |
          
CRITICAL ALERT
          {{ range .Alerts }}
          
{{ .Annotations.summary }}
          
{{ .Annotations.description }}
          
Instance: {{ .Labels.instance }}
          
Started: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
          {{ end }}

Configure Slack notifications

Add Slack webhook integration to receive alerts in your team channels.

cat >> alertmanager/alertmanager.yml << 'EOF'

  - name: 'slack-notifications'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
        channel: '#alerts'
        username: 'Alertmanager'
        title: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        text: |
          {{ range .Alerts }}
          Alert: {{ .Annotations.summary }}
          Description: {{ .Annotations.description }}
          Instance: {{ .Labels.instance }}
          Severity: {{ .Labels.severity }}
          {{ end }}
EOF

Set up Grafana alerting

Configure Grafana's unified alerting system for dashboard-based alerts with multiple notification channels.

docker exec -it grafana grafana-cli admin reset-admin-password newpassword123

Access Grafana at http://your-server-ip:3000 and configure notification policies under Alerting → Notification policies.

Test alerting system

Trigger a test alert to verify your notification channels are working correctly.

# Stop node-exporter to trigger a service down alert
docker compose stop node-exporter

Check alert status in Prometheus
curl -s http://localhost:9090/api/v1/alerts | jq '.data.alerts[] | select(.labels.alertname=="ServiceDown")'

Restart the service
docker compose start node-exporter

Import community dashboards

Download popular community dashboards for comprehensive system monitoring.

# Download Node Exporter Full dashboard
curl -s https://grafana.com/api/dashboards/1860/revisions/37/download | jq '.dashboard' > grafana/provisioning/dashboards/node-exporter-full.json

Download Docker container dashboard
curl -s https://grafana.com/api/dashboards/193/revisions/2/download | jq '.dashboard' > grafana/provisioning/dashboards/docker-containers.json

Create custom application dashboard

Build a custom dashboard for monitoring your specific applications and services.

{
  "dashboard": {
    "id": null,
    "title": "Application Metrics",
    "tags": ["application", "custom"],
    "style": "dark",
    "timezone": "browser",
    "panels": [
      {
        "id": 1,
        "title": "HTTP Request Rate",
        "type": "graph",
        "targets": [
          {
            "expr": "rate(http_requests_total[5m])",
            "refId": "A",
            "legendFormat": "{{ method }} {{ status_code }}"
          }
        ],
        "yAxes": [
          {
            "label": "Requests/sec",
            "min": 0
          }
        ],
        "gridPos": {"h": 9, "w": 12, "x": 0, "y": 0}
      },
      {
        "id": 2,
        "title": "Response Time",
        "type": "graph",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))",
            "refId": "A",
            "legendFormat": "95th percentile"
          },
          {
            "expr": "histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))",
            "refId": "B",
            "legendFormat": "50th percentile"
          }
        ],
        "yAxes": [
          {
            "label": "Duration (seconds)",
            "min": 0
          }
        ],
        "gridPos": {"h": 9, "w": 12, "x": 12, "y": 0}
      }
    ],
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "refresh": "10s"
  }
}

Configure dashboard permissions

Set up role-based access control for dashboards to control who can view and edit monitoring data.

# Restart Grafana to load new dashboards
docker compose restart grafana

Wait for Grafana to start
sleep 30

Verify dashboards loaded
curl -s -u admin:admin123 http://localhost:3000/api/search | jq '.[].title'