Configure advanced Grafana dashboards and alerting with Prometheus integration

Configure Prometheus data source with advanced settings

Set up the Prometheus data source in Grafana with optimized query settings and authentication. This configuration enables efficient metric queries and proper security integration.

curl -X POST http://admin:admin@localhost:3000/api/datasources \
  -H "Content-Type: application/json" \
  -d '{
    "name": "prometheus-advanced",
    "type": "prometheus",
    "url": "http://localhost:9090",
    "access": "proxy",
    "basicAuth": false,
    "jsonData": {
      "timeInterval": "15s",
      "queryTimeout": "60s",
      "httpMethod": "POST",
      "customQueryParameters": "",
      "manageAlerts": true,
      "exemplarTraceIdDestinations": []
    }
  }'

Create dashboard provisioning directory structure

Set up the directory structure for dashboard provisioning to enable version control and automated deployment of dashboard configurations.

sudo mkdir -p /etc/grafana/provisioning/dashboards
sudo mkdir -p /etc/grafana/provisioning/datasources
sudo mkdir -p /etc/grafana/provisioning/alerting
sudo mkdir -p /var/lib/grafana/dashboards

sudo chown -R grafana:grafana /etc/grafana/provisioning
sudo chown -R grafana:grafana /var/lib/grafana/dashboards
sudo chmod -R 755 /etc/grafana/provisioning
sudo chmod -R 755 /var/lib/grafana/dashboards

Configure dashboard provisioning provider

Create the dashboard provisioning configuration to automatically load dashboards from the file system and enable version control integration.

apiVersion: 1

providers:
  - name: 'production-dashboards'
    type: file
    disableDeletion: false
    updateIntervalSeconds: 30
    allowUiUpdates: true
    options:
      path: /var/lib/grafana/dashboards
      foldersFromFilesStructure: true

  - name: 'infrastructure-dashboards'
    type: file
    disableDeletion: false
    updateIntervalSeconds: 30
    allowUiUpdates: false
    options:
      path: /var/lib/grafana/dashboards/infrastructure
      foldersFromFilesStructure: true

Create advanced system monitoring dashboard with templating

Build a comprehensive system monitoring dashboard with variables for dynamic server selection and advanced templating features for scalable monitoring.

sudo mkdir -p /var/lib/grafana/dashboards/infrastructure

{
  "dashboard": {
    "id": null,
    "title": "Advanced System Monitoring",
    "tags": ["infrastructure", "system", "monitoring"],
    "timezone": "browser",
    "refresh": "30s",
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "templating": {
      "list": [
        {
          "name": "instance",
          "type": "query",
          "label": "Instance",
          "multi": true,
          "includeAll": true,
          "allValue": ".*",
          "query": "label_values(up, instance)",
          "datasource": "prometheus-advanced",
          "refresh": 1,
          "regex": "",
          "sort": 1
        },
        {
          "name": "job",
          "type": "query",
          "label": "Job",
          "multi": true,
          "includeAll": true,
          "allValue": ".*",
          "query": "label_values(up{instance=~\"$instance\"}, job)",
          "datasource": "prometheus-advanced",
          "refresh": 1,
          "regex": "",
          "sort": 1
        }
      ]
    },
    "panels": [
      {
        "id": 1,
        "title": "CPU Usage by Instance",
        "type": "timeseries",
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
        "targets": [
          {
            "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"$instance\",job=~\"$job\"}[5m])) * 100)",
            "legendFormat": "{{instance}}",
            "refId": "A"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "percent",
            "min": 0,
            "max": 100,
            "thresholds": {
              "steps": [
                {"color": "green", "value": null},
                {"color": "yellow", "value": 70},
                {"color": "red", "value": 90}
              ]
            }
          }
        }
      },
      {
        "id": 2,
        "title": "Memory Usage",
        "type": "timeseries",
        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
        "targets": [
          {
            "expr": "(1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\",job=~\"$job\"} / node_memory_MemTotal_bytes{instance=~\"$instance\",job=~\"$job\"})) * 100",
            "legendFormat": "{{instance}}",
            "refId": "A"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "percent",
            "min": 0,
            "max": 100,
            "thresholds": {
              "steps": [
                {"color": "green", "value": null},
                {"color": "yellow", "value": 80},
                {"color": "red", "value": 95}
              ]
            }
          }
        }
      }
    ]
  }
}

Create application performance dashboard with custom queries

Build an application-focused dashboard with advanced PromQL queries for request rates, error tracking, and performance analysis across multiple services.

{
  "dashboard": {
    "id": null,
    "title": "Application Performance Monitoring",
    "tags": ["application", "performance", "sla"],
    "timezone": "browser",
    "refresh": "15s",
    "time": {
      "from": "now-30m",
      "to": "now"
    },
    "templating": {
      "list": [
        {
          "name": "service",
          "type": "query",
          "label": "Service",
          "multi": true,
          "includeAll": true,
          "query": "label_values(http_requests_total, service)",
          "datasource": "prometheus-advanced"
        }
      ]
    },
    "panels": [
      {
        "id": 1,
        "title": "Request Rate (RPS)",
        "type": "stat",
        "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{service=~\"$service\"}[5m]))",
            "refId": "A"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "reqps",
            "color": {"mode": "thresholds"},
            "thresholds": {
              "steps": [
                {"color": "green", "value": null},
                {"color": "yellow", "value": 100},
                {"color": "red", "value": 500}
              ]
            }
          }
        }
      },
      {
        "id": 2,
        "title": "Error Rate",
        "type": "stat",
        "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
        "targets": [
          {
            "expr": "(sum(rate(http_requests_total{service=~\"$service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{service=~\"$service\"}[5m]))) * 100",
            "refId": "A"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "percent",
            "color": {"mode": "thresholds"},
            "thresholds": {
              "steps": [
                {"color": "green", "value": null},
                {"color": "yellow", "value": 1},
                {"color": "red", "value": 5}
              ]
            }
          }
        }
      },
      {
        "id": 3,
        "title": "Response Time P95",
        "type": "timeseries",
        "gridPos": {"h": 8, "w": 24, "x": 0, "y": 4},
        "targets": [
          {
            "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{service=~\"$service\"}[5m])) by (le, service))",
            "legendFormat": "{{service}} - P95",
            "refId": "A"
          },
          {
            "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{service=~\"$service\"}[5m])) by (le, service))",
            "legendFormat": "{{service}} - P50",
            "refId": "B"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "s",
            "thresholds": {
              "steps": [
                {"color": "green", "value": null},
                {"color": "yellow", "value": 0.5},
                {"color": "red", "value": 2}
              ]
            }
          }
        }
      }
    ]
  }
}

Configure alert rule provisioning

Set up alert rule provisioning to manage alerting rules as code and enable version control for alert configurations.

apiVersion: 1

groups:
  - name: infrastructure-alerts
    orgId: 1
    folder: Infrastructure
    interval: 1m
    rules:
      - uid: cpu-high-usage
        title: High CPU Usage
        condition: A
        data:
          - refId: A
            queryType: ""
            relativeTimeRange:
              from: 300
              to: 0
            datasourceUid: prometheus-advanced
            model:
              expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90'
              intervalMs: 1000
              maxDataPoints: 43200
              refId: A
        noDataState: NoData
        execErrState: Alerting
        for: 5m
        annotations:
          description: "CPU usage is above 90% on {{ $labels.instance }}"
          runbook_url: "https://example.com/runbooks/high-cpu"
          summary: "High CPU usage detected"
        labels:
          severity: warning
          team: infrastructure

      - uid: memory-high-usage
        title: High Memory Usage
        condition: A
        data:
          - refId: A
            queryType: ""
            relativeTimeRange:
              from: 300
              to: 0
            datasourceUid: prometheus-advanced
            model:
              expr: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 95'
              intervalMs: 1000
              maxDataPoints: 43200
              refId: A
        noDataState: NoData
        execErrState: Alerting
        for: 3m
        annotations:
          description: "Memory usage is above 95% on {{ $labels.instance }}"
          runbook_url: "https://example.com/runbooks/high-memory"
          summary: "High memory usage detected"
        labels:
          severity: critical
          team: infrastructure

Configure notification channels

Set up multiple notification channels including Slack, email, and webhook integrations for comprehensive alert delivery across different communication platforms.

apiVersion: 1

notificationPolicies:
  - orgId: 1
    receiver: default-receiver
    group_by:
      - grafana_folder
      - alertname
    group_wait: 10s
    group_interval: 10s
    repeat_interval: 1h
    routes:
      - receiver: infrastructure-slack
        object_matchers:
          - ['team', '=', 'infrastructure']
        group_by: ['alertname', 'instance']
        group_wait: 5s
        group_interval: 5s
        repeat_interval: 30m
        routes:
          - receiver: infrastructure-pager
            object_matchers:
              - ['severity', '=', 'critical']
            group_wait: 0s
            repeat_interval: 15m

      - receiver: application-team
        object_matchers:
          - ['team', '=', 'application']
        group_by: ['alertname', 'service']
        group_wait: 10s
        repeat_interval: 1h

Configure contact points for notifications

Create contact points for different notification channels with proper authentication and formatting for effective alert delivery.

apiVersion: 1

contactPoints:
  - orgId: 1
    name: default-receiver
    receivers:
      - uid: default-email
        type: email
        settings:
          addresses: "admin@example.com"
          subject: "Grafana Alert: {{ .GroupLabels.alertname }}"
          message: |
            {{ range .Alerts }}
            Alert: {{ .Annotations.summary }}
            Description: {{ .Annotations.description }}
            Labels:
            {{ range .Labels.SortedPairs }}
              {{ .Name }}: {{ .Value }}
            {{ end }}
            {{ end }}

  - orgId: 1
    name: infrastructure-slack
    receivers:
      - uid: infra-slack
        type: slack
        settings:
          url: "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK"
          channel: "#infrastructure-alerts"
          username: "Grafana"
          title: "{{ .GroupLabels.alertname }}"
          text: |
            {{ range .Alerts }}
            *Alert:* {{ .Annotations.summary }}
            *Description:* {{ .Annotations.description }}
            *Instance:* {{ .Labels.instance }}
            *Runbook:* {{ .Annotations.runbook_url }}
            {{ end }}

  - orgId: 1
    name: infrastructure-pager
    receivers:
      - uid: infra-webhook
        type: webhook
        settings:
          url: "https://events.pagerduty.com/integration/YOUR_INTEGRATION_KEY/enqueue"
          httpMethod: "POST"
          maxAlerts: 0
          message: |
            {
              "routing_key": "YOUR_INTEGRATION_KEY",
              "event_action": "trigger",
              "dedup_key": "{{ .GroupLabels.alertname }}-{{ .GroupLabels.instance }}",
              "payload": {
                "summary": "{{ .GroupLabels.alertname }}: {{ .Annotations.summary }}",
                "source": "{{ .Labels.instance }}",
                "severity": "{{ .Labels.severity }}",
                "custom_details": {
                  "description": "{{ .Annotations.description }}",
                  "runbook_url": "{{ .Annotations.runbook_url }}"
                }
              }
            }

  - orgId: 1
    name: application-team
    receivers:
      - uid: app-email
        type: email
        settings:
          addresses: "developers@example.com"
          subject: "Application Alert: {{ .GroupLabels.alertname }}"
          message: |
            Service: {{ .Labels.service }}
            Alert: {{ .Annotations.summary }}
            Description: {{ .Annotations.description }}
            
            Dashboard: http://grafana.example.com/d/app-perf
            Runbook: {{ .Annotations.runbook_url }}

Create application alert rules

Configure application-specific alert rules for monitoring service health, error rates, and performance thresholds with appropriate severity levels.

apiVersion: 1

groups:
  - name: application-alerts
    orgId: 1
    folder: Applications
    interval: 30s
    rules:
      - uid: high-error-rate
        title: High Error Rate
        condition: A
        data:
          - refId: A
            queryType: ""
            relativeTimeRange:
              from: 300
              to: 0
            datasourceUid: prometheus-advanced
            model:
              expr: '(sum(rate(http_requests_total{status=~"5.."}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)) * 100 > 5'
              intervalMs: 1000
              maxDataPoints: 43200
              refId: A
        noDataState: NoData
        execErrState: Alerting
        for: 2m
        annotations:
          description: "Error rate is {{ $value }}% for service {{ $labels.service }}"
          runbook_url: "https://example.com/runbooks/high-error-rate"
          summary: "High error rate detected in {{ $labels.service }}"
        labels:
          severity: critical
          team: application

      - uid: slow-response-time
        title: Slow Response Time
        condition: A
        data:
          - refId: A
            queryType: ""
            relativeTimeRange:
              from: 600