Implement custom Prometheus exporters for application metrics collection and monitoring

Install required dependencies

Install Python development tools and Go compiler for building custom exporters.

sudo apt update
sudo apt install -y python3 python3-pip python3-venv golang-go curl

sudo dnf install -y python3 python3-pip golang curl
sudo dnf groupinstall -y "Development Tools"

Create dedicated user for exporters

Create a non-privileged user to run the exporters for security isolation.

sudo useradd --system --no-create-home --shell /bin/false prometheus-exporters
sudo mkdir -p /opt/prometheus-exporters
sudo chown prometheus-exporters:prometheus-exporters /opt/prometheus-exporters

Build Python-based HTTP exporter

Create a Python exporter that collects application metrics and exposes them via HTTP.

sudo mkdir -p /opt/prometheus-exporters/python-app
cd /opt/prometheus-exporters/python-app

python3 -m venv venv
source venv/bin/activate
pip install prometheus_client psutil requests

Create Python exporter application

Build a comprehensive exporter that monitors application processes, file metrics, and custom business logic.

#!/usr/bin/env python3
import time
import psutil
import os
import json
from prometheus_client import start_http_server, Gauge, Counter, Histogram, Info
from prometheus_client.core import CollectorRegistry
from threading import Thread
import logging

Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

class ApplicationExporter:
    def __init__(self, config_file='/etc/prometheus-exporters/app-config.json'):
        self.config = self.load_config(config_file)
        self.registry = CollectorRegistry()
        
        # Define metrics
        self.app_info = Info('app_version', 'Application version info', registry=self.registry)
        self.process_count = Gauge('app_processes_total', 'Number of application processes', ['name'], registry=self.registry)
        self.file_size_bytes = Gauge('app_file_size_bytes', 'Size of monitored files', ['path'], registry=self.registry)
        self.file_age_seconds = Gauge('app_file_age_seconds', 'Age of monitored files', ['path'], registry=self.registry)
        self.request_count = Counter('app_requests_total', 'Total application requests', ['method', 'endpoint'], registry=self.registry)
        self.response_time = Histogram('app_response_duration_seconds', 'Response time histogram', ['endpoint'], registry=self.registry)
        self.memory_usage_bytes = Gauge('app_memory_usage_bytes', 'Memory usage by process', ['process'], registry=self.registry)
        self.cpu_usage_percent = Gauge('app_cpu_usage_percent', 'CPU usage by process', ['process'], registry=self.registry)
        
        # Set application info
        self.app_info.info({
            'version': self.config.get('version', '1.0.0'),
            'environment': self.config.get('environment', 'production')
        })
    
    def load_config(self, config_file):
        try:
            with open(config_file, 'r') as f:
                return json.load(f)
        except FileNotFoundError:
            logger.warning(f"Config file {config_file} not found, using defaults")
            return {
                'monitored_processes': ['nginx', 'postgres', 'redis-server'],
                'monitored_files': ['/var/log/app.log', '/var/lib/app/data.db'],
                'version': '1.0.0',
                'environment': 'production'
            }
    
    def collect_process_metrics(self):
        """Collect metrics for monitored processes"""
        for process_name in self.config.get('monitored_processes', []):
            count = 0
            total_memory = 0
            total_cpu = 0
            
            for proc in psutil.process_iter(['pid', 'name', 'memory_info', 'cpu_percent']):
                try:
                    if proc.info['name'] == process_name:
                        count += 1
                        total_memory += proc.info['memory_info'].rss
                        total_cpu += proc.info['cpu_percent']
                except (psutil.NoSuchProcess, psutil.AccessDenied):
                    continue
            
            self.process_count.labels(name=process_name).set(count)
            if count > 0:
                self.memory_usage_bytes.labels(process=process_name).set(total_memory)
                self.cpu_usage_percent.labels(process=process_name).set(total_cpu)
    
    def collect_file_metrics(self):
        """Collect metrics for monitored files"""
        for file_path in self.config.get('monitored_files', []):
            try:
                stat = os.stat(file_path)
                self.file_size_bytes.labels(path=file_path).set(stat.st_size)
                self.file_age_seconds.labels(path=file_path).set(time.time() - stat.st_mtime)
            except FileNotFoundError:
                logger.warning(f"File {file_path} not found")
                self.file_size_bytes.labels(path=file_path).set(0)
                self.file_age_seconds.labels(path=file_path).set(-1)
    
    def simulate_request_metrics(self):
        """Simulate request metrics (replace with actual application integration)"""
        endpoints = ['/api/users', '/api/orders', '/api/health']
        methods = ['GET', 'POST', 'PUT']
        
        import random
        endpoint = random.choice(endpoints)
        method = random.choice(methods)
        response_time = random.uniform(0.01, 2.0)
        
        self.request_count.labels(method=method, endpoint=endpoint).inc()
        self.response_time.labels(endpoint=endpoint).observe(response_time)
    
    def collect_metrics(self):
        """Main metric collection method"""
        while True:
            try:
                self.collect_process_metrics()
                self.collect_file_metrics()
                self.simulate_request_metrics()
                time.sleep(10)  # Collect metrics every 10 seconds
            except Exception as e:
                logger.error(f"Error collecting metrics: {e}")
                time.sleep(10)
    
    def start(self, port=8000):
        """Start the HTTP server and metric collection"""
        logger.info(f"Starting application exporter on port {port}")
        start_http_server(port, registry=self.registry)
        
        # Start metric collection in background thread
        collection_thread = Thread(target=self.collect_metrics)
        collection_thread.daemon = True
        collection_thread.start()
        
        logger.info("Application exporter started successfully")
        return collection_thread

if __name__ == '__main__':
    exporter = ApplicationExporter()
    collection_thread = exporter.start(port=8000)
    
    try:
        # Keep the main thread alive
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        logger.info("Shutting down application exporter")

Create exporter configuration

Configure which processes and files to monitor for metrics collection.

sudo mkdir -p /etc/prometheus-exporters

{
  "version": "2.1.0",
  "environment": "production",
  "monitored_processes": [
    "nginx",
    "postgres",
    "redis-server",
    "gunicorn",
    "celery"
  ],
  "monitored_files": [
    "/var/log/nginx/access.log",
    "/var/log/nginx/error.log",
    "/var/lib/postgresql/data/postgresql.log",
    "/var/log/app/application.log",
    "/etc/nginx/nginx.conf"
  ]
}

Build Go-based exporter

Create a high-performance Go exporter for collecting system and application metrics.

mkdir -p /opt/prometheus-exporters/go-system
cd /opt/prometheus-exporters/go-system

go mod init system-exporter
go get github.com/prometheus/client_golang/prometheus
go get github.com/prometheus/client_golang/prometheus/promhttp

Create Go exporter application

Implement a Go exporter that provides efficient system metric collection.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type Config struct {
	Port             int      json:"port"
	MetricsPath      string   json:"metrics_path"
	ScrapeInterval   int      json:"scrape_interval"
	MonitoredPorts   []int    json:"monitored_ports"
	LogFiles         []string json:"log_files"
}

type SystemExporter struct {
	config                *Config
	systemLoad1           prometheus.Gauge
	systemLoad5           prometheus.Gauge
	systemLoad15          prometheus.Gauge
	memoryTotal           prometheus.Gauge
	memoryAvailable       prometheus.Gauge
	memoryUsed            prometheus.Gauge
	diskUsedBytes         *prometheus.GaugeVec
	diskTotalBytes        *prometheus.GaugeVec
	networkReceiveBytes   *prometheus.CounterVec
	networkTransmitBytes  *prometheus.CounterVec
	portStatus            *prometheus.GaugeVec
	logFileSizeBytes      *prometheus.GaugeVec
	logFileModTime        *prometheus.GaugeVec
	uptime                prometheus.Gauge
}

func NewSystemExporter(configPath string) (*SystemExporter, error) {
	config, err := loadConfig(configPath)
	if err != nil {
		return nil, fmt.Errorf("failed to load config: %w", err)
	}

	exporter := &SystemExporter{
		config: config,
		systemLoad1: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "system_load_1m",
			Help: "System load average over 1 minute",
		}),
		systemLoad5: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "system_load_5m",
			Help: "System load average over 5 minutes",
		}),
		systemLoad15: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "system_load_15m",
			Help: "System load average over 15 minutes",
		}),
		memoryTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "system_memory_total_bytes",
			Help: "Total system memory in bytes",
		}),
		memoryAvailable: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "system_memory_available_bytes",
			Help: "Available system memory in bytes",
		}),
		memoryUsed: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "system_memory_used_bytes",
			Help: "Used system memory in bytes",
		}),
		diskUsedBytes: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Name: "system_disk_used_bytes",
				Help: "Used disk space in bytes",
			},
			[]string{"device", "mountpoint"},
		),
		diskTotalBytes: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Name: "system_disk_total_bytes",
				Help: "Total disk space in bytes",
			},
			[]string{"device", "mountpoint"},
		),
		networkReceiveBytes: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Name: "system_network_receive_bytes_total",
				Help: "Total bytes received on network interfaces",
			},
			[]string{"interface"},
		),
		networkTransmitBytes: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Name: "system_network_transmit_bytes_total",
				Help: "Total bytes transmitted on network interfaces",
			},
			[]string{"interface"},
		),
		portStatus: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Name: "system_port_status",
				Help: "Status of monitored ports (1=listening, 0=not listening)",
			},
			[]string{"port"},
		),
		logFileSizeBytes: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Name: "system_log_file_size_bytes",
				Help: "Size of log files in bytes",
			},
			[]string{"file"},
		),
		logFileModTime: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Name: "system_log_file_modification_time",
				Help: "Last modification time of log files",
			},
			[]string{"file"},
		),
		uptime: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: "system_uptime_seconds",
			Help: "System uptime in seconds",
		}),
	}

	// Register metrics
	prometheus.MustRegister(
		exporter.systemLoad1,
		exporter.systemLoad5,
		exporter.systemLoad15,
		exporter.memoryTotal,
		exporter.memoryAvailable,
		exporter.memoryUsed,
		exporter.diskUsedBytes,
		exporter.diskTotalBytes,
		exporter.networkReceiveBytes,
		exporter.networkTransmitBytes,
		exporter.portStatus,
		exporter.logFileSizeBytes,
		exporter.logFileModTime,
		exporter.uptime,
	)

	return exporter, nil
}

func loadConfig(configPath string) (*Config, error) {
	config := &Config{
		Port:           8001,
		MetricsPath:    "/metrics",
		ScrapeInterval: 15,
		MonitoredPorts: []int{22, 80, 443, 5432, 6379},
		LogFiles:       []string{"/var/log/syslog", "/var/log/auth.log"},
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) {
		log.Printf("Config file %s not found, using defaults", configPath)
		return config, nil
	}

	data, err := ioutil.ReadFile(configPath)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(data, config)
	if err != nil {
		return nil, err
	}

	return config, nil
}

func (e *SystemExporter) collectLoadAverage() {
	data, err := ioutil.ReadFile("/proc/loadavg")
	if err != nil {
		log.Printf("Error reading load average: %v", err)
		return
	}

	fields := strings.Fields(string(data))
	if len(fields) >= 3 {
		if load1, err := strconv.ParseFloat(fields[0], 64); err == nil {
			e.systemLoad1.Set(load1)
		}
		if load5, err := strconv.ParseFloat(fields[1], 64); err == nil {
			e.systemLoad5.Set(load5)
		}
		if load15, err := strconv.ParseFloat(fields[2], 64); err == nil {
			e.systemLoad15.Set(load15)
		}
	}
}

func (e *SystemExporter) collectMemoryStats() {
	data, err := ioutil.ReadFile("/proc/meminfo")
	if err != nil {
		log.Printf("Error reading memory info: %v", err)
		return
	}

	lines := strings.Split(string(data), "\n")
	for _, line := range lines {
		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}

		value, err := strconv.ParseFloat(fields[1], 64)
		if err != nil {
			continue
		}
		// Convert KB to bytes
		value *= 1024

		switch fields[0] {
		case "MemTotal:":
			e.memoryTotal.Set(value)
		case "MemAvailable:":
			e.memoryAvailable.Set(value)
		}
	}

	// Calculate used memory
	total := e.memoryTotal
	available := e.memoryAvailable
	if total != nil && available != nil {
		// This is a simplified calculation
		// In production, you'd want more accurate memory calculations
	}
}

func (e *SystemExporter) collectPortStatus() {
	for _, port := range e.config.MonitoredPorts {
		address := fmt.Sprintf("127.0.0.1:%d", port)
		conn, err := net.DialTimeout("tcp", address, 1*time.Second)
		if err != nil {
			e.portStatus.WithLabelValues(strconv.Itoa(port)).Set(0)
		} else {
			conn.Close()
			e.portStatus.WithLabelValues(strconv.Itoa(port)).Set(1)
		}
	}
}

func (e *SystemExporter) collectLogFileStats() {
	for _, logFile := range e.config.LogFiles {
		stat, err := os.Stat(logFile)
		if err != nil {
			log.Printf("Error getting stats for %s: %v", logFile, err)
			e.logFileSizeBytes.WithLabelValues(logFile).Set(0)
			e.logFileModTime.WithLabelValues(logFile).Set(0)
			continue
		}

		e.logFileSizeBytes.WithLabelValues(logFile).Set(float64(stat.Size()))
		e.logFileModTime.WithLabelValues(logFile).Set(float64(stat.ModTime().Unix()))
	}
}

func (e *SystemExporter) collectUptime() {
	data, err := ioutil.ReadFile("/proc/uptime")
	if err != nil {
		log.Printf("Error reading uptime: %v", err)
		return
	}

	fields := strings.Fields(string(data))
	if len(fields) >= 1 {
		if uptime, err := strconv.ParseFloat(fields[0], 64); err == nil {
			e.uptime.Set(uptime)
		}
	}
}

func (e *SystemExporter) collectMetrics() {
	for {
		e.collectLoadAverage()
		e.collectMemoryStats()
		e.collectPortStatus()
		e.collectLogFileStats()
		e.collectUptime()
		time.Sleep(time.Duration(e.config.ScrapeInterval) * time.Second)
	}
}

func (e *SystemExporter) Start() {
	go e.collectMetrics()

	http.Handle(e.config.MetricsPath, promhttp.Handler())
	log.Printf("System exporter starting on port %d", e.config.Port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", e.config.Port), nil))
}

func main() {
	exporter, err := NewSystemExporter("/etc/prometheus-exporters/system-config.json")
	if err != nil {
		log.Fatalf("Failed to create system exporter: %v", err)
	}

	exporter.Start()
}

Create Go exporter configuration

Configure the Go exporter with specific ports and files to monitor.

{
  "port": 8001,
  "metrics_path": "/metrics",
  "scrape_interval": 15,
  "monitored_ports": [
    22,
    80,
    443,
    5432,
    6379,
    9090,
    3000
  ],
  "log_files": [
    "/var/log/syslog",
    "/var/log/auth.log",
    "/var/log/nginx/access.log",
    "/var/log/nginx/error.log"
  ]
}

Build the Go exporter

Compile the Go exporter into an optimized binary for production deployment.

cd /opt/prometheus-exporters/go-system
go build -o system-exporter -ldflags "-w -s" main.go

Set proper permissions

Configure ownership and permissions for the exporter files and directories.

sudo chown -R prometheus-exporters:prometheus-exporters /opt/prometheus-exporters
sudo chown -R prometheus-exporters:prometheus-exporters /etc/prometheus-exporters
sudo chmod +x /opt/prometheus-exporters/python-app/app_exporter.py
sudo chmod +x /opt/prometheus-exporters/go-system/system-exporter

Create systemd service for Python exporter

Deploy the Python exporter as a systemd service for automatic startup and process management.

[Unit]
Description=Prometheus Application Exporter
After=network.target
Wants=network.target

[Service]
Type=simple
User=prometheus-exporters
Group=prometheus-exporters
WorkingDirectory=/opt/prometheus-exporters/python-app
Environment=PYTHONPATH=/opt/prometheus-exporters/python-app
ExecStart=/opt/prometheus-exporters/python-app/venv/bin/python /opt/prometheus-exporters/python-app/app_exporter.py
Restart=always
RestartSec=10
KillMode=mixed
TimeoutStopSec=30
SyslogIdentifier=prometheus-app-exporter

Security settings
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=yes
ReadWritePaths=/var/log
CapabilityBoundingSet=
AmbientCapabilities=
SystemCallFilter=@system-service
SystemCallFilter=~@privileged

[Install]
WantedBy=multi-user.target

Create systemd service for Go exporter

Deploy the Go exporter as a systemd service with security hardening.

[Unit]
Description=Prometheus System Exporter
After=network.target
Wants=network.target

[Service]
Type=simple
User=prometheus-exporters
Group=prometheus-exporters
WorkingDirectory=/opt/prometheus-exporters/go-system
ExecStart=/opt/prometheus-exporters/go-system/system-exporter
Restart=always
RestartSec=10
KillMode=mixed
TimeoutStopSec=30
SyslogIdentifier=prometheus-system-exporter

Security settings
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=yes
ReadOnlyPaths=/proc
ReadOnlyPaths=/sys
ReadOnlyPaths=/var/log
CapabilityBoundingSet=
AmbientCapabilities=
SystemCallFilter=@system-service
SystemCallFilter=~@privileged

[Install]
WantedBy=multi-user.target

Enable and start the exporters

Start both exporters and enable them for automatic startup on boot.

sudo systemctl daemon-reload
sudo systemctl enable --now prometheus-app-exporter
sudo systemctl enable --now prometheus-system-exporter

Configure Prometheus integration

Add the custom exporters to your Prometheus configuration for metric scraping.

# Add these scrape configs to your existing prometheus.yml
scrape_configs:
  - job_name: 'custom-app-exporter'
    static_configs:
      - targets: ['localhost:8000']
    scrape_interval: 15s
    scrape_timeout: 10s
    metrics_path: /metrics
    scheme: http
    
  - job_name: 'custom-system-exporter'
    static_configs:
      - targets: ['localhost:8001']
    scrape_interval: 15s
    scrape_timeout: 10s
    metrics_path: /metrics
    scheme: http

Restart Prometheus to apply configuration

Reload Prometheus configuration to start collecting metrics from your custom exporters.

sudo systemctl restart prometheus
sudo systemctl status prometheus