Configure PM2 log rotation and advanced monitoring with Prometheus integration

Install Node.js and PM2

Start by installing the latest LTS version of Node.js and PM2 globally on your system.

curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash -
sudo apt update
sudo apt install -y nodejs
sudo npm install -g pm2
pm2 install pm2-logrotate
pm2 install pm2-auto-pull

curl -fsSL https://rpm.nodesource.com/setup_lts.x | sudo bash -
sudo dnf install -y nodejs npm
sudo npm install -g pm2
pm2 install pm2-logrotate
pm2 install pm2-auto-pull

Create a sample Node.js application

Create a basic Express application to demonstrate PM2 monitoring and logging features.

mkdir -p /opt/nodeapp
cd /opt/nodeapp

{
  "name": "sample-node-app",
  "version": "1.0.0",
  "description": "Sample app for PM2 monitoring",
  "main": "app.js",
  "dependencies": {
    "express": "^4.18.0",
    "prom-client": "^15.0.0",
    "winston": "^3.8.0"
  }
}

npm install

Create application with logging and metrics

Build a sample application that includes structured logging and Prometheus metrics endpoints.

const express = require('express');
const winston = require('winston');
const client = require('prom-client');
const app = express();
const port = process.env.PORT || 3000;

// Configure Winston logger
const logger = winston.createLogger({
  level: 'info',
  format: winston.format.combine(
    winston.format.timestamp(),
    winston.format.json()
  ),
  transports: [
    new winston.transports.File({ filename: '/var/log/nodeapp/error.log', level: 'error' }),
    new winston.transports.File({ filename: '/var/log/nodeapp/combined.log' }),
    new winston.transports.Console()
  ]
});

// Prometheus metrics
const register = new client.Registry();
const httpRequestDuration = new client.Histogram({
  name: 'http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  labelNames: ['method', 'route', 'status_code']
});
const httpRequestTotal = new client.Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests',
  labelNames: ['method', 'route', 'status_code']
});

register.registerMetric(httpRequestDuration);
register.registerMetric(httpRequestTotal);
client.collectDefaultMetrics({ register });

// Middleware for metrics
app.use((req, res, next) => {
  const start = Date.now();
  res.on('finish', () => {
    const duration = (Date.now() - start) / 1000;
    httpRequestDuration.observe(
      { method: req.method, route: req.route?.path || req.path, status_code: res.statusCode },
      duration
    );
    httpRequestTotal.inc({
      method: req.method,
      route: req.route?.path || req.path,
      status_code: res.statusCode
    });
    logger.info(`${req.method} ${req.path} ${res.statusCode} ${duration}s`);
  });
  next();
});

// Routes
app.get('/', (req, res) => {
  res.json({ message: 'Hello from PM2 monitored app!', timestamp: new Date().toISOString() });
});

app.get('/health', (req, res) => {
  res.json({ status: 'healthy', uptime: process.uptime(), memory: process.memoryUsage() });
});

app.get('/metrics', (req, res) => {
  res.set('Content-Type', register.contentType);
  res.end(register.metrics());
});

app.get('/error', (req, res) => {
  logger.error('Intentional error for testing');
  res.status(500).json({ error: 'This is a test error' });
});

app.listen(port, () => {
  logger.info(`Server running on port ${port}`);
});

Create log directory and set permissions

Create the log directory structure with appropriate permissions for the Node.js application.

sudo mkdir -p /var/log/nodeapp
sudo chown $USER:$USER /var/log/nodeapp
sudo chmod 755 /var/log/nodeapp

Configure PM2 ecosystem file

Create a comprehensive PM2 ecosystem configuration that enables cluster mode, monitoring, and proper logging.

module.exports = {
  apps: [{
    name: 'nodeapp-cluster',
    script: './app.js',
    instances: 'max',
    exec_mode: 'cluster',
    env: {
      NODE_ENV: 'production',
      PORT: 3000
    },
    log_file: '/var/log/nodeapp/app.log',
    out_file: '/var/log/nodeapp/out.log',
    error_file: '/var/log/nodeapp/error.log',
    log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
    merge_logs: true,
    max_memory_restart: '300M',
    restart_delay: 4000,
    max_restarts: 10,
    min_uptime: '10s',
    kill_timeout: 5000,
    listen_timeout: 8000,
    shutdown_with_message: true,
    pmx: true,
    monitoring: true,
    ignore_watch: ['node_modules', 'logs'],
    watch_options: {
      followSymlinks: false
    }
  }]
};

Configure PM2 log rotation

Set up automated log rotation to prevent disk space issues and maintain manageable log file sizes.

pm2 set pm2-logrotate:max_size 50M
pm2 set pm2-logrotate:retain 7
pm2 set pm2-logrotate:compress true
pm2 set pm2-logrotate:dateFormat 'YYYY-MM-DD_HH-mm-ss'
pm2 set pm2-logrotate:rotateModule true
pm2 set pm2-logrotate:workerInterval 3600
pm2 set pm2-logrotate:rotateInterval '0 0 * * *'

Configure system logrotate integration

Create a system logrotate configuration for additional log file management and integration with systemd journal.

/var/log/nodeapp/*.log {
    daily
    missingok
    rotate 14
    compress
    delaycompress
    notifempty
    sharedscripts
    copytruncate
    postrotate
        systemctl reload rsyslog > /dev/null 2>&1 || true
    endscript
}

sudo chown root:root /etc/logrotate.d/nodeapp
sudo chmod 644 /etc/logrotate.d/nodeapp

Install and configure Prometheus

Install Prometheus to collect metrics from your PM2 applications and system resources.

sudo useradd --no-create-home --shell /bin/false prometheus
wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prometheus-2.45.0.linux-amd64.tar.gz
tar xzf prometheus-2.45.0.linux-amd64.tar.gz
sudo cp prometheus-2.45.0.linux-amd64/prometheus /usr/local/bin/
sudo cp prometheus-2.45.0.linux-amd64/promtool /usr/local/bin/
sudo chown prometheus:prometheus /usr/local/bin/prometheus
sudo chown prometheus:prometheus /usr/local/bin/promtool
sudo mkdir -p /etc/prometheus /var/lib/prometheus
sudo chown prometheus:prometheus /etc/prometheus /var/lib/prometheus

sudo useradd --no-create-home --shell /bin/false prometheus
wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prometheus-2.45.0.linux-amd64.tar.gz
tar xzf prometheus-2.45.0.linux-amd64.tar.gz
sudo cp prometheus-2.45.0.linux-amd64/prometheus /usr/local/bin/
sudo cp prometheus-2.45.0.linux-amd64/promtool /usr/local/bin/
sudo chown prometheus:prometheus /usr/local/bin/prometheus
sudo chown prometheus:prometheus /usr/local/bin/promtool
sudo mkdir -p /etc/prometheus /var/lib/prometheus
sudo chown prometheus:prometheus /etc/prometheus /var/lib/prometheus

Configure Prometheus for PM2 monitoring

Set up Prometheus configuration to scrape metrics from PM2 applications and Node Exporter.

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "/etc/prometheus/rules/*.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - localhost:9093

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['localhost:9100']
    scrape_interval: 5s

  - job_name: 'nodeapp-pm2'
    static_configs:
      - targets: ['localhost:3000']
    metrics_path: '/metrics'
    scrape_interval: 10s

  - job_name: 'pm2-metrics'
    static_configs:
      - targets: ['localhost:9209']
    scrape_interval: 10s

sudo chown prometheus:prometheus /etc/prometheus/prometheus.yml

Install Node Exporter for system metrics

Install Prometheus Node Exporter to collect detailed system metrics alongside PM2 application metrics.

wget https://github.com/prometheus/node_exporter/releases/download/v1.6.1/node_exporter-1.6.1.linux-amd64.tar.gz
tar xzf node_exporter-1.6.1.linux-amd64.tar.gz
sudo cp node_exporter-1.6.1.linux-amd64/node_exporter /usr/local/bin/
sudo useradd --no-create-home --shell /bin/false node_exporter
sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter

Install PM2 Prometheus exporter

Install a dedicated PM2 metrics exporter to expose process-level metrics to Prometheus.

sudo npm install -g pm2-prometheus-exporter
pm2 install pm2-prometheus-exporter

Create systemd services

Create systemd service files for Prometheus, Node Exporter, and PM2 to ensure automatic startup and proper service management.

[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/prometheus \
    --config.file /etc/prometheus/prometheus.yml \
    --storage.tsdb.path /var/lib/prometheus/ \
    --web.console.templates=/etc/prometheus/consoles \
    --web.console.libraries=/etc/prometheus/console_libraries \
    --web.listen-address=0.0.0.0:9090 \
    --web.enable-lifecycle
Restart=always

[Install]
WantedBy=multi-user.target

[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter --web.listen-address=:9100
Restart=always

[Install]
WantedBy=multi-user.target

[Unit]
Description=PM2 process manager for Node.js applications
Documentation=https://pm2.keymetrics.io/
After=network.target

[Service]
Type=forking
User=$USER
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
Environment=PM2_HOME=/home/$USER/.pm2
PIDFile=/home/$USER/.pm2/pm2.pid
ExecStart=/usr/local/bin/pm2 resurrect
ExecReload=/usr/local/bin/pm2 reload all
ExecStop=/usr/local/bin/pm2 kill
PrivateTmp=false

[Install]
WantedBy=multi-user.target

Start PM2 application with monitoring

Deploy the application using PM2 ecosystem configuration and enable monitoring features.

cd /opt/nodeapp
pm2 start ecosystem.config.js
pm2 save
pm2 startup
sudo env PATH=$PATH:/usr/local/bin /usr/local/lib/node_modules/pm2/bin/pm2 startup systemd -u $USER --hp /home/$USER

Configure PM2 monitoring dashboard

Enable PM2's built-in monitoring dashboard and configure web interface access.

pm2 set pm2:web-interface true
pm2 set pm2:web-port 9615
pm2 set pm2:web-host 0.0.0.0
pm2 web

Create Prometheus alerting rules

Configure alerting rules for PM2 applications to monitor process health, memory usage, and error rates.

sudo mkdir -p /etc/prometheus/rules

groups:
  - name: pm2-alerts
    rules:
      - alert: PM2ProcessDown
        expr: pm2_process_status == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "PM2 process {{ $labels.name }} is down"
          description: "PM2 process {{ $labels.name }} has been down for more than 30 seconds"

      - alert: PM2HighMemoryUsage
        expr: (pm2_process_memory_usage / pm2_process_memory_limit) * 100 > 80
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "PM2 process {{ $labels.name }} high memory usage"
          description: "PM2 process {{ $labels.name }} is using {{ $value }}% of allocated memory"

      - alert: PM2HighRestarts
        expr: increase(pm2_process_restarts_total[5m]) > 3
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "PM2 process {{ $labels.name }} restarting frequently"
          description: "PM2 process {{ $labels.name }} has restarted {{ $value }} times in the last 5 minutes"

      - alert: NodeAppHighErrorRate
        expr: rate(http_requests_total{status_code=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.1
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "High error rate in Node.js application"
          description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes"

sudo chown -R prometheus:prometheus /etc/prometheus/rules

Configure health check monitoring

Set up automated health checks for PM2 applications with custom monitoring scripts.

const http = require('http');
const fs = require('fs');
const path = require('path');

function healthCheck() {
  const options = {
    hostname: 'localhost',
    port: 3000,
    path: '/health',
    method: 'GET',
    timeout: 5000
  };

  const req = http.request(options, (res) => {
    let data = '';
    res.on('data', chunk => data += chunk);
    res.on('end', () => {
      if (res.statusCode === 200) {
        const response = JSON.parse(data);
        console.log(`Health check passed: ${JSON.stringify(response)}`);
        process.exit(0);
      } else {
        console.error(`Health check failed with status: ${res.statusCode}`);
        process.exit(1);
      }
    });
  });

  req.on('error', (err) => {
    console.error(`Health check failed: ${err.message}`);
    process.exit(1);
  });

  req.on('timeout', () => {
    console.error('Health check timeout');
    req.destroy();
    process.exit(1);
  });

  req.end();
}

healthCheck();

chmod +x /opt/nodeapp/healthcheck.js

Enable systemd services and start monitoring

Enable all monitoring services and verify they start correctly.

sudo systemctl daemon-reload
sudo systemctl enable --now prometheus
sudo systemctl enable --now node-exporter
sudo systemctl enable --now pm2-nodeapp
sudo systemctl status prometheus node-exporter pm2-nodeapp

Configure PM2 journal logging

Integrate PM2 logs with systemd journal for centralized log management and better integration with system monitoring.

# PM2 log forwarding to systemd journal
$ModLoad imfile

# PM2 application logs
$InputFileName /var/log/nodeapp/combined.log
$InputFileTag pm2-nodeapp:
$InputFileStateFile stat-pm2-nodeapp
$InputFileSeverity info
$InputFileFacility local0
$InputRunFileMonitor

# PM2 error logs
$InputFileName /var/log/nodeapp/error.log
$InputFileTag pm2-nodeapp-error:
$InputFileStateFile stat-pm2-nodeapp-error
$InputFileSeverity error
$InputFileFacility local0
$InputRunFileMonitor

# Send to journal
local0.* :omjournal:

sudo systemctl restart rsyslog

Install Grafana for visualization

Install Grafana to create comprehensive dashboards for PM2 and application monitoring. For detailed Grafana setup, see our Grafana installation guide.

wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -
echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee /etc/apt/sources.list.d/grafana.list
sudo apt update
sudo apt install -y grafana
sudo systemctl enable --now grafana-server

sudo dnf install -y https://dl.grafana.com/oss/release/grafana-10.1.0-1.x86_64.rpm
sudo systemctl enable --now grafana-server

Configure automated alerting

Set up automated alerts for PM2 process failures and performance issues using a simple notification script.

#!/bin/bash

# PM2 Alert Handler Script
ALERT_TYPE=$1
PROCESS_NAME=$2
MESSAGE=$3
LOG_FILE="/var/log/nodeapp/alerts.log"

case $ALERT_TYPE in
  "restart")
    echo "$(date): ALERT - Process $PROCESS_NAME restarted: $MESSAGE" | tee -a $LOG_FILE
    # Add webhook or email notification here
    ;;
  "memory")
    echo "$(date): WARNING - Process $PROCESS_NAME high memory usage: $MESSAGE" | tee -a $LOG_FILE
    ;;
  "error")
    echo "$(date): ERROR - Process $PROCESS_NAME error: $MESSAGE" | tee -a $LOG_FILE
    ;;
esac

# Optional: Send to external monitoring service