Set up centralized logging with Winston and Elasticsearch for Node.js applications

Update system packages

Start by updating your package manager to ensure you have the latest packages available.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install Elasticsearch dependencies

Install Java 11 or later and essential tools required for Elasticsearch.

sudo apt install -y openjdk-11-jre-headless curl wget gnupg2

sudo dnf install -y java-11-openjdk-headless curl wget gnupg2

Add Elasticsearch repository and install

Add the official Elasticsearch repository and install Elasticsearch 8 with security features.

wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-8.x.list
sudo apt update
sudo apt install -y elasticsearch

sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
sudo tee /etc/yum.repos.d/elasticsearch.repo << EOF
[elasticsearch]
name=Elasticsearch repository for 8.x packages
baseurl=https://artifacts.elastic.co/packages/8.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=0
autorefresh=1
type=rpm-md
EOF
sudo dnf install -y --enablerepo=elasticsearch elasticsearch

Configure Elasticsearch for centralized logging

Configure Elasticsearch with appropriate settings for log aggregation and performance optimization.

cluster.name: logging-cluster
node.name: node-1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 127.0.0.1
http.port: 9200
discovery.type: single-node

Security settings
xpack.security.enabled: true
xpack.security.enrollment.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: certs/http.p12
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: certs/transport.p12
xpack.security.transport.ssl.truststore.path: certs/transport.p12

Configure JVM heap size

Set appropriate JVM heap size based on available memory. Use half of available RAM, maximum 32GB.

-Xms2g
-Xmx2g

Start and enable Elasticsearch

Enable Elasticsearch to start automatically and start the service.

sudo systemctl daemon-reload
sudo systemctl enable --now elasticsearch
sudo systemctl status elasticsearch

Reset Elasticsearch password and get security token

Reset the elastic user password and note it for later configuration.

sudo /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic
sudo /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana

Install and configure Kibana

Install Kibana for log visualization and analysis dashboards.

sudo apt install -y kibana

sudo dnf install -y --enablerepo=elasticsearch kibana

Configure Kibana

Configure Kibana to connect to Elasticsearch and bind to the correct interface.

server.port: 5601
server.host: "0.0.0.0"
server.name: "logging-kibana"
elasticsearch.hosts: ["https://127.0.0.1:9200"]
elasticsearch.username: "kibana_system"
elasticsearch.password: "your-kibana-password"
elasticsearch.ssl.verificationMode: none

Set Kibana system user password

Set the password for the kibana_system user and update the configuration.

sudo /usr/share/elasticsearch/bin/elasticsearch-reset-password -u kibana_system

Update the password in /etc/kibana/kibana.yml with the generated password.

Start Kibana

Enable and start the Kibana service.

sudo systemctl enable --now kibana
sudo systemctl status kibana

Install Node.js and npm

Install Node.js and npm for building the logging application.

curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -
sudo apt install -y nodejs

curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo bash -
sudo dnf install -y nodejs npm

Create Node.js logging application

Create a sample Node.js application directory and initialize the project.

mkdir -p /opt/nodejs-logging
cd /opt/nodejs-logging
npm init -y

Install Winston and Elasticsearch transport

Install Winston logging library with Elasticsearch transport for centralized logging.

cd /opt/nodejs-logging
npm install winston winston-elasticsearch @elastic/elasticsearch express

Configure Winston with Elasticsearch transport

Create a logging configuration that sends structured logs to Elasticsearch.

const winston = require('winston');
const { ElasticsearchTransport } = require('winston-elasticsearch');

const esTransportOpts = {
  level: 'info',
  clientOpts: {
    node: 'https://127.0.0.1:9200',
    auth: {
      username: 'elastic',
      password: 'your-elastic-password'
    },
    ssl: {
      rejectUnauthorized: false
    }
  },
  index: 'app-logs',
  indexTemplate: {
    name: 'app-logs-template',
    pattern: 'app-logs-*',
    settings: {
      number_of_shards: 1,
      number_of_replicas: 0,
      'index.refresh_interval': '5s'
    },
    mappings: {
      properties: {
        '@timestamp': { type: 'date' },
        level: { type: 'keyword' },
        message: { type: 'text' },
        meta: { type: 'object' },
        hostname: { type: 'keyword' },
        service: { type: 'keyword' },
        environment: { type: 'keyword' }
      }
    }
  },
  transformer: (logData) => {
    return {
      '@timestamp': new Date().toISOString(),
      level: logData.level,
      message: logData.message,
      meta: logData.meta || {},
      hostname: require('os').hostname(),
      service: 'nodejs-app',
      environment: process.env.NODE_ENV || 'development'
    };
  }
};

const logger = winston.createLogger({
  level: 'info',
  format: winston.format.combine(
    winston.format.timestamp(),
    winston.format.errors({ stack: true }),
    winston.format.json()
  ),
  transports: [
    new winston.transports.Console({
      format: winston.format.combine(
        winston.format.colorize(),
        winston.format.simple()
      )
    }),
    new ElasticsearchTransport(esTransportOpts)
  ],
  exitOnError: false
});

module.exports = logger;

Create sample Express application with logging

Create a sample Express.js application that demonstrates structured logging patterns.

const express = require('express');
const logger = require('./logger');

const app = express();
const port = process.env.PORT || 3000;

// Request logging middleware
app.use((req, res, next) => {
  const startTime = Date.now();
  
  res.on('finish', () => {
    const duration = Date.now() - startTime;
    
    logger.info('HTTP Request', {
      method: req.method,
      url: req.url,
      statusCode: res.statusCode,
      duration: duration,
      userAgent: req.get('User-Agent'),
      ip: req.ip,
      requestId: req.headers['x-request-id'] || 'unknown'
    });
  });
  
  next();
});

// Sample routes with different log levels
app.get('/', (req, res) => {
  logger.info('Home page accessed', {
    userId: req.query.userId,
    sessionId: req.headers['x-session-id']
  });
  
  res.json({ message: 'Hello World', timestamp: new Date().toISOString() });
});

app.get('/api/users/:id', (req, res) => {
  const userId = req.params.id;
  
  logger.info('User data requested', {
    userId: userId,
    requestSource: req.get('Referer')
  });
  
  if (!userId || isNaN(userId)) {
    logger.warn('Invalid user ID requested', {
      userId: userId,
      ip: req.ip
    });
    
    return res.status(400).json({ error: 'Invalid user ID' });
  }
  
  res.json({ 
    id: userId, 
    name: User ${userId},
    lastAccess: new Date().toISOString()
  });
});

app.get('/api/error', (req, res) => {
  logger.error('Intentional error triggered', {
    endpoint: '/api/error',
    userId: req.query.userId,
    stack: new Error('Test error').stack
  });
  
  res.status(500).json({ error: 'Internal server error' });
});

// Global error handler
app.use((err, req, res, next) => {
  logger.error('Unhandled application error', {
    error: err.message,
    stack: err.stack,
    url: req.url,
    method: req.method,
    ip: req.ip
  });
  
  res.status(500).json({ error: 'Internal server error' });
});

app.listen(port, () => {
  logger.info('Application started', {
    port: port,
    environment: process.env.NODE_ENV || 'development',
    nodeVersion: process.version
  });
});

module.exports = app;

Create systemd service for the Node.js application

Create a systemd service to manage the logging application automatically.

[Unit]
Description=Node.js Logging Application
After=network.target elasticsearch.service

[Service]
Type=simple
User=nodejs
Group=nodejs
WorkingDirectory=/opt/nodejs-logging
Environment=NODE_ENV=production
Environment=PORT=3000
ExecStart=/usr/bin/node app.js
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
SyslogIdentifier=nodejs-logging

[Install]
WantedBy=multi-user.target

Create nodejs user and set permissions

Create a dedicated user for running the Node.js application securely.

sudo useradd -r -s /bin/false nodejs
sudo chown -R nodejs:nodejs /opt/nodejs-logging
sudo chmod 755 /opt/nodejs-logging
sudo chmod 644 /opt/nodejs-logging/*

Configure log rotation for application logs

Set up log rotation to prevent disk space issues with application logs.

/var/log/nodejs-logging/*.log {
    daily
    missingok
    rotate 14
    compress
    delaycompress
    notifempty
    copytruncate
    create 644 nodejs nodejs
}

Start the Node.js logging application

Enable and start the Node.js application service.

sudo systemctl daemon-reload
sudo systemctl enable --now nodejs-logging
sudo systemctl status nodejs-logging

Create index pattern in Kibana

Access Kibana web interface and create an index pattern for the application logs.

Navigate to http://your-server-ip:5601 and login with the elastic user credentials.

1. Go to Stack Management > Index Patterns
2. Click "Create index pattern"
3. Enter app-logs-* as the index pattern
4. Select @timestamp as the time field
5. Click "Create index pattern"

Create log monitoring dashboard

Build a comprehensive dashboard for monitoring application logs and performance.

In Kibana, go to Analytics > Dashboard and create visualizations for:

• Log level distribution (pie chart)
• Request response times (line chart)
• Error rate trends (area chart)
• Top error messages (data table)
• Request volume by endpoint (bar chart)

Implement structured logging for microservices

Add correlation IDs and service metadata for better distributed system observability.

const { v4: uuidv4 } = require('uuid');
const logger = require('../logger');

function correlationMiddleware(req, res, next) {
  // Generate or extract correlation ID
  const correlationId = req.headers['x-correlation-id'] || uuidv4();
  
  // Add correlation ID to request object
  req.correlationId = correlationId;
  
  // Set response header
  res.setHeader('x-correlation-id', correlationId);
  
  // Create child logger with correlation context
  req.logger = logger.child({
    correlationId: correlationId,
    service: 'user-service',
    version: process.env.APP_VERSION || '1.0.0'
  });
  
  next();
}

module.exports = correlationMiddleware;

Configure log sampling for high-traffic applications

Implement intelligent log sampling to reduce storage costs while maintaining observability.

const winston = require('winston');
const { ElasticsearchTransport } = require('winston-elasticsearch');

// Sampling configuration
const SAMPLE_RATES = {
  error: 1.0,    // Log all errors
  warn: 0.5,     // Log 50% of warnings
  info: 0.1,     // Log 10% of info messages
  debug: 0.01    // Log 1% of debug messages
};

// Custom format with sampling
const samplingFormat = winston.format((info) => {
  const sampleRate = SAMPLE_RATES[info.level] || 0.1;
  
  if (Math.random() > sampleRate) {
    return false; // Don't log this message
  }
  
  // Add sampling metadata
  info.sampled = true;
  info.sampleRate = sampleRate;
  
  return info;
});

const logger = winston.createLogger({
  level: 'debug',
  format: winston.format.combine(
    samplingFormat(),
    winston.format.timestamp(),
    winston.format.errors({ stack: true }),
    winston.format.json()
  ),
  transports: [
    new winston.transports.Console(),
    new ElasticsearchTransport({
      level: 'info',
      clientOpts: {
        node: 'https://127.0.0.1:9200',
        auth: {
          username: 'elastic',
          password: 'your-elastic-password'
        },
        ssl: { rejectUnauthorized: false }
      },
      index: 'app-logs'
    })
  ]
});

module.exports = logger;