Learn to develop custom Nagios plugins for specialized monitoring requirements including setting up the development environment, writing check scripts in multiple languages, and integrating them into your Nagios Core monitoring infrastructure.
Prerequisites
- Nagios Core 4.5 installed
- Python 3 and pip3
- Basic knowledge of shell scripting
- Understanding of monitoring concepts
What this solves
Default Nagios plugins cover basic system monitoring, but production environments often need custom checks for applications, APIs, databases, or business metrics. This guide shows you how to develop, test, and deploy custom Nagios plugins that extend your monitoring capabilities beyond standard system resources.
Step-by-step plugin development setup
Install development dependencies
Set up the essential tools for plugin development including compilers, interpreters, and Nagios plugin utilities.
sudo apt update
sudo apt install -y build-essential python3 python3-pip perl libmonitoring-plugin-perl curl wget git
sudo dnf install -y gcc gcc-c++ make python3 python3-pip perl-Monitoring-Plugin curl wget git
Create plugin development directory
Organize your custom plugins in a dedicated directory structure with proper permissions.
sudo mkdir -p /usr/local/nagios/plugins/custom
sudo mkdir -p /usr/local/nagios/plugins/development
sudo chown -R nagios:nagios /usr/local/nagios/plugins/
sudo chmod 755 /usr/local/nagios/plugins/custom
sudo chmod 775 /usr/local/nagios/plugins/development
Install Python monitoring libraries
Install essential Python libraries for building robust monitoring plugins with proper exit codes and performance data.
sudo pip3 install nagiosplugin requests psutil pymongo redis elasticsearch
Download Nagios plugin development utils
Get the official plugin development utilities that provide standard functions and exit codes.
cd /tmp
wget https://www.nagios-plugins.org/download/nagios-plugins-2.4.6.tar.gz
tar -xzf nagios-plugins-2.4.6.tar.gz
cd nagios-plugins-2.4.6
./configure --with-nagios-user=nagios --with-nagios-group=nagios
make
sudo cp plugins-root/utils.sh /usr/local/nagios/plugins/
sudo cp plugins/utils.c /usr/local/nagios/plugins/
sudo chmod 644 /usr/local/nagios/plugins/utils.*
Writing custom check plugins
Create a basic shell script plugin
Start with a simple disk usage plugin that demonstrates proper Nagios plugin structure and exit codes.
#!/bin/bash
# Source the utils for standard functions
. /usr/local/nagios/plugins/utils.sh
# Plugin info
PLUGIN_NAME="Custom Disk Check"
PLUGIN_VERSION="1.0"
# Default values
WARNING_THRESHOLD=80
CRITICAL_THRESHOLD=90
PATH_TO_CHECK="/"
# Function to display help
print_help() {
echo "$PLUGIN_NAME $PLUGIN_VERSION"
echo "Usage: $0 -w
Create a Python API monitoring plugin
Build a more advanced plugin that monitors API endpoints with JSON response validation and performance metrics.
#!/usr/bin/env python3
import sys
import argparse
import requests
import time
import json
from urllib.parse import urlparse
# Nagios exit codes
OK = 0
WARNING = 1
CRITICAL = 2
UNKNOWN = 3
def parse_args():
parser = argparse.ArgumentParser(description='Monitor API endpoint health')
parser.add_argument('-u', '--url', required=True, help='API endpoint URL')
parser.add_argument('-t', '--timeout', type=int, default=10, help='Request timeout in seconds')
parser.add_argument('-w', '--warning', type=float, default=2.0, help='Warning threshold for response time')
parser.add_argument('-c', '--critical', type=float, default=5.0, help='Critical threshold for response time')
parser.add_argument('-k', '--key', help='JSON key to validate in response')
parser.add_argument('-v', '--value', help='Expected value for the JSON key')
parser.add_argument('-H', '--header', action='append', help='Custom headers (format: "Key: Value")')
parser.add_argument('-s', '--status', type=int, default=200, help='Expected HTTP status code')
return parser.parse_args()
def make_request(url, timeout, headers=None):
"""Make HTTP request and return response with timing"""
custom_headers = {}
if headers:
for header in headers:
key, value = header.split(':', 1)
custom_headers[key.strip()] = value.strip()
start_time = time.time()
try:
response = requests.get(url, timeout=timeout, headers=custom_headers)
response_time = time.time() - start_time
return response, response_time
except requests.exceptions.Timeout:
return None, timeout
except requests.exceptions.RequestException as e:
print(f"CRITICAL - Request failed: {str(e)}")
sys.exit(CRITICAL)
def validate_json_content(response, key, expected_value):
"""Validate JSON response content"""
try:
data = response.json()
if key in data:
actual_value = data[key]
if str(actual_value) == str(expected_value):
return True, f"JSON validation passed: {key}={actual_value}"
else:
return False, f"JSON validation failed: {key}={actual_value}, expected={expected_value}"
else:
return False, f"JSON key '{key}' not found in response"
except json.JSONDecodeError:
return False, "Response is not valid JSON"
def main():
args = parse_args()
# Validate URL
parsed_url = urlparse(args.url)
if not parsed_url.scheme or not parsed_url.netloc:
print("UNKNOWN - Invalid URL format")
sys.exit(UNKNOWN)
# Validate thresholds
if args.warning >= args.critical:
print("UNKNOWN - Warning threshold must be less than critical threshold")
sys.exit(UNKNOWN)
# Make request
response, response_time = make_request(args.url, args.timeout, args.header)
if response is None:
print(f"CRITICAL - Request timed out after {args.timeout} seconds")
sys.exit(CRITICAL)
# Check HTTP status
if response.status_code != args.status:
print(f"CRITICAL - HTTP {response.status_code}, expected {args.status}")
sys.exit(CRITICAL)
# Validate JSON content if specified
json_status = "OK"
json_message = ""
if args.key and args.value:
is_valid, message = validate_json_content(response, args.key, args.value)
if not is_valid:
print(f"CRITICAL - {message}")
sys.exit(CRITICAL)
json_message = f", {message}"
# Performance data
perf_data = f"response_time={response_time:.3f}s;{args.warning};{args.critical};0"
# Determine status based on response time
if response_time >= args.critical:
print(f"CRITICAL - Response time {response_time:.3f}s{json_message}|{perf_data}")
sys.exit(CRITICAL)
elif response_time >= args.warning:
print(f"WARNING - Response time {response_time:.3f}s{json_message}|{perf_data}")
sys.exit(WARNING)
else:
print(f"OK - Response time {response_time:.3f}s{json_message}|{perf_data}")
sys.exit(OK)
if __name__ == "__main__":
main()
Create a database connection plugin
Build a plugin that monitors database connectivity and query performance for PostgreSQL.
#!/usr/bin/env python3
import sys
import argparse
import time
try:
import psycopg2
except ImportError:
print("UNKNOWN - psycopg2 library not installed. Run: pip3 install psycopg2-binary")
sys.exit(3)
# Nagios exit codes
OK = 0
WARNING = 1
CRITICAL = 2
UNKNOWN = 3
def parse_args():
parser = argparse.ArgumentParser(description='Monitor PostgreSQL query performance')
parser.add_argument('-H', '--host', default='localhost', help='Database host')
parser.add_argument('-P', '--port', type=int, default=5432, help='Database port')
parser.add_argument('-d', '--database', required=True, help='Database name')
parser.add_argument('-u', '--username', required=True, help='Database username')
parser.add_argument('-p', '--password', required=True, help='Database password')
parser.add_argument('-q', '--query', default='SELECT 1', help='SQL query to execute')
parser.add_argument('-w', '--warning', type=float, default=1.0, help='Warning threshold in seconds')
parser.add_argument('-c', '--critical', type=float, default=3.0, help='Critical threshold in seconds')
parser.add_argument('-t', '--timeout', type=int, default=10, help='Connection timeout')
return parser.parse_args()
def execute_query(host, port, database, username, password, query, timeout):
"""Execute database query and return timing"""
try:
start_time = time.time()
conn = psycopg2.connect(
host=host,
port=port,
database=database,
user=username,
password=password,
connect_timeout=timeout
)
cursor = conn.cursor()
cursor.execute(query)
result = cursor.fetchall()
query_time = time.time() - start_time
cursor.close()
conn.close()
return True, query_time, len(result)
except psycopg2.OperationalError as e:
return False, 0, f"Connection error: {str(e)}"
except psycopg2.Error as e:
return False, 0, f"Database error: {str(e)}"
except Exception as e:
return False, 0, f"Unexpected error: {str(e)}"
def main():
args = parse_args()
# Validate thresholds
if args.warning >= args.critical:
print("UNKNOWN - Warning threshold must be less than critical threshold")
sys.exit(UNKNOWN)
# Execute query
success, query_time, result_info = execute_query(
args.host, args.port, args.database,
args.username, args.password, args.query, args.timeout
)
if not success:
print(f"CRITICAL - {result_info}")
sys.exit(CRITICAL)
# Performance data
perf_data = f"query_time={query_time:.3f}s;{args.warning};{args.critical};0"
# Determine status
if query_time >= args.critical:
print(f"CRITICAL - Query took {query_time:.3f}s, returned {result_info} rows|{perf_data}")
sys.exit(CRITICAL)
elif query_time >= args.warning:
print(f"WARNING - Query took {query_time:.3f}s, returned {result_info} rows|{perf_data}")
sys.exit(WARNING)
else:
print(f"OK - Query took {query_time:.3f}s, returned {result_info} rows|{perf_data}")
sys.exit(OK)
if __name__ == "__main__":
main()
Set executable permissions on plugins
Make your custom plugins executable by the Nagios user with proper security permissions.
sudo chmod 755 /usr/local/nagios/plugins/development/check_custom_disk.sh
sudo chmod 755 /usr/local/nagios/plugins/development/check_api_endpoint.py
sudo chmod 755 /usr/local/nagios/plugins/development/check_postgres_query.py
sudo chown nagios:nagios /usr/local/nagios/plugins/development/*
Plugin testing and validation
Test plugins manually
Run each plugin from the command line to verify functionality and output format before integration.
# Test disk check plugin
sudo -u nagios /usr/local/nagios/plugins/development/check_custom_disk.sh -w 80 -c 90 -p /
# Test API endpoint plugin
sudo -u nagios /usr/local/nagios/plugins/development/check_api_endpoint.py -u https://httpbin.org/json -w 2 -c 5
# Test with JSON validation
sudo -u nagios /usr/local/nagios/plugins/development/check_api_endpoint.py -u https://httpbin.org/json -k slideshow -v "" -w 2 -c 5
Validate plugin output format
Ensure plugins follow Nagios standards for output format, exit codes, and performance data.
# Check exit codes
echo $? # Should be 0, 1, 2, or 3
# Validate performance data format
# Should be: label=value[UOM];[warn];[crit];[min];[max]
# Test all threshold conditions
sudo -u nagios /usr/local/nagios/plugins/development/check_custom_disk.sh -w 10 -c 20 -p /
sudo -u nagios /usr/local/nagios/plugins/development/check_api_endpoint.py -u https://httpbin.org/delay/3 -w 1 -c 2
Create plugin validation script
Build an automated test script to verify plugin behavior across different scenarios.
#!/bin/bash
PLUGIN_DIR="/usr/local/nagios/plugins/development"
TEST_RESULTS="/tmp/plugin_tests.log"
echo "Nagios Plugin Validation Results" > $TEST_RESULTS
echo "Generated: $(date)" >> $TEST_RESULTS
echo "========================================" >> $TEST_RESULTS
test_plugin() {
local plugin="$1"
local args="$2"
local expected_exit="$3"
local test_name="$4"
echo "Testing: $test_name" >> $TEST_RESULTS
echo "Command: $plugin $args" >> $TEST_RESULTS
output=$(sudo -u nagios $plugin $args 2>&1)
exit_code=$?
echo "Output: $output" >> $TEST_RESULTS
echo "Exit Code: $exit_code" >> $TEST_RESULTS
if [ "$exit_code" = "$expected_exit" ]; then
echo "Result: PASS" >> $TEST_RESULTS
echo "✓ $test_name"
else
echo "Result: FAIL (expected $expected_exit, got $exit_code)" >> $TEST_RESULTS
echo "✗ $test_name"
fi
echo "" >> $TEST_RESULTS
}
# Test disk plugin
test_plugin "$PLUGIN_DIR/check_custom_disk.sh" "-w 80 -c 90 -p /" "0" "Disk Check - Normal"
test_plugin "$PLUGIN_DIR/check_custom_disk.sh" "-w 10 -c 20 -p /" "1" "Disk Check - Warning"
test_plugin "$PLUGIN_DIR/check_custom_disk.sh" "-w 90 -c 80 -p /" "3" "Disk Check - Invalid Thresholds"
# Test API plugin
test_plugin "$PLUGIN_DIR/check_api_endpoint.py" "-u https://httpbin.org/status/200 -w 2 -c 5" "0" "API Check - Normal"
test_plugin "$PLUGIN_DIR/check_api_endpoint.py" "-u https://httpbin.org/status/404 -w 2 -c 5" "2" "API Check - 404 Error"
test_plugin "$PLUGIN_DIR/check_api_endpoint.py" "-u https://httpbin.org/delay/3 -w 1 -c 2" "2" "API Check - Slow Response"
echo "Validation complete. Results saved to $TEST_RESULTS"
echo "Review with: cat $TEST_RESULTS"
sudo chmod 755 /usr/local/nagios/plugins/development/validate_plugins.sh
sudo /usr/local/nagios/plugins/development/validate_plugins.sh
Integration with Nagios Core
Deploy tested plugins to production directory
Move validated plugins to the production plugin directory and set final permissions.
sudo cp /usr/local/nagios/plugins/development/check_custom_disk.sh /usr/local/nagios/plugins/custom/
sudo cp /usr/local/nagios/plugins/development/check_api_endpoint.py /usr/local/nagios/plugins/custom/
sudo cp /usr/local/nagios/plugins/development/check_postgres_query.py /usr/local/nagios/plugins/custom/
sudo chown nagios:nagios /usr/local/nagios/plugins/custom/*
sudo chmod 755 /usr/local/nagios/plugins/custom/*
Define command definitions
Create Nagios command definitions for your custom plugins in the commands configuration file.
# Custom Disk Check Command
define command {
command_name check_custom_disk
command_line /usr/local/nagios/plugins/custom/check_custom_disk.sh -w $ARG1$ -c $ARG2$ -p $ARG3$
}
# API Endpoint Check Command
define command {
command_name check_api_endpoint
command_line /usr/local/nagios/plugins/custom/check_api_endpoint.py -u $ARG1$ -w $ARG2$ -c $ARG3$ -s $ARG4$
}
# API with JSON validation
define command {
command_name check_api_json
command_line /usr/local/nagios/plugins/custom/check_api_endpoint.py -u $ARG1$ -w $ARG2$ -c $ARG3$ -k $ARG4$ -v $ARG5$
}
# PostgreSQL Query Check Command
define command {
command_name check_postgres_query
command_line /usr/local/nagios/plugins/custom/check_postgres_query.py -H $ARG1$ -d $ARG2$ -u $ARG3$ -p $ARG4$ -q "$ARG5$" -w $ARG6$ -c $ARG7$
}
Configure service checks
Create service definitions that use your custom plugins for monitoring specific resources.
# Custom disk monitoring
define service {
use generic-service
host_name localhost
service_description Custom Disk Usage /var
check_command check_custom_disk!80!90!/var
check_interval 5
retry_interval 1
}
# API endpoint monitoring
define service {
use generic-service
host_name localhost
service_description API Health Check
check_command check_api_endpoint!https://api.example.com/health!2!5!200
check_interval 2
retry_interval 1
}
# Database query monitoring
define service {
use generic-service
host_name localhost
service_description Database Query Performance
check_command