Monitor cron jobs and systemd timers with Prometheus and Grafana alerting

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
NODE_EXPORTER_VERSION="1.7.0"
NODE_EXPORTER_USER="node_exporter"
TEXTFILE_DIR="/var/lib/node_exporter/textfile_collector"
MONITOR_SCRIPT_PATH="/usr/local/bin/monitor-job"

# Cleanup function
cleanup() {
    echo -e "${RED}[ERROR] Installation failed. Cleaning up...${NC}"
    systemctl stop node_exporter 2>/dev/null || true
    systemctl disable node_exporter 2>/dev/null || true
    rm -f /etc/systemd/system/node_exporter.service
    rm -f /usr/local/bin/node_exporter
    rm -f "$MONITOR_SCRIPT_PATH"
    rm -rf /var/lib/node_exporter
    userdel "$NODE_EXPORTER_USER" 2>/dev/null || true
    systemctl daemon-reload
}

trap cleanup ERR

# Check if running as root or with sudo
check_privileges() {
    if [[ $EUID -ne 0 ]]; then
        echo -e "${RED}This script must be run as root or with sudo${NC}"
        exit 1
    fi
}

# Auto-detect distribution
detect_distro() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian) 
                PKG_MGR="apt"
                PKG_UPDATE="apt update"
                PKG_INSTALL="apt install -y"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora) 
                PKG_MGR="dnf"
                PKG_UPDATE="dnf makecache"
                PKG_INSTALL="dnf install -y"
                # Try yum if dnf is not available
                if ! command -v dnf &> /dev/null; then
                    PKG_MGR="yum"
                    PKG_UPDATE="yum makecache"
                    PKG_INSTALL="yum install -y"
                fi
                ;;
            amzn) 
                PKG_MGR="yum"
                PKG_UPDATE="yum makecache"
                PKG_INSTALL="yum install -y"
                ;;
            *) 
                echo -e "${RED}Unsupported distribution: $ID${NC}"
                exit 1
                ;;
        esac
        echo -e "${BLUE}Detected distribution: $PRETTY_NAME${NC}"
    else
        echo -e "${RED}Cannot detect distribution. /etc/os-release not found.${NC}"
        exit 1
    fi
}

# Update package repositories
update_packages() {
    echo -e "${BLUE}[1/8] Updating package repositories...${NC}"
    $PKG_UPDATE
}

# Install required packages
install_dependencies() {
    echo -e "${BLUE}[2/8] Installing dependencies...${NC}"
    $PKG_INSTALL wget tar
}

# Download and install node_exporter
install_node_exporter() {
    echo -e "${BLUE}[3/8] Downloading and installing node_exporter...${NC}"
    
    cd /tmp
    wget -q "https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz"
    tar xzf "node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz"
    
    cp "node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64/node_exporter" /usr/local/bin/
    chown root:root /usr/local/bin/node_exporter
    chmod 755 /usr/local/bin/node_exporter
    
    # Clean up
    rm -rf "node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64"*
}

# Create node_exporter system user
create_user() {
    echo -e "${BLUE}[4/8] Creating node_exporter system user...${NC}"
    
    if ! id "$NODE_EXPORTER_USER" &>/dev/null; then
        useradd --no-create-home --shell /bin/false --system "$NODE_EXPORTER_USER"
    fi
}

# Create textfile collector directory
create_directories() {
    echo -e "${BLUE}[5/8] Creating directories...${NC}"
    
    mkdir -p "$TEXTFILE_DIR"
    chown "$NODE_EXPORTER_USER:$NODE_EXPORTER_USER" "$TEXTFILE_DIR"
    chmod 755 "$TEXTFILE_DIR"
    
    # Create parent directory with proper permissions
    chown "$NODE_EXPORTER_USER:$NODE_EXPORTER_USER" "$(dirname $TEXTFILE_DIR)"
}

# Create systemd service
create_service() {
    echo -e "${BLUE}[6/8] Creating systemd service...${NC}"
    
    cat > /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter --collector.systemd --collector.textfile.directory=/var/lib/node_exporter/textfile_collector
Restart=always
RestartSec=3
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=yes
ReadWritePaths=/var/lib/node_exporter

[Install]
WantedBy=multi-user.target
EOF

    systemctl daemon-reload
    systemctl enable node_exporter
    systemctl start node_exporter
}

# Create job monitoring script
create_monitor_script() {
    echo -e "${BLUE}[7/8] Creating job monitoring script...${NC}"
    
    cat > "$MONITOR_SCRIPT_PATH" << 'EOF'
#!/bin/bash

JOB_NAME="$1"
JOB_STATUS="$2"  # success, failed, or running
JOB_DURATION="$3"  # optional duration in seconds
METRIC_FILE="/var/lib/node_exporter/textfile_collector/${JOB_NAME}.prom"
TIMESTAMP=$(date +%s)

if [ -z "$JOB_NAME" ] || [ -z "$JOB_STATUS" ]; then
    echo "Usage: $0 <job_name> <status> [duration_seconds]"
    echo "Status: success, failed, or running"
    exit 1
fi

# Create temporary file
TEMP_FILE="$(mktemp)"

# Write job status metric
echo "# HELP job_last_status Last execution status of scheduled job (1=success, 0=failed)" >> "$TEMP_FILE"
echo "# TYPE job_last_status gauge" >> "$TEMP_FILE"
if [ "$JOB_STATUS" = "success" ]; then
    echo "job_last_status{job=\"$JOB_NAME\"} 1" >> "$TEMP_FILE"
else
    echo "job_last_status{job=\"$JOB_NAME\"} 0" >> "$TEMP_FILE"
fi

# Write timestamp metric
echo "# HELP job_last_run_timestamp Unix timestamp of last job execution" >> "$TEMP_FILE"
echo "# TYPE job_last_run_timestamp gauge" >> "$TEMP_FILE"
echo "job_last_run_timestamp{job=\"$JOB_NAME\"} $TIMESTAMP" >> "$TEMP_FILE"

# Write duration metric if provided
if [ -n "$JOB_DURATION" ]; then
    echo "# HELP job_duration_seconds Duration of last job execution in seconds" >> "$TEMP_FILE"
    echo "# TYPE job_duration_seconds gauge" >> "$TEMP_FILE"
    echo "job_duration_seconds{job=\"$JOB_NAME\"} $JOB_DURATION" >> "$TEMP_FILE"
fi

# Atomically move file to final location
mv "$TEMP_FILE" "$METRIC_FILE"
chmod 644 "$METRIC_FILE"
EOF

    chmod 755 "$MONITOR_SCRIPT_PATH"
}

# Verify installation
verify_installation() {
    echo -e "${BLUE}[8/8] Verifying installation...${NC}"
    
    # Check if node_exporter is running
    if systemctl is-active --quiet node_exporter; then
        echo -e "${GREEN}✓ node_exporter service is running${NC}"
    else
        echo -e "${RED}✗ node_exporter service is not running${NC}"
        exit 1
    fi
    
    # Check if metrics endpoint is accessible
    sleep 2
    if curl -s http://localhost:9100/metrics | grep -q "node_exporter_build_info"; then
        echo -e "${GREEN}✓ node_exporter metrics endpoint is accessible${NC}"
    else
        echo -e "${RED}✗ node_exporter metrics endpoint is not accessible${NC}"
        exit 1
    fi
    
    # Check if monitor script exists and is executable
    if [[ -x "$MONITOR_SCRIPT_PATH" ]]; then
        echo -e "${GREEN}✓ Job monitoring script is installed${NC}"
    else
        echo -e "${RED}✗ Job monitoring script is not properly installed${NC}"
        exit 1
    fi
    
    # Test monitor script
    if "$MONITOR_SCRIPT_PATH" test_job success 10; then
        echo -e "${GREEN}✓ Job monitoring script is working${NC}"
    else
        echo -e "${RED}✗ Job monitoring script test failed${NC}"
        exit 1
    fi
    
    echo -e "${GREEN}Installation completed successfully!${NC}"
    echo -e "${YELLOW}Usage examples:${NC}"
    echo "  # Report successful job:"
    echo "  $MONITOR_SCRIPT_PATH backup_job success 120"
    echo "  # Report failed job:"
    echo "  $MONITOR_SCRIPT_PATH backup_job failed"
    echo "  # Add to crontab:"
    echo "  0 2 * * * /path/to/backup.sh && $MONITOR_SCRIPT_PATH backup success || $MONITOR_SCRIPT_PATH backup failed"
    echo -e "${YELLOW}node_exporter is running on http://localhost:9100/metrics${NC}"
}

# Main execution
main() {
    echo -e "${GREEN}Prometheus Job Monitoring Setup${NC}"
    echo "=================================="
    
    check_privileges
    detect_distro
    update_packages
    install_dependencies
    install_node_exporter
    create_user
    create_directories
    create_service
    create_monitor_script
    verify_installation
}

main "$@"