Monitor system time drift with Prometheus and Grafana alerts

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Configuration
NODE_EXPORTER_VERSION="1.7.0"
PROMETHEUS_PORT="9090"
NODE_EXPORTER_PORT="9100"

# Usage function
usage() {
    echo "Usage: $0 [--prometheus-host HOSTNAME] [--no-prometheus]"
    echo "  --prometheus-host: Hostname/IP where Prometheus is running (default: localhost)"
    echo "  --no-prometheus: Skip Prometheus configuration steps"
    exit 1
}

# Parse arguments
PROMETHEUS_HOST="localhost"
SKIP_PROMETHEUS=false

while [[ $# -gt 0 ]]; do
    case $1 in
        --prometheus-host)
            PROMETHEUS_HOST="$2"
            shift 2
            ;;
        --no-prometheus)
            SKIP_PROMETHEUS=true
            shift
            ;;
        -h|--help)
            usage
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Cleanup function for rollback
cleanup() {
    if [[ $? -ne 0 ]]; then
        echo -e "${RED}Installation failed. Cleaning up...${NC}"
        systemctl stop node_exporter 2>/dev/null || true
        systemctl disable node_exporter 2>/dev/null || true
        rm -f /etc/systemd/system/node_exporter.service
        rm -f /usr/local/bin/node_exporter
        userdel node_exporter 2>/dev/null || true
    fi
}
trap cleanup ERR

# Check prerequisites
echo -e "${YELLOW}[1/8] Checking prerequisites...${NC}"
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root${NC}"
    exit 1
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            PKG_UPDATE="apt update"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            PKG_UPDATE="dnf check-update || true"
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            PKG_UPDATE="yum check-update || true"
            ;;
        *)
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}Cannot detect distribution${NC}"
    exit 1
fi

echo -e "${GREEN}Distribution detected: $PRETTY_NAME${NC}"

# Update package manager
echo -e "${YELLOW}[2/8] Updating package manager...${NC}"
$PKG_UPDATE

# Install required packages
echo -e "${YELLOW}[3/8] Installing required packages...${NC}"
$PKG_INSTALL wget tar chrony curl

# Download and install node exporter
echo -e "${YELLOW}[4/8] Installing Node Exporter...${NC}"
cd /tmp
wget -q "https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz"
tar xzf "node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz"
cp "node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64/node_exporter" /usr/local/bin/
chown root:root /usr/local/bin/node_exporter
chmod 755 /usr/local/bin/node_exporter

# Create node_exporter user
if ! id "node_exporter" &>/dev/null; then
    useradd --no-create-home --shell /bin/false node_exporter
fi

# Create systemd service
echo -e "${YELLOW}[5/8] Creating systemd service...${NC}"
cat > /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter --collector.systemd --collector.ntp --collector.time
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

chmod 644 /etc/systemd/system/node_exporter.service
systemctl daemon-reload
systemctl enable node_exporter
systemctl start node_exporter

# Configure chrony
echo -e "${YELLOW}[6/8] Configuring chrony...${NC}"
cat > /etc/chrony.conf << 'EOF'
# Public NTP servers
pool 2.pool.ntp.org iburst
pool 1.pool.ntp.org iburst
pool 0.pool.ntp.org iburst

# Record statistics
driftfile /var/lib/chrony/chrony.drift
dumpdir /var/lib/chrony
logdir /var/log/chrony
log statistics measurements tracking

# Maximum allowed offset
maxupdateskew 100.0

# Enable command port for monitoring
cmdport 323
cmdallow 127.0.0.1

# Step clock if offset is larger than 1 second
makestep 1.0 3

# Enable RTC synchronization
rtcsync
EOF

chmod 644 /etc/chrony.conf
systemctl enable chronyd
systemctl restart chronyd

# Configure firewall
echo -e "${YELLOW}[7/8] Configuring firewall...${NC}"
if command -v firewall-cmd &> /dev/null && systemctl is-active firewalld &> /dev/null; then
    firewall-cmd --permanent --add-port=${NODE_EXPORTER_PORT}/tcp
    firewall-cmd --reload
elif command -v ufw &> /dev/null; then
    ufw allow ${NODE_EXPORTER_PORT}/tcp
fi

# Create Prometheus configuration if not skipped
if [[ "$SKIP_PROMETHEUS" == false ]]; then
    echo -e "${YELLOW}[8/8] Creating Prometheus configuration templates...${NC}"
    
    # Create Prometheus config directory if it doesn't exist
    mkdir -p /etc/prometheus
    
    # Create time drift rules file
    cat > /etc/prometheus/time_drift_rules.yml << EOF
groups:
  - name: time_drift_alerts
    rules:
    - alert: ClockDriftHigh
      expr: abs(node_timex_offset_seconds) > 0.05
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: "System clock drift detected on {{ \$labels.instance }}"
        description: "Clock drift is {{ \$value }} seconds on {{ \$labels.instance }}"
    
    - alert: NTPSyncLost
      expr: node_timex_sync_status == 0
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: "NTP synchronization lost on {{ \$labels.instance }}"
        description: "System is not synchronized with NTP on {{ \$labels.instance }}"
    
    - alert: ClockSkewHigh
      expr: node_timex_estimated_error_seconds > 0.1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "High clock skew detected on {{ \$labels.instance }}"
        description: "Clock skew is {{ \$value }} seconds on {{ \$labels.instance }}"
EOF
    
    # Create sample Prometheus config
    cat > /etc/prometheus/prometheus_time_monitoring.yml << EOF
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "time_drift_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - ${PROMETHEUS_HOST}:9093

scrape_configs:
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['${PROMETHEUS_HOST}:${NODE_EXPORTER_PORT}']
    scrape_interval: 10s
    metrics_path: /metrics
EOF
    
    chmod 644 /etc/prometheus/time_drift_rules.yml
    chmod 644 /etc/prometheus/prometheus_time_monitoring.yml
    chown -R root:root /etc/prometheus
    
    echo -e "${GREEN}Prometheus configuration files created in /etc/prometheus/${NC}"
else
    echo -e "${YELLOW}[8/8] Skipping Prometheus configuration as requested${NC}"
fi

# Verification
echo -e "${YELLOW}Verifying installation...${NC}"

# Check node_exporter service
if systemctl is-active node_exporter &> /dev/null; then
    echo -e "${GREEN}✓ Node Exporter is running${NC}"
else
    echo -e "${RED}✗ Node Exporter failed to start${NC}"
    exit 1
fi

# Check chrony service
if systemctl is-active chronyd &> /dev/null; then
    echo -e "${GREEN}✓ Chrony is running${NC}"
else
    echo -e "${RED}✗ Chrony failed to start${NC}"
    exit 1
fi

# Check metrics endpoint
if curl -s "http://localhost:${NODE_EXPORTER_PORT}/metrics" | grep -q "node_timex_offset_seconds"; then
    echo -e "${GREEN}✓ Time metrics are available${NC}"
else
    echo -e "${RED}✗ Time metrics not found${NC}"
    exit 1
fi

# Check chrony synchronization
sleep 5
if chrony sources &> /dev/null; then
    echo -e "${GREEN}✓ NTP sources are configured${NC}"
else
    echo -e "${YELLOW}⚠ NTP synchronization may take time to establish${NC}"
fi

echo -e "\n${GREEN}Installation completed successfully!${NC}"
echo -e "Node Exporter is running on port ${NODE_EXPORTER_PORT}"
echo -e "Time metrics endpoint: http://localhost:${NODE_EXPORTER_PORT}/metrics"
if [[ "$SKIP_PROMETHEUS" == false ]]; then
    echo -e "Prometheus configuration templates created in /etc/prometheus/"
fi
echo -e "\nKey metrics to monitor:"
echo -e "  - node_timex_offset_seconds (clock offset)"
echo -e "  - node_timex_sync_status (NTP sync status)"
echo -e "  - node_timex_estimated_error_seconds (clock accuracy)"

rm -f "/tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz"
rm -rf "/tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64"