Implement log-based monitoring and alerting with Grafana and Loki

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Global variables
LOKI_VERSION="2.9.3"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# Usage function
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo "Options:"
    echo "  -h, --help     Show this help message"
    echo "  --version VER  Specify Loki version (default: $LOKI_VERSION)"
    exit 1
}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            usage
            ;;
        --version)
            LOKI_VERSION="$2"
            shift 2
            ;;
        *)
            echo -e "${RED}Unknown option: $1${NC}"
            usage
            ;;
    esac
done

# Cleanup function for rollback
cleanup() {
    echo -e "${RED}Installation failed. Rolling back...${NC}"
    systemctl stop loki promtail 2>/dev/null || true
    systemctl disable loki promtail 2>/dev/null || true
    userdel -r loki 2>/dev/null || true
    userdel -r promtail 2>/dev/null || true
    rm -rf /etc/loki /var/lib/loki /etc/promtail /var/lib/promtail
    rm -f /usr/local/bin/loki /usr/local/bin/promtail
    rm -f /etc/systemd/system/loki.service /etc/systemd/system/promtail.service
    echo -e "${RED}Rollback completed${NC}"
    exit 1
}

trap cleanup ERR

# Check prerequisites
check_prerequisites() {
    if [[ $EUID -ne 0 ]]; then
        echo -e "${RED}This script must be run as root${NC}"
        exit 1
    fi
    
    if ! command -v curl >/dev/null 2>&1; then
        echo -e "${RED}curl is required but not installed${NC}"
        exit 1
    fi
}

# Auto-detect distro
detect_distro() {
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        case "$ID" in
            ubuntu|debian) 
                PKG_MGR="apt"
                PKG_INSTALL="apt install -y"
                PKG_UPDATE="apt update && apt upgrade -y"
                ;;
            almalinux|rocky|centos|rhel|ol|fedora) 
                PKG_MGR="dnf"
                PKG_INSTALL="dnf install -y"
                PKG_UPDATE="dnf update -y"
                ;;
            amzn) 
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
                PKG_UPDATE="yum update -y"
                ;;
            *) 
                echo -e "${RED}Unsupported distro: $ID${NC}"
                exit 1 
                ;;
        esac
    else
        echo -e "${RED}Cannot detect distribution${NC}"
        exit 1
    fi
}

# Update system packages
update_system() {
    echo -e "${GREEN}[1/10] Updating system packages...${NC}"
    $PKG_UPDATE
    $PKG_INSTALL curl wget unzip
}

# Create system users
create_users() {
    echo -e "${GREEN}[2/10] Creating system users...${NC}"
    useradd --system --no-create-home --shell /bin/false loki || true
    useradd --system --no-create-home --shell /bin/false promtail || true
}

# Download and install Loki
install_loki() {
    echo -e "${GREEN}[3/10] Downloading and installing Loki...${NC}"
    cd /tmp
    wget -q "https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/loki-linux-amd64.zip"
    unzip -o loki-linux-amd64.zip
    mv loki-linux-amd64 /usr/local/bin/loki
    chmod 755 /usr/local/bin/loki
    rm -f loki-linux-amd64.zip
}

# Create Loki directories and configuration
setup_loki_dirs() {
    echo -e "${GREEN}[4/10] Setting up Loki directories...${NC}"
    mkdir -p /etc/loki /var/lib/loki/{wal,boltdb-shipper-active,boltdb-shipper-cache,chunks,rules}
    chown -R loki:loki /etc/loki /var/lib/loki
    chmod 755 /etc/loki /var/lib/loki
    find /var/lib/loki -type d -exec chmod 755 {} \;
}

# Configure Loki
configure_loki() {
    echo -e "${GREEN}[5/10] Configuring Loki...${NC}"
    cat > /etc/loki/loki.yml << 'EOF'
auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

ingester:
  wal:
    enabled: true
    dir: /var/lib/loki/wal
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 1h
  max_chunk_age: 1h
  chunk_target_size: 1048576
  chunk_retain_period: 30s
  max_transfer_retries: 0

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  boltdb_shipper:
    active_index_directory: /var/lib/loki/boltdb-shipper-active
    cache_location: /var/lib/loki/boltdb-shipper-cache
    cache_ttl: 24h
    shared_store: filesystem
  filesystem:
    directory: /var/lib/loki/chunks

compactor:
  working_directory: /var/lib/loki
  shared_store: filesystem

limits_config:
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  ingestion_rate_mb: 16
  ingestion_burst_size_mb: 32

chunk_store_config:
  max_look_back_period: 0s

table_manager:
  retention_deletes_enabled: false
  retention_period: 0s

ruler:
  storage:
    type: local
    local:
      directory: /var/lib/loki/rules
  rule_path: /var/lib/loki/rules
  alertmanager_url: http://localhost:9093
  ring:
    kvstore:
      store: inmemory
  enable_api: true
EOF
    chown loki:loki /etc/loki/loki.yml
    chmod 644 /etc/loki/loki.yml
}

# Create Loki systemd service
create_loki_service() {
    echo -e "${GREEN}[6/10] Creating Loki systemd service...${NC}"
    cat > /etc/systemd/system/loki.service << 'EOF'
[Unit]
Description=Loki service
After=network.target

[Service]
Type=simple
User=loki
Group=loki
ExecStart=/usr/local/bin/loki -config.file /etc/loki/loki.yml
Restart=on-failure
RestartSec=20
StandardOutput=journal
StandardError=journal
SyslogIdentifier=loki
KillMode=mixed
KillSignal=SIGTERM

[Install]
WantedBy=multi-user.target
EOF
}

# Download and install Promtail
install_promtail() {
    echo -e "${GREEN}[7/10] Downloading and installing Promtail...${NC}"
    cd /tmp
    wget -q "https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/promtail-linux-amd64.zip"
    unzip -o promtail-linux-amd64.zip
    mv promtail-linux-amd64 /usr/local/bin/promtail
    chmod 755 /usr/local/bin/promtail
    rm -f promtail-linux-amd64.zip
}

# Create Promtail directories and configuration
setup_promtail() {
    echo -e "${GREEN}[8/10] Setting up Promtail...${NC}"
    mkdir -p /etc/promtail /var/lib/promtail
    chown -R promtail:promtail /etc/promtail /var/lib/promtail
    chmod 755 /etc/promtail /var/lib/promtail
    
    cat > /etc/promtail/promtail.yml << 'EOF'
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /var/lib/promtail/positions.yaml

clients:
  - url: http://localhost:3100/loki/api/v1/push

scrape_configs:
  - job_name: system
    static_configs:
      - targets:
          - localhost
        labels:
          job: varlogs
          __path__: /var/log/*log
  - job_name: syslog
    static_configs:
      - targets:
          - localhost
        labels:
          job: syslog
          __path__: /var/log/syslog
EOF
    chown promtail:promtail /etc/promtail/promtail.yml
    chmod 644 /etc/promtail/promtail.yml
}

# Create Promtail systemd service
create_promtail_service() {
    echo -e "${GREEN}[9/10] Creating Promtail systemd service...${NC}"
    cat > /etc/systemd/system/promtail.service << 'EOF'
[Unit]
Description=Promtail service
After=network.target

[Service]
Type=simple
User=promtail
Group=promtail
ExecStart=/usr/local/bin/promtail -config.file /etc/promtail/promtail.yml
Restart=on-failure
RestartSec=20
StandardOutput=journal
StandardError=journal
SyslogIdentifier=promtail
KillMode=mixed
KillSignal=SIGTERM

[Install]
WantedBy=multi-user.target
EOF
}

# Start and enable services
start_services() {
    echo -e "${GREEN}[10/10] Starting and enabling services...${NC}"
    systemctl daemon-reload
    systemctl enable loki promtail
    systemctl start loki
    sleep 5
    systemctl start promtail
}

# Verify installation
verify_installation() {
    echo -e "${GREEN}Verifying installation...${NC}"
    
    if systemctl is-active --quiet loki; then
        echo -e "${GREEN}✓ Loki is running${NC}"
    else
        echo -e "${RED}✗ Loki is not running${NC}"
        return 1
    fi
    
    if systemctl is-active --quiet promtail; then
        echo -e "${GREEN}✓ Promtail is running${NC}"
    else
        echo -e "${RED}✗ Promtail is not running${NC}"
        return 1
    fi
    
    if curl -s http://localhost:3100/ready >/dev/null; then
        echo -e "${GREEN}✓ Loki API is accessible${NC}"
    else
        echo -e "${RED}✗ Loki API is not accessible${NC}"
        return 1
    fi
    
    echo -e "${GREEN}Installation completed successfully!${NC}"
    echo -e "${YELLOW}Loki is running on http://localhost:3100${NC}"
    echo -e "${YELLOW}Configure Grafana to use Loki as a data source at: http://localhost:3100${NC}"
}

# Main execution
main() {
    check_prerequisites
    detect_distro
    update_system
    create_users
    install_loki
    setup_loki_dirs
    configure_loki
    create_loki_service
    install_promtail
    setup_promtail
    create_promtail_service
    start_services
    verify_installation
}

main "$@"