Implement Apache log analysis with GoAccess and ELK stack for real-time monitoring

#!/usr/bin/env bash

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Variables
DOMAIN="${1:-localhost}"
ELASTIC_VERSION="8.x"
GOACCESS_PORT="7890"
KIBANA_PORT="5601"

# Usage
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
    echo "Usage: $0 [domain_name]"
    echo "Example: $0 example.com"
    exit 0
fi

# Cleanup function
cleanup() {
    echo -e "${RED}[ERROR] Installation failed. Cleaning up...${NC}"
    systemctl stop elasticsearch logstash kibana 2>/dev/null || true
}
trap cleanup ERR

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    echo -e "${RED}This script must be run as root${NC}"
    exit 1
fi

# Detect distribution
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian)
            PKG_MGR="apt"
            PKG_INSTALL="apt install -y"
            APACHE_LOG_DIR="/var/log/apache2"
            APACHE_CONFIG_DIR="/etc/apache2"
            JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64"
            ;;
        almalinux|rocky|centos|rhel|ol|fedora)
            PKG_MGR="dnf"
            PKG_INSTALL="dnf install -y"
            APACHE_LOG_DIR="/var/log/httpd"
            APACHE_CONFIG_DIR="/etc/httpd"
            JAVA_HOME="/usr/lib/jvm/java-11-openjdk"
            if [[ "$VERSION_ID" =~ ^7 ]]; then
                PKG_MGR="yum"
                PKG_INSTALL="yum install -y"
            fi
            ;;
        amzn)
            PKG_MGR="yum"
            PKG_INSTALL="yum install -y"
            APACHE_LOG_DIR="/var/log/httpd"
            APACHE_CONFIG_DIR="/etc/httpd"
            JAVA_HOME="/usr/lib/jvm/java-11-openjdk"
            ;;
        *)
            echo -e "${RED}Unsupported distribution: $ID${NC}"
            exit 1
            ;;
    esac
else
    echo -e "${RED}Cannot detect distribution${NC}"
    exit 1
fi

echo -e "${GREEN}Installing Apache log analysis stack on $ID...${NC}"

# Update system
echo -e "${GREEN}[1/8] Updating system packages...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    apt update && apt upgrade -y
    $PKG_INSTALL curl wget gnupg2 software-properties-common
else
    $PKG_INSTALL curl wget gnupg2
fi

# Install Java
echo -e "${GREEN}[2/8] Installing Java 11...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL openjdk-11-jdk
else
    $PKG_INSTALL java-11-openjdk java-11-openjdk-devel
    if [[ "$ID" =~ ^(centos|rhel)$ && "$VERSION_ID" =~ ^7 ]]; then
        $PKG_INSTALL epel-release
    fi
fi

echo "JAVA_HOME=$JAVA_HOME" >> /etc/environment
export JAVA_HOME

# Install GoAccess
echo -e "${GREEN}[3/8] Installing GoAccess...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL goaccess
else
    if ! rpm -qa | grep -q epel-release; then
        $PKG_INSTALL epel-release
    fi
    $PKG_INSTALL goaccess
fi

# Configure GoAccess
echo -e "${GREEN}[4/8] Configuring GoAccess...${NC}"
mkdir -p /var/www/html
cat > /etc/goaccess/goaccess.conf << EOF
time-format %H:%M:%S
date-format %d/%b/%Y
log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u"

real-time-html true
port $GOACCESS_PORT
ws-url wss://$DOMAIN:$GOACCESS_PORT

html-prefs {"theme":"bright","perPage":10}
html-report-title Apache Log Analysis

ignore-panel REFERRERS
ignore-panel KEYPHRASES
ignore-panel GEOLOCATION

output /var/www/html/goaccess-report.html
EOF

chown -R www-data:www-data /var/www/html 2>/dev/null || chown -R apache:apache /var/www/html

# Add Elastic repository
echo -e "${GREEN}[5/8] Adding Elastic Stack repository...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | gpg --dearmor -o /usr/share/keyrings/elastic-keyring.gpg
    echo "deb [signed-by=/usr/share/keyrings/elastic-keyring.gpg] https://artifacts.elastic.co/packages/$ELASTIC_VERSION/apt stable main" > /etc/apt/sources.list.d/elastic-$ELASTIC_VERSION.list
    apt update
else
    rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
    cat > /etc/yum.repos.d/elastic.repo << EOF
[elasticsearch]
name=Elasticsearch repository for $ELASTIC_VERSION packages
baseurl=https://artifacts.elastic.co/packages/$ELASTIC_VERSION/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=0
autorefresh=1
type=rpm-md
EOF
fi

# Install Elasticsearch
echo -e "${GREEN}[6/8] Installing Elasticsearch...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL elasticsearch
else
    $PKG_MGR install -y --enablerepo=elasticsearch elasticsearch
fi

# Configure Elasticsearch
cat > /etc/elasticsearch/elasticsearch.yml << EOF
cluster.name: apache-logs
node.name: node-1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 127.0.0.1
http.port: 9200
discovery.type: single-node
xpack.security.enabled: false
EOF

systemctl enable elasticsearch
systemctl start elasticsearch

# Install Logstash
echo -e "${GREEN}[7/8] Installing and configuring Logstash...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL logstash
else
    $PKG_MGR install -y --enablerepo=elasticsearch logstash
fi

# Create Logstash configuration
cat > /etc/logstash/conf.d/apache.conf << EOF
input {
  file {
    path => "$APACHE_LOG_DIR/access.log"
    type => "apache_access"
    start_position => "beginning"
  }
  file {
    path => "$APACHE_LOG_DIR/error.log"
    type => "apache_error"
    start_position => "beginning"
  }
}

filter {
  if [type] == "apache_access" {
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
    date {
      match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    mutate {
      convert => { "response" => "integer" }
      convert => { "bytes" => "integer" }
    }
    if [clientip] {
      geoip {
        source => "clientip"
      }
    }
  }
  
  if [type] == "apache_error" {
    grok {
      match => { "message" => "\[%{HTTPDATE:timestamp}\] \[%{WORD:level}\] \[pid %{POSINT:pid}\] %{GREEDYDATA:error_message}" }
    }
  }
}

output {
  elasticsearch {
    hosts => ["localhost:9200"]
    index => "apache-logs-%{+YYYY.MM.dd}"
  }
}
EOF

systemctl enable logstash
systemctl start logstash

# Install and configure Kibana
echo -e "${GREEN}[8/8] Installing and configuring Kibana...${NC}"
if [[ "$PKG_MGR" == "apt" ]]; then
    $PKG_INSTALL kibana
else
    $PKG_MGR install -y --enablerepo=elasticsearch kibana
fi

cat > /etc/kibana/kibana.yml << EOF
server.port: $KIBANA_PORT
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:9200"]
server.name: "apache-logs-kibana"
EOF

systemctl enable kibana
systemctl start kibana

# Configure firewall
echo -e "${YELLOW}Configuring firewall...${NC}"
if command -v ufw >/dev/null 2>&1; then
    ufw allow $GOACCESS_PORT
    ufw allow $KIBANA_PORT
elif command -v firewall-cmd >/dev/null 2>&1; then
    firewall-cmd --permanent --add-port=$GOACCESS_PORT/tcp
    firewall-cmd --permanent --add-port=$KIBANA_PORT/tcp
    firewall-cmd --reload
fi

# Set permissions
chown -R elasticsearch:elasticsearch /var/lib/elasticsearch /var/log/elasticsearch
chown -R logstash:logstash /etc/logstash
chown -R kibana:kibana /etc/kibana
chmod 750 /etc/elasticsearch /etc/logstash /etc/kibana
chmod 644 /etc/elasticsearch/elasticsearch.yml /etc/logstash/conf.d/apache.conf /etc/kibana/kibana.yml

# Wait for services to start
echo -e "${YELLOW}Waiting for services to start...${NC}"
sleep 30

# Verification
echo -e "${GREEN}Verifying installation...${NC}"
if systemctl is-active --quiet elasticsearch; then
    echo -e "${GREEN}✓ Elasticsearch is running${NC}"
else
    echo -e "${RED}✗ Elasticsearch failed to start${NC}"
fi

if systemctl is-active --quiet logstash; then
    echo -e "${GREEN}✓ Logstash is running${NC}"
else
    echo -e "${RED}✗ Logstash failed to start${NC}"
fi

if systemctl is-active --quiet kibana; then
    echo -e "${GREEN}✓ Kibana is running${NC}"
else
    echo -e "${RED}✗ Kibana failed to start${NC}"
fi

echo -e "${GREEN}Installation completed successfully!${NC}"
echo -e "${YELLOW}Access points:${NC}"
echo "- GoAccess HTML report: http://$DOMAIN/goaccess-report.html"
echo "- Kibana dashboard: http://$DOMAIN:$KIBANA_PORT"
echo ""
echo -e "${YELLOW}To generate GoAccess real-time report:${NC}"
echo "goaccess $APACHE_LOG_DIR/access.log -o /var/www/html/goaccess-report.html --log-format=COMBINED --real-time-html"