Configure Kubernetes network monitoring with Hubble and Cilium for traffic visibility

#!/usr/bin/env bash
set -euo pipefail

# ---------------------------------------------------------------------------
# Configure Kubernetes network monitoring with Hubble + Cilium
# ---------------------------------------------------------------------------

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

log_info()  { echo -e "${GREEN}[+] $*${NC}"; }
log_warn()  { echo -e "${YELLOW}[!] $*${NC}"; }
log_error() { echo -e "${RED}[x] $*${NC}"; }

usage() {
    cat <<EOF
Usage: $0 [-n cilium-test-namespace] [-t] [-p hubble-ui-port]

  -n NAMESPACE   Namespace used for cilium connectivity test (default: cilium-test)
  -t             Run full cilium connectivity test (slow, creates workloads)
  -p PORT        Local port to port-forward Hubble UI to (default: 12000)
  -h             Show this help message
EOF
    exit 1
}

TEST_NAMESPACE="cilium-test"
RUN_CONN_TEST="false"
UI_PORT="12000"

while getopts ":n:tp:h" opt; do
    case "$opt" in
        n) TEST_NAMESPACE="$OPTARG" ;;
        t) RUN_CONN_TEST="true" ;;
        p) UI_PORT="$OPTARG" ;;
        h) usage ;;
        *) usage ;;
    esac
done

TMP_ARCHIVE=""
cleanup() {
    local exit_code=$?
    if [ -n "$TMP_ARCHIVE" ] && [ -f "$TMP_ARCHIVE" ]; then
        rm -f "$TMP_ARCHIVE"
    fi
    if [ $exit_code -ne 0 ]; then
        log_error "Script failed at line $BASH_LINENO. Rolling back partial state where possible."
    fi
    exit $exit_code
}
trap cleanup ERR EXIT

TOTAL_STEPS=9

# ---------------------------------------------------------------------------
# [1/9] Root / sudo check
# ---------------------------------------------------------------------------
echo "[1/${TOTAL_STEPS}] Checking privileges..."
if [ "$(id -u)" -ne 0 ]; then
    if ! command -v sudo >/dev/null 2>&1; then
        log_error "This script requires root or sudo. Neither is available."
        exit 1
    fi
    SUDO="sudo"
else
    SUDO=""
fi
log_info "Privilege check passed."

# ---------------------------------------------------------------------------
# [2/9] Detect distro (Debian-based vs RHEL-based)
# ---------------------------------------------------------------------------
echo "[2/${TOTAL_STEPS}] Detecting distribution..."
if [ -f /etc/os-release ]; then
    . /etc/os-release
    case "$ID" in
        ubuntu|debian) PKG_MGR="apt"; PKG_INSTALL="$SUDO apt install -y" ;;
        almalinux|rocky|centos|rhel|ol|fedora) PKG_MGR="dnf"; PKG_INSTALL="$SUDO dnf install -y" ;;
        amzn) PKG_MGR="yum"; PKG_INSTALL="$SUDO yum install -y" ;;
        *) log_error "Unsupported distro: $ID"; exit 1 ;;
    esac
else
    log_error "/etc/os-release not found. Cannot detect distro."
    exit 1
fi
log_info "Detected distro: $ID (using $PKG_MGR)"

# ---------------------------------------------------------------------------
# [3/9] Prerequisite tools: curl, tar, kubectl, helm
# ---------------------------------------------------------------------------
echo "[3/${TOTAL_STEPS}] Checking prerequisite tooling..."
if [ "$PKG_MGR" = "apt" ]; then
    $SUDO apt update -y
fi

for tool in curl tar; do
    if ! command -v "$tool" >/dev/null 2>&1; then
        log_warn "$tool not found, installing..."
        $PKG_INSTALL "$tool"
    fi
done

if ! command -v kubectl >/dev/null 2>&1; then
    log_error "kubectl not found. Install and configure kubectl against your cluster before running this script."
    exit 1
fi

if ! command -v helm >/dev/null 2>&1; then
    log_error "Helm 3 not found. Install Helm before running this script."
    exit 1
fi

kubectl version --client >/dev/null
helm version >/dev/null
log_info "kubectl and helm are present."

if ! kubectl get nodes -o wide >/dev/null 2>&1; then
    log_error "kubectl cannot reach the cluster. Check your kubeconfig."
    exit 1
fi
log_info "Cluster reachable."

# ---------------------------------------------------------------------------
# [4/9] Install Cilium CLI (arch-aware, works across all supported distros)
# ---------------------------------------------------------------------------
echo "[4/${TOTAL_STEPS}] Installing Cilium CLI..."
if command -v cilium >/dev/null 2>&1; then
    log_warn "cilium CLI already installed, skipping download."
else
    ARCH="amd64"
    case "$(uname -m)" in
        x86_64) ARCH="amd64" ;;
        aarch64|arm64) ARCH="arm64" ;;
        *) log_error "Unsupported architecture: $(uname -m)"; exit 1 ;;
    esac

    CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
    if [ -z "$CILIUM_CLI_VERSION" ]; then
        log_error "Could not determine latest cilium-cli version."
        exit 1
    fi

    WORKDIR=$(mktemp -d)
    TMP_ARCHIVE="${WORKDIR}/cilium-linux-${ARCH}.tar.gz"

    curl -L --fail --remote-name-all \
        "https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${ARCH}.tar.gz" \
        -o "$TMP_ARCHIVE"

    $SUDO tar xzvfC "$TMP_ARCHIVE" /usr/local/bin
    $SUDO chmod 755 /usr/local/bin/cilium
    rm -f "$TMP_ARCHIVE"
    TMP_ARCHIVE=""
    log_info "Cilium CLI ${CILIUM_CLI_VERSION} installed."
fi

# ---------------------------------------------------------------------------
# [5/9] Verify Cilium is healthy before enabling Hubble
# ---------------------------------------------------------------------------
echo "[5/${TOTAL_STEPS}] Verifying Cilium CNI health..."
if ! kubectl -n kube-system get pods -l k8s-app=cilium >/dev/null 2>&1; then
    log_error "No Cilium pods found in kube-system. Install Cilium CNI before running this script."
    exit 1
fi

cilium status --wait
log_info "Cilium reports healthy status."

if [ "$RUN_CONN_TEST" = "true" ]; then
    log_warn "Running full connectivity test in namespace '${TEST_NAMESPACE}' — this may take several minutes."
    cilium connectivity test --test-namespace "$TEST_NAMESPACE"
fi

# ---------------------------------------------------------------------------
# [6/9] Enable Hubble with UI and Relay
# ---------------------------------------------------------------------------
echo "[6/${TOTAL_STEPS}] Enabling Hubble (relay + UI) on Cilium..."
cilium hubble enable --ui

log_info "Waiting for Cilium DaemonSet rollout..."
kubectl -n kube-system rollout status daemonset/cilium --timeout=300s
cilium status --wait

# ---------------------------------------------------------------------------
# [7/9] Verify Hubble flow reporting and Relay/UI pods
# ---------------------------------------------------------------------------
echo "[7/${TOTAL_STEPS}] Verifying Hubble is active..."
if ! kubectl -n kube-system exec -it ds/cilium -- cilium status | grep -q "Hubble:.*Ok"; then
    log_error "Hubble does not report Ok status inside the Cilium agent."
    exit 1
fi
log_info "Hubble server active inside Cilium agents."

log_info "Waiting for Hubble Relay pods..."
kubectl -n kube-system wait --for=condition=Ready pod -l k8s-app=hubble-relay --timeout=180s
kubectl -n kube-system get pods -l k8s-app=hubble-relay

log_info "Waiting for Hubble UI pods..."
kubectl -n kube-system wait --for=condition=Ready pod -l k8s-app=hubble-ui --timeout=180s
kubectl -n kube-system get pods -l k8s-app=hubble-ui

# ---------------------------------------------------------------------------
# [8/9] Configure Hubble CLI locally for flow inspection
# ---------------------------------------------------------------------------
echo "[8/${TOTAL_STEPS}] Configuring Hubble CLI..."
if ! command -v hubble >/dev/null 2>&1; then
    ARCH="amd64"
    case "$(uname -m)" in
        x86_64) ARCH="amd64" ;;
        aarch64|arm64) ARCH="arm64" ;;
    esac
    HUBBLE_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/hubble/master/stable.txt)
    WORKDIR=$(mktemp -d)
    TMP_ARCHIVE="${WORKDIR}/hubble-linux-${ARCH}.tar.gz"
    curl -L --fail --remote-name-all \
        "https://github.com/cilium/hubble/releases/download/${HUBBLE_VERSION}/hubble-linux-${ARCH}.tar.gz" \
        -o "$TMP_ARCHIVE"
    $SUDO tar xzvfC "$TMP_ARCHIVE" /usr/local/bin
    $SUDO chmod 755 /usr/local/bin/hubble
    rm -f "$TMP_ARCHIVE"
    TMP_ARCHIVE=""
    log_info "Hubble CLI ${HUBBLE_VERSION} installed."
else
    log_warn "Hubble CLI already installed, skipping."
fi

# Start a background port-forward to Hubble Relay so the local CLI can query flows.
PF_PID_FILE="/tmp/hubble-relay-portforward.pid"
if [ -f "$PF_PID_FILE" ] && kill -0 "$(cat "$PF_PID_FILE")" 2>/dev/null; then
    log_warn "Existing hubble relay port-forward already running."
else
    kubectl -n kube-system port-forward svc/hubble-relay 4245:80 >/tmp/hubble-relay-pf.log 2>&1 &
    echo $! > "$