Set up Linux performance monitoring with perf tools and flame graphs

#!/usr/bin/env bash

set -euo pipefail

# Linux Performance Monitoring with perf tools and flame graphs installer
# Supports Ubuntu, Debian, AlmaLinux, Rocky Linux, CentOS, RHEL, and Amazon Linux

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

INSTALL_DIR="/opt/FlameGraph"
PERF_DATA_DIR="/tmp/perf-data"
SCRIPT_NAME="perf-profile.sh"

cleanup() {
    echo -e "${RED}[ERROR]${NC} Installation failed. Cleaning up..."
    sudo rm -rf "$INSTALL_DIR" 2>/dev/null || true
    sudo rm -rf "$PERF_DATA_DIR" 2>/dev/null || true
    sudo rm -f "/usr/local/bin/$SCRIPT_NAME" 2>/dev/null || true
}

trap cleanup ERR

usage() {
    echo "Usage: $0"
    echo "Install Linux performance monitoring with perf tools and flame graphs"
    exit 1
}

log_step() {
    echo -e "${GREEN}[$1]${NC} $2"
}

log_warn() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Check if running as root or with sudo
if [[ $EUID -eq 0 ]]; then
    log_error "Please run this script as a regular user with sudo privileges, not as root"
    exit 1
fi

if ! sudo -n true 2>/dev/null; then
    log_error "This script requires sudo privileges"
    exit 1
fi

# Detect distribution
if [ ! -f /etc/os-release ]; then
    log_error "Cannot detect Linux distribution"
    exit 1
fi

. /etc/os-release

case "$ID" in
    ubuntu|debian)
        PKG_MGR="apt"
        PKG_UPDATE="apt update && apt upgrade -y"
        PKG_INSTALL="apt install -y"
        PERF_PACKAGES="linux-tools-common linux-tools-generic linux-tools-$(uname -r) libc6-dbg build-essential git perl"
        DEBUG_PACKAGES="libc6-dbg"
        ;;
    almalinux|rocky|centos|rhel|ol|fedora)
        PKG_MGR="dnf"
        PKG_UPDATE="dnf update -y"
        PKG_INSTALL="dnf install -y"
        PERF_PACKAGES="perf kernel-debuginfo-$(uname -r) glibc-debuginfo make gcc git perl which"
        DEBUG_PACKAGES="glibc-debuginfo kernel-debuginfo"
        ;;
    amzn)
        PKG_MGR="yum"
        PKG_UPDATE="yum update -y"
        PKG_INSTALL="yum install -y"
        PERF_PACKAGES="perf glibc-debuginfo make gcc git perl which"
        DEBUG_PACKAGES="glibc-debuginfo"
        ;;
    *)
        log_error "Unsupported distribution: $ID"
        exit 1
        ;;
esac

log_step "1/8" "Updating system packages"
sudo $PKG_UPDATE

log_step "2/8" "Installing perf tools and dependencies"
if ! sudo $PKG_INSTALL $PERF_PACKAGES; then
    log_warn "Some perf packages may not be available, continuing with available ones"
fi

log_step "3/8" "Configuring kernel symbol access"
echo 'kernel.perf_event_paranoid = 1' | sudo tee -a /etc/sysctl.conf >/dev/null
echo 'kernel.kptr_restrict = 0' | sudo tee -a /etc/sysctl.conf >/dev/null
sudo sysctl -p >/dev/null

log_step "4/8" "Installing debug information packages"
case "$ID" in
    ubuntu|debian)
        if [ "$ID" = "ubuntu" ]; then
            echo "deb http://ddebs.ubuntu.com $(lsb_release -cs) main restricted universe multiverse" | \
                sudo tee /etc/apt/sources.list.d/ddebs.list >/dev/null
            echo "deb http://ddebs.ubuntu.com $(lsb_release -cs)-updates main restricted universe multiverse" | \
                sudo tee -a /etc/apt/sources.list.d/ddebs.list >/dev/null
            sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 428D7C01 C8CAB6595FDFF622 >/dev/null 2>&1 || true
            sudo apt update >/dev/null 2>&1 || true
        fi
        ;;
    *)
        if command -v dnf >/dev/null 2>&1; then
            sudo dnf debuginfo-install -y kernel glibc >/dev/null 2>&1 || log_warn "Debug info packages may not be available"
        fi
        ;;
esac

log_step "5/8" "Cloning FlameGraph toolkit"
sudo rm -rf "$INSTALL_DIR"
sudo git clone https://github.com/brendangregg/FlameGraph.git "$INSTALL_DIR" >/dev/null 2>&1
sudo chown -R root:root "$INSTALL_DIR"
sudo chmod -R 755 "$INSTALL_DIR"

log_step "6/8" "Creating perf data directory"
sudo mkdir -p "$PERF_DATA_DIR"
sudo chmod 755 "$PERF_DATA_DIR"

log_step "7/8" "Creating performance profiling script"
sudo tee "/usr/local/bin/$SCRIPT_NAME" >/dev/null <<'EOF'
#!/bin/bash

# Perf profiling script with flame graph generation
# Usage: ./perf-profile.sh [duration] [process_name_or_pid]

set -euo pipefail

DURATION=${1:-30}
TARGET=${2:-""}
OUTPUT_DIR="/tmp/perf-data/$(date +%Y%m%d_%H%M%S)"
DATA_FILE="$OUTPUT_DIR/perf.data"
FLAMEGRAPH_FILE="$OUTPUT_DIR/flamegraph.svg"
FLAMEGRAPH_PATH="/opt/FlameGraph"

if [ ! -d "$FLAMEGRAPH_PATH" ]; then
    echo "Error: FlameGraph toolkit not found at $FLAMEGRAPH_PATH"
    exit 1
fi

mkdir -p "$OUTPUT_DIR"

echo "Starting perf recording for $DURATION seconds..."
echo "Output directory: $OUTPUT_DIR"

if [ -n "$TARGET" ]; then
    # Profile specific process
    if [[ "$TARGET" =~ ^[0-9]+$ ]]; then
        # PID provided
        perf record -F 997 -g --call-graph=dwarf -p "$TARGET" \
                   -o "$DATA_FILE" sleep "$DURATION"
    else
        # Process name provided
        PID=$(pgrep -f "$TARGET" | head -1)
        if [ -z "$PID" ]; then
            echo "Error: Process '$TARGET' not found"
            exit 1
        fi
        perf record -F 997 -g --call-graph=dwarf -p "$PID" \
                   -o "$DATA_FILE" sleep "$DURATION"
    fi
else
    # System-wide profiling
    sudo perf record -F 997 -g --call-graph=dwarf -a \
                     -o "$DATA_FILE" sleep "$DURATION"
fi

echo "Generating flame graph..."
perf script -i "$DATA_FILE" | "$FLAMEGRAPH_PATH/stackcollapse-perf.pl" | \
    "$FLAMEGRAPH_PATH/flamegraph.pl" > "$FLAMEGRAPH_FILE"

echo "Performance analysis complete!"
echo "Perf data: $DATA_FILE"
echo "Flame graph: $FLAMEGRAPH_FILE"
echo "Report: perf report -i $DATA_FILE"
EOF

sudo chmod 755 "/usr/local/bin/$SCRIPT_NAME"

log_step "8/8" "Configuring system limits"
echo '* soft nofile 65536
* hard nofile 65536
* soft nproc 32768
* hard nproc 32768' | sudo tee /etc/security/limits.d/99-perf.conf >/dev/null

echo -e "\n${GREEN}Installation completed successfully!${NC}"
echo -e "\nUsage examples:"
echo -e "  ${YELLOW}$SCRIPT_NAME${NC}                    # System-wide profiling for 30 seconds"
echo -e "  ${YELLOW}$SCRIPT_NAME 60${NC}                 # System-wide profiling for 60 seconds"
echo -e "  ${YELLOW}$SCRIPT_NAME 30 nginx${NC}           # Profile nginx process for 30 seconds"
echo -e "  ${YELLOW}$SCRIPT_NAME 30 1234${NC}            # Profile PID 1234 for 30 seconds"

echo -e "\nManual commands:"
echo -e "  ${YELLOW}perf stat -e cycles,instructions,cache-references,cache-misses sleep 3${NC}"
echo -e "  ${YELLOW}perf top${NC}                        # Live performance monitoring"

echo -e "\nFlameGraph tools are available in: ${YELLOW}$INSTALL_DIR${NC}"
echo -e "Add to PATH: ${YELLOW}export PATH=\"$INSTALL_DIR:\$PATH\"${NC}"

log_step "VERIFY" "Testing basic perf functionality"
if command -v perf >/dev/null 2>&1; then
    echo -e "${GREEN}✓${NC} perf command available"
    if perf stat -e cycles,instructions sleep 1 >/dev/null 2>&1; then
        echo -e "${GREEN}✓${NC} perf profiling functional"
    else
        log_warn "perf profiling may require additional permissions"
    fi
else
    log_error "perf command not found after installation"
    exit 1
fi

if [ -f "$INSTALL_DIR/flamegraph.pl" ]; then
    echo -e "${GREEN}✓${NC} FlameGraph toolkit installed"
else
    log_error "FlameGraph toolkit installation failed"
    exit 1
fi

echo -e "\n${GREEN}All checks passed! Performance monitoring is ready.${NC}"