Implement Cilium Tetragon runtime security for Kubernetes with eBPF monitoring and threat detection

Prepare the system and install dependencies

Update your system and install required packages for Kubernetes and eBPF functionality.

sudo apt update && sudo apt upgrade -y
sudo apt install -y curl wget gnupg2 software-properties-common apt-transport-https ca-certificates

sudo dnf update -y
sudo dnf install -y curl wget gnupg2 yum-utils

Verify Kubernetes cluster readiness

Ensure your Kubernetes cluster is running and you have cluster admin privileges for Tetragon deployment.

kubectl cluster-info
kubectl get nodes
kubectl auth can-i '' '' --all-namespaces

Install Helm for Tetragon deployment

Tetragon uses Helm charts for deployment. Install Helm if not already available in your environment.

curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list

sudo apt update
sudo apt install -y helm

curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh

Add Cilium Helm repository

Add the official Cilium Helm repository to access Tetragon charts and verify the repository.

helm repo add cilium https://helm.cilium.io/
helm repo update
helm search repo cilium/tetragon

Create Tetragon namespace and configuration

Create a dedicated namespace for Tetragon and prepare the configuration values file.

kubectl create namespace kube-system || true
kubectl create namespace tetragon

tetragon:
  enabled: true
  image:
    repository: quay.io/cilium/tetragon
    tag: v1.0.2
  grpc:
    enabled: true
    address: "localhost:54321"
  gops:
    enabled: true
    address: "localhost"
    port: 8080
  prometheus:
    enabled: true
    port: 2112
    serviceMonitor:
      enabled: true
  resources:
    limits:
      cpu: 500m
      memory: 512Mi
    requests:
      cpu: 100m
      memory: 128Mi
  securityContext:
    privileged: false
    runAsUser: 0
    runAsGroup: 0
    capabilities:
      add:
      - SYS_ADMIN
      - SYS_RESOURCE
      - BPF
  export:
    filenames:
    - /var/log/tetragon/tetragon.log
    stdout: true
    mode: json
tetragonOperator:
  enabled: true
  image:
    repository: quay.io/cilium/tetragon-operator
    tag: v1.0.2
  resources:
    limits:
      cpu: 500m
      memory: 128Mi
    requests:
      cpu: 100m
      memory: 64Mi
serviceAccount:
  create: true
  name: "tetragon"

Deploy Tetragon using Helm

Install Tetragon in your Kubernetes cluster with the custom configuration and verify the deployment.

helm install tetragon cilium/tetragon -n tetragon -f /tmp/tetragon-values.yaml
kubectl get pods -n tetragon -w

Install Tetragon CLI tool

Install the tetra CLI for interacting with Tetragon and managing security policies.

curl -L https://github.com/cilium/tetragon/releases/latest/download/tetra-linux-amd64.tar.gz | tar -xz
sudo mv tetra /usr/local/bin/
sudo chmod +x /usr/local/bin/tetra
tetra version

Create basic process monitoring policy

Deploy a TracingPolicy to monitor process execution and system calls across the cluster.

apiVersion: cilium.io/v1alpha1
kind: TracingPolicy
metadata:
  name: process-monitoring
  namespace: tetragon
spec:
  kprobes:
  - call: "sys_execve"
    syscall: true
    args:
    - index: 0
      type: "string"
    - index: 1
      type: "char_buf"
      sizeArgIndex: 2
    - index: 2
      type: "char_buf"
      sizeArgIndex: 3
    selectors:
    - matchArgs:
      - index: 0
        operator: "Equal"
        values:
        - "/bin/bash"
        - "/bin/sh"
        - "/usr/bin/curl"
        - "/usr/bin/wget"
      matchActions:
      - action: Post
  - call: "sys_openat"
    syscall: true
    args:
    - index: 0
      type: "int"
    - index: 1
      type: "string"
    - index: 2
      type: "int"
    selectors:
    - matchArgs:
      - index: 1
        operator: "Prefix"
        values:
        - "/etc/passwd"
        - "/etc/shadow"
        - "/root"
      matchActions:
      - action: Post

kubectl apply -f /tmp/process-monitoring.yaml

Create network security policy

Configure network monitoring to detect suspicious connections and data exfiltration attempts.

apiVersion: cilium.io/v1alpha1
kind: TracingPolicy
metadata:
  name: network-monitoring
  namespace: tetragon
spec:
  kprobes:
  - call: "tcp_connect"
    syscall: false
    args:
    - index: 0
      type: "sock"
    selectors:
    - matchArgs:
      - index: 0
        operator: "DAddr"
        values:
        - "0.0.0.0/0"
      matchActions:
      - action: Post
  - call: "sys_socket"
    syscall: true
    args:
    - index: 0
      type: "int"
    - index: 1
      type: "int"
    - index: 2
      type: "int"
    selectors:
    - matchArgs:
      - index: 0
        operator: "Equal"
        values:
        - "2"  # AF_INET
        - "10" # AF_INET6
      matchActions:
      - action: Post

kubectl apply -f /tmp/network-monitoring.yaml

Create file system security policy

Monitor sensitive file access and detect potential privilege escalation attempts.

apiVersion: cilium.io/v1alpha1
kind: TracingPolicy
metadata:
  name: filesystem-monitoring
  namespace: tetragon
spec:
  kprobes:
  - call: "security_file_permission"
    syscall: false
    args:
    - index: 0
      type: "file"
    - index: 1
      type: "int"
    selectors:
    - matchArgs:
      - index: 0
        operator: "Prefix"
        values:
        - "/etc/"
        - "/var/run/secrets/"
        - "/root/"
        - "/home/"
      matchActions:
      - action: Post
  - call: "sys_setuid"
    syscall: true
    args:
    - index: 0
      type: "int"
    selectors:
    - matchArgs:
      - index: 0
        operator: "Equal"
        values:
        - "0" # root UID
      matchActions:
      - action: Post
      - action: Sigkill # Optional: kill processes attempting to escalate to root

kubectl apply -f /tmp/filesystem-monitoring.yaml

Configure Grafana dashboard for Tetragon

Create a custom Grafana dashboard to visualize Tetragon security events and metrics.

{
  "dashboard": {
    "id": null,
    "title": "Tetragon Runtime Security",
    "tags": ["tetragon", "security", "kubernetes"],
    "timezone": "browser",
    "panels": [
      {
        "id": 1,
        "title": "Process Executions",
        "type": "stat",
        "targets": [
          {
            "expr": "rate(tetragon_events_total{event_type=\"PROCESS_EXEC\"}[5m])",
            "refId": "A"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
      },
      {
        "id": 2,
        "title": "Network Connections",
        "type": "stat",
        "targets": [
          {
            "expr": "rate(tetragon_events_total{event_type=\"PROCESS_KPROBE\"}[5m])",
            "refId": "B"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
      },
      {
        "id": 3,
        "title": "Security Events Timeline",
        "type": "logs",
        "targets": [
          {
            "expr": "{job=\"tetragon\"} | json | line_format \"{{.time}} {{.process.binary}} {{.event_type}}\"",
            "refId": "C"
          }
        ],
        "gridPos": {"h": 12, "w": 24, "x": 0, "y": 8}
      }
    ],
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "refresh": "5s"
  }
}

Configure log aggregation with Fluentd

Set up log collection for Tetragon events using Fluentd for centralized analysis.

apiVersion: v1
kind: ConfigMap
metadata:
  name: tetragon-fluentd-config
  namespace: tetragon
data:
  fluent.conf: |
    
      @type tail
      path /var/log/tetragon/tetragon.log
      pos_file /var/log/fluentd-tetragon.log.pos
      tag tetragon.security
      
        @type json
        time_key time
        time_format %Y-%m-%dT%H:%M:%S.%LZ
      
    
    
    
      @type record_transformer
      
        severity ${record["process"] ? "INFO" : "WARNING"}
        cluster_name kubernetes-production
        source tetragon
      
    
    
    
      @type forward
      
        name log-server
        host elasticsearch.monitoring.svc.cluster.local
        port 24224
      
      
        @type file
        path /var/log/fluentd-buffers/tetragon
        flush_mode interval
        flush_interval 10s
      
    
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: tetragon-fluentd
  namespace: tetragon
spec:
  selector:
    matchLabels:
      name: tetragon-fluentd
  template:
    metadata:
      labels:
        name: tetragon-fluentd
    spec:
      containers:
      - name: fluentd
        image: fluent/fluentd:v1.16-debian-1
        volumeMounts:
        - name: tetragon-logs
          mountPath: /var/log/tetragon
          readOnly: true
        - name: fluentd-config
          mountPath: /fluentd/etc
        resources:
          limits:
            memory: 200Mi
            cpu: 100m
          requests:
            memory: 100Mi
            cpu: 50m
      volumes:
      - name: tetragon-logs
        hostPath:
          path: /var/log/tetragon
      - name: fluentd-config
        configMap:
          name: tetragon-fluentd-config

kubectl apply -f /tmp/tetragon-fluentd.yaml

Create alerting rules for security events

Configure Prometheus alerting rules for critical security events detected by Tetragon.

apiVersion: v1
kind: ConfigMap
metadata:
  name: tetragon-alerts
  namespace: monitoring
data:
  tetragon.rules: |
    groups:
    - name: tetragon.security
      rules:
      - alert: TetragonHighProcessExecution
        expr: rate(tetragon_events_total{event_type="PROCESS_EXEC"}[5m]) > 10
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High process execution rate detected"
          description: "Process execution rate is {{ $value }} events/sec, above threshold"
      
      - alert: TetragonPrivilegeEscalation
        expr: increase(tetragon_events_total{event_type="PROCESS_KPROBE",function_name="sys_setuid"}[5m]) > 0
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: "Potential privilege escalation detected"
          description: "Process attempted to change UID, potential privilege escalation"
      
      - alert: TetragonSensitiveFileAccess
        expr: increase(tetragon_events_total{event_type="PROCESS_KPROBE",function_name="security_file_permission"}[5m]) > 5
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Suspicious file access pattern"
          description: "Multiple sensitive file access attempts detected"
      
      - alert: TetragonAgentDown
        expr: up{job="tetragon"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Tetragon agent is down"
          description: "Tetragon security monitoring agent is not responding"

kubectl apply -f /tmp/tetragon-alerts.yaml

Deploy test workload for security monitoring

Create a test pod to validate that Tetragon is detecting and logging security events.

apiVersion: v1
kind: Pod
metadata:
  name: security-test-pod
  namespace: default
  labels:
    app: security-test
spec:
  containers:
  - name: test-container
    image: ubuntu:22.04
    command: ["/bin/sleep", "3600"]
    securityContext:
      runAsUser: 1000
      runAsGroup: 1000
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      readOnlyRootFilesystem: false
  restartPolicy: Never

kubectl apply -f /tmp/test-pod.yaml
kubectl wait --for=condition=Ready pod/security-test-pod --timeout=60s

Generate security events for testing

Execute commands inside the test pod to trigger Tetragon monitoring and generate sample events.

kubectl exec -it security-test-pod -- /bin/bash -c "whoami && id"
kubectl exec -it security-test-pod -- /bin/bash -c "ls -la /etc/passwd"
kubectl exec -it security-test-pod -- /bin/bash -c "curl -I https://example.com || true"

Observe Tetragon events

Use the tetra CLI to view real-time security events generated by the test activities.

kubectl port-forward -n tetragon svc/tetragon 54321:54321 &
tetra getevents --server localhost:54321 | head -20
kill %1