Configure systemd service resource limits and security isolation

Verify cgroups v2 support

Check that your system has cgroups v2 enabled, which is required for modern systemd resource limits.

cat /proc/filesystems | grep cgroup
cat /sys/fs/cgroup/cgroup.controllers

Create a test service for demonstration

Create a simple service to demonstrate resource limits and security features.

[Unit]
Description=Demo Application for Resource Limits
After=network.target

[Service]
Type=simple
User=nobody
Group=nogroup
ExecStart=/bin/bash -c 'while true; do echo "Demo app running"; sleep 10; done'
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

Configure memory limits

Set memory limits to prevent the service from consuming excessive RAM and causing system instability.

sudo mkdir -p /etc/systemd/system/demo-app.service.d
sudo tee /etc/systemd/system/demo-app.service.d/resource-limits.conf > /dev/null << 'EOF'
[Service]
# Memory limits
MemoryMax=512M
MemoryHigh=400M
MemorySwapMax=0

# Prevent OOM killer from killing other processes
OOMPolicy=stop
EOF

Configure CPU limits

Set CPU limits to prevent the service from monopolizing processor resources.

[Service]
# CPU limits
CPUQuota=50%
CPUWeight=100

# Nice priority (higher number = lower priority)
Nice=10

Configure I/O limits

Set I/O limits to prevent the service from saturating disk bandwidth and affecting other processes.

[Service]
# I/O limits (bytes per second)
IOReadBandwidthMax=/ 10M
IOWriteBandwidthMax=/ 5M

# I/O operations per second limits
IOReadIOPSMax=/ 1000
IOWriteIOPSMax=/ 500

# I/O scheduling class (0=none, 1=realtime, 2=best-effort, 3=idle)
IOSchedulingClass=2
IOSchedulingPriority=4

Configure basic security isolation

Apply security sandboxing to limit what the service can access on the system.

[Service]
# Filesystem access restrictions
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log/demo-app
ReadOnlyPaths=/etc/demo-app

# Network restrictions
PrivateNetwork=false
RestrictAddressFamilies=AF_INET AF_INET6

# Process restrictions
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true

Configure advanced security features

Enable additional security features including capability restrictions and system call filtering.

[Service]
# Capability restrictions
CapabilityBoundingSet=
AmbientCapabilities=

# System call filtering
SystemCallFilter=@system-service
SystemCallArchitectures=native
SystemCallErrorNumber=EPERM

# Memory and execution restrictions
MemoryDenyWriteExecute=true
RestrictRealtime=true
RestrictSUIDSGID=true
RestrictNamespaces=true

# Hostname and clock restrictions
ProtectHostname=true
ProtectClock=true

Create required directories and set permissions

Create necessary directories for the service with proper ownership and permissions.

sudo mkdir -p /var/log/demo-app /etc/demo-app
sudo chown nobody:nogroup /var/log/demo-app
sudo chmod 755 /var/log/demo-app
sudo chmod 755 /etc/demo-app

Reload systemd and start the service

Reload systemd configuration to apply the new limits and start the service.

sudo systemctl daemon-reload
sudo systemctl enable demo-app.service
sudo systemctl start demo-app.service

Configure resource limits for existing services

Apply resource limits to critical system services like nginx or database services.

sudo mkdir -p /etc/systemd/system/nginx.service.d
sudo tee /etc/systemd/system/nginx.service.d/limits.conf > /dev/null << 'EOF'
[Service]
# Memory limits for nginx
MemoryMax=1G
MemoryHigh=800M

# CPU limits
CPUQuota=200%

# I/O limits
IOReadBandwidthMax=/ 50M
IOWriteBandwidthMax=/ 20M

# Security hardening
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ReadWritePaths=/var/log/nginx /var/lib/nginx
EOF

Apply limits and restart services

Reload systemd and restart services to apply the new resource limits.

sudo systemctl daemon-reload
sudo systemctl restart nginx
sudo systemctl restart demo-app

Monitor service resource usage

Use systemd tools to monitor how much resources your services are consuming.

# Check current resource usage
sudo systemctl status demo-app
sudo systemd-cgtop

# Detailed cgroup information
sudo systemctl show demo-app --property=MemoryCurrent,CPUUsageNSec,IOReadBytes,IOWriteBytes

# Monitor resource limits in real-time
sudo systemd-run --scope --slice=demo-app.slice systemd-cgtop

Check resource limit violations

Monitor the journal for resource limit violations and OOM events.

# Check for memory limit violations
sudo journalctl -u demo-app | grep -i "memory\|oom"

# Check for CPU throttling
sudo journalctl -u demo-app | grep -i "cpu\|throttl"

# Monitor all cgroup-related events
sudo journalctl -f | grep cgroup

Create monitoring scripts

Set up automated monitoring scripts to track resource usage and send alerts.

#!/bin/bash

SERVICE_NAME="demo-app"
MEMORY_THRESHOLD=80
CPU_THRESHOLD=90

# Get memory usage percentage
MEMORY_LIMIT=$(systemctl show $SERVICE_NAME --property=MemoryMax --value)
MEMORY_CURRENT=$(systemctl show $SERVICE_NAME --property=MemoryCurrent --value)

if [ "$MEMORY_LIMIT" != "infinity" ] && [ "$MEMORY_CURRENT" -gt 0 ]; then
    MEMORY_PERCENT=$((MEMORY_CURRENT * 100 / MEMORY_LIMIT))
    if [ "$MEMORY_PERCENT" -gt "$MEMORY_THRESHOLD" ]; then
        echo "WARNING: $SERVICE_NAME memory usage at ${MEMORY_PERCENT}%"
        logger -t resource-monitor "$SERVICE_NAME memory usage high: ${MEMORY_PERCENT}%"
    fi
fi

# Check for recent OOM events
if journalctl -u $SERVICE_NAME --since="5 minutes ago" | grep -q "oom-kill"; then
    echo "CRITICAL: $SERVICE_NAME was killed by OOM killer"
    logger -t resource-monitor "$SERVICE_NAME OOM kill detected"
fi

echo "Resource monitoring complete for $SERVICE_NAME"

sudo chmod 755 /usr/local/bin/monitor-service-resources.sh

Set up automated monitoring

Create a systemd timer to run resource monitoring automatically.

[Unit]
Description=Service Resource Monitor

[Service]
Type=oneshot
ExecStart=/usr/local/bin/monitor-service-resources.sh
User=root

[Unit]
Description=Run resource monitoring every 5 minutes
Requires=resource-monitor.service

[Timer]
OnCalendar=*:0/5
Persistent=true

[Install]
WantedBy=timers.target

sudo systemctl enable --now resource-monitor.timer