Configure intrusion detection with OSSEC and fail2ban integration

Update system packages

Start by updating your package manager to ensure you get the latest security updates.

sudo apt update && sudo apt upgrade -y

sudo dnf update -y

Install OSSEC dependencies

Install the required packages for building OSSEC from source and running the system.

sudo apt install -y build-essential gcc make unzip sendmail inotify-tools wget libevent-dev libssl-dev libz-dev

sudo dnf install -y gcc gcc-c++ make unzip sendmail inotify-tools wget libevent-devel openssl-devel zlib-devel

Download and install OSSEC

Download the latest OSSEC release and compile it for your system.

cd /tmp
wget https://github.com/ossec/ossec-hids/archive/v3.7.0.tar.gz
tar -xzf v3.7.0.tar.gz
cd ossec-hids-3.7.0

Configure OSSEC installation

Run the interactive installation script and select server installation for local monitoring.

sudo ./install.sh

When prompted, select these options:

• Installation type: local
• Email notification: y (enter your email)
• SMTP server: localhost
• Firewall response: y
• System integrity check: y
• Rootkit detection: y
• Active response: y

Install fail2ban

Install fail2ban which will handle the automatic IP blocking based on OSSEC alerts.

sudo apt install -y fail2ban

sudo dnf install -y epel-release
sudo dnf install -y fail2ban

Configure OSSEC for fail2ban integration

Modify OSSEC configuration to enable log output that fail2ban can parse.

  yes
  localhost
  ossec@example.com
  admin@example.com
  yes
  no



  rules_config.xml
  pam_rules.xml
  sshd_rules.xml
  telnetd_rules.xml
  syslog_rules.xml
  arpwatch_rules.xml
  symantec-av_rules.xml
  symantec-ws_rules.xml
  pix_rules.xml
  named_rules.xml
  smbd_rules.xml
  vsftpd_rules.xml
  pure-ftpd_rules.xml
  proftpd_rules.xml
  ms_ftpd_rules.xml
  ftpd_rules.xml
  hordeimp_rules.xml
  roundcube_rules.xml
  wordpress_rules.xml
  cimserver_rules.xml
  vpopmail_rules.xml
  vmpop3d_rules.xml
  courier_rules.xml
  web_rules.xml
  web_appsec_rules.xml
  apache_rules.xml
  nginx_rules.xml
  php_rules.xml
  mysql_rules.xml
  postgresql_rules.xml
  ids_rules.xml
  squid_rules.xml
  firewall_rules.xml
  cisco-ios_rules.xml
  netscreenfw_rules.xml
  sonicwall_rules.xml
  postfix_rules.xml
  sendmail_rules.xml
  imapd_rules.xml
  mailscanner_rules.xml
  dovecot_rules.xml
  ms-exchange_rules.xml
  racoon_rules.xml
  vpn_concentrator_rules.xml
  spamd_rules.xml
  msauth_rules.xml
  mcafee_av_rules.xml
  trend-osce_rules.xml
  ms-se_rules.xml
  zeus_rules.xml
  solaris_bsm_rules.xml
  vmware_rules.xml
  ms_dhcp_rules.xml
  asterisk_rules.xml
  ossec_rules.xml
  attack_rules.xml
  local_rules.xml



  79200
  /etc,/usr/bin,/usr/sbin
  /bin,/sbin,/boot
  /etc/mtab
  /etc/hosts.deny
  /etc/mail/statistics
  /etc/random-seed
  /etc/random.seed
  /etc/adjtime
  /etc/httpd/logs
  /etc/utmpx
  /etc/wtmpx
  /etc/cups/certs
  /etc/dumpdates
  /etc/svc/volatile



  /var/ossec/etc/shared/rootkit_files.txt
  /var/ossec/etc/shared/rootkit_trojans.txt
  /var/ossec/etc/shared/system_audit_rcl.txt
  /var/ossec/etc/shared/system_audit_ssh.txt
  /var/ossec/etc/shared/cis_debian_linux_rcl.txt



  syslog
  /var/log/messages



  syslog
  /var/log/secure



  syslog
  /var/log/auth.log



  apache
  /var/log/apache2/access.log



  apache
  /var/log/apache2/error.log



  no
  /var/ossec/etc/wpk_root.pem

Create custom OSSEC rules for fail2ban integration

Add custom rules to detect specific attack patterns and generate alerts that fail2ban can process.

  
  
  
    5716
    sshd: brute force trying to get access to the system.
    
    authentication_failures,pci_dss_10.2.4,pci_dss_10.2.5,
  
  
  
  
    31151
    HTTP brute force attack detected.
    
    attack,pci_dss_10.2.4,pci_dss_10.2.5,
  
  
  
  
    5503
    Multiple authentication failures from same source.
    
    authentication_failures,pci_dss_10.2.4,pci_dss_10.2.5,
  
  
  
  
    4151
    Port scan detected from same source.
    
    recon,pci_dss_10.6.1,
  
  

Configure fail2ban jail for OSSEC

Create a custom jail configuration that monitors OSSEC alerts and blocks offending IPs.

[DEFAULT]
bantime = 3600
findtime = 600
maxretry = 3
banaction = iptables-multiport
protocol = tcp
chain = INPUT
action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
           %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"]
action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
            %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
action = %(action_)s

[sshd]
enabled = true
port = ssh
filter = sshd
logpath = /var/log/auth.log
maxretry = 3
bantime = 3600

[apache-auth]
enabled = true
port = http,https
filter = apache-auth
logpath = /var/log/apache2/*error.log
maxretry = 6
bantime = 3600

[apache-badbots]
enabled = true
port = http,https
filter = apache-badbots
logpath = /var/log/apache2/*error.log
maxretry = 2
bantime = 86400

[apache-noscript]
enabled = true
port = http,https
filter = apache-noscript
logpath = /var/log/apache2/*error.log
maxretry = 6
bantime = 3600

[apache-overflows]
enabled = true
port = http,https
filter = apache-overflows
logpath = /var/log/apache2/*error.log
maxretry = 2
bantime = 3600

[ossec]
enabled = true
filter = ossec
logpath = /var/ossec/logs/alerts/alerts.log
maxretry = 1
bantime = 86400
port = all
protocol = all
banaction = iptables-allports

Create OSSEC filter for fail2ban

Create a custom filter that parses OSSEC alert logs to extract IP addresses for blocking.

[INCLUDES]

[Definition]
failregex = .\[\d+\]\s+\(\d+\).Src IP: .Rule: \d+ fired.Level: (?:1[0-5]|[6-9]).*
            .\[\d+\]\s+\(\d+\).srcip: ;.rule: \d+; level: (?:1[0-5]|[6-9]);.
            .\[\d+\]\s+\(\d+\).Attack Signature.Src IP: .
            .\[\d+\]\s+\(\d+\).srcip: .rule: 100001.
            .\[\d+\]\s+\(\d+\).srcip: .rule: 100002.
            .\[\d+\]\s+\(\d+\).srcip: .rule: 100003.
            .\[\d+\]\s+\(\d+\).srcip: .rule: 100004.

ignoreregex = 

[Init]
journealmatch = _SYSTEMD_UNIT=ossec.service

Configure OSSEC active response integration

Create a custom active response script that interfaces with fail2ban for coordinated blocking.

#!/bin/bash
fail2ban-ban.sh - OSSEC active response script for fail2ban integration
Author: Binadit Infrastructure Team

LOGFILE="/var/ossec/logs/active-responses.log"
FAIL2BAN_CLIENT="/usr/bin/fail2ban-client"

Function to log messages
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - fail2ban-ban.sh: $1" >> $LOGFILE
}

Read the action and IP from OSSEC
read ACTION
read USER
read IP
read ALERTID
read RULEID
read AGENT
read FILENAME

Extract the IP address if it's in CIDR format
IP=$(echo $IP | cut -d'/' -f1)

Validate IP address
if [[ ! $IP =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
    log "Invalid IP address: $IP"
    exit 1
fi

Skip private IP addresses
if [[ $IP =~ ^10\. ]] || [[ $IP =~ ^172\.(1[6-9]|2[0-9]|3[0-1])\. ]] || [[ $IP =~ ^192\.168\. ]]; then
    log "Skipping private IP: $IP"
    exit 0
fi

case $ACTION in
    add)
        log "Banning IP $IP (Rule ID: $RULEID)"
        # Ban the IP using fail2ban
        $FAIL2BAN_CLIENT set ossec banip $IP
        if [ $? -eq 0 ]; then
            log "Successfully banned IP $IP"
        else
            log "Failed to ban IP $IP"
        fi
        ;;
    delete)
        log "Unbanning IP $IP (Rule ID: $RULEID)"
        # Unban the IP using fail2ban
        $FAIL2BAN_CLIENT set ossec unbanip $IP
        if [ $? -eq 0 ]; then
            log "Successfully unbanned IP $IP"
        else
            log "Failed to unban IP $IP"
        fi
        ;;
    *)
        log "Unknown action: $ACTION"
        ;;
esac

exit 0

Set permissions for active response script

Set proper permissions for the active response script so OSSEC can execute it.

sudo chmod 750 /var/ossec/active-response/bin/fail2ban-ban.sh
sudo chown root:ossec /var/ossec/active-response/bin/fail2ban-ban.sh

Add active response configuration to OSSEC

Configure OSSEC to use the custom active response script for high-level alerts.

  fail2ban-ban
  fail2ban-ban.sh
  srcip
  yes



  fail2ban-ban
  local
  10
  3600

Start and enable services

Enable and start both OSSEC and fail2ban services to begin monitoring.

sudo systemctl enable fail2ban
sudo systemctl start fail2ban
sudo /var/ossec/bin/ossec-control start
sudo systemctl status fail2ban

Configure log monitoring integration

Set up centralized logging to ensure both systems can access and monitor the same log files effectively. This configuration was referenced in our centralized logging tutorial.

# OSSEC log integration
Send OSSEC alerts to a separate log file
if $programname startswith 'ossec' then /var/log/ossec-alerts.log
& stop

Ensure fail2ban can read OSSEC logs
$FileCreateMode 0644
$DirCreateMode 0755

Restart rsyslog service

Restart rsyslog to apply the new logging configuration.

sudo systemctl restart rsyslog

Set up email notifications

Configure email alerts for both OSSEC and fail2ban to notify administrators of security events.

[Definition]
actionstart = printf %%b "Subject: [Fail2ban] : started on 
Date: LC_ALL=C date +'%%a, %%d %%h %%Y %%T %%z'
From: 
To: 

Hi,

The jail  has been started successfully.

Regards,
Fail2ban" | /usr/sbin/sendmail -f  

actionstop = printf %%b "Subject: [Fail2ban] : stopped on 
Date: LC_ALL=C date +'%%a, %%d %%h %%Y %%T %%z'
From: 
To: 

Hi,

The jail  has been stopped.

Regards,
Fail2ban" | /usr/sbin/sendmail -f  

actionban = printf %%b "Subject: [Fail2ban] : banned  from 
Date: LC_ALL=C date +'%%a, %%d %%h %%Y %%T %%z'
From: 
To: 

Hi,

The IP  has just been banned by Fail2ban after
 attempts against .

Here is more information about :
/usr/bin/whois  || echo "Whois lookup failed"

Regards,
Fail2ban" | /usr/sbin/sendmail -f  

actionunban = printf %%b "Subject: [Fail2ban] : unbanned  from 
Date: LC_ALL=C date +'%%a, %%d %%h %%Y %%T %%z'
From: 
To: 

Hi,

The IP  has just been unbanned by Fail2ban.

Regards,
Fail2ban" | /usr/sbin/sendmail -f  

Create monitoring dashboard script

Create a script to monitor the status of both systems and generate periodic reports.

#!/bin/bash
security-monitor.sh - Monitor OSSEC and fail2ban status
Author: Binadit Infrastructure Team

LOGFILE="/var/log/security-monitor.log"
EMAIL="admin@example.com"

Function to log messages
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a $LOGFILE
}

Check OSSEC status
check_ossec() {
    if pgrep -f "ossec-syscheckd" > /dev/null; then
        log "OSSEC: Running"
        # Get recent alerts count
        RECENT_ALERTS=$(tail -n 100 /var/ossec/logs/alerts/alerts.log | grep "$(date '+%Y %b %d')" | wc -l)
        log "OSSEC: $RECENT_ALERTS alerts today"
    else
        log "OSSEC: Not running - ALERT!"
        echo "OSSEC service is down on $(hostname)" | mail -s "OSSEC Alert" $EMAIL
    fi
}

Check fail2ban status
check_fail2ban() {
    if systemctl is-active --quiet fail2ban; then
        log "Fail2ban: Running"
        # Get banned IPs count
        BANNED_IPS=$(/usr/bin/fail2ban-client status | grep "Currently banned" | awk '{print $4}')
        log "Fail2ban: $BANNED_IPS IPs currently banned"
        
        # List active jails
        ACTIVE_JAILS=$(/usr/bin/fail2ban-client status | grep "Jail list" | cut -d: -f2 | xargs)
        log "Fail2ban: Active jails: $ACTIVE_JAILS"
    else
        log "Fail2ban: Not running - ALERT!"
        echo "Fail2ban service is down on $(hostname)" | mail -s "Fail2ban Alert" $EMAIL
    fi
}

Check disk space for logs
check_disk_space() {
    DISK_USAGE=$(df /var/log | tail -1 | awk '{print $5}' | sed 's/%//')
    if [ $DISK_USAGE -gt 80 ]; then
        log "Disk space warning: /var/log is ${DISK_USAGE}% full"
        echo "Log partition is ${DISK_USAGE}% full on $(hostname)" | mail -s "Disk Space Alert" $EMAIL
    else
        log "Disk space: /var/log is ${DISK_USAGE}% full"
    fi
}

Main monitoring function
main() {
    log "=== Security Monitor Report ==="
    check_ossec
    check_fail2ban
    check_disk_space
    log "=== End Report ==="
    echo "" >> $LOGFILE
}

main

Set up monitoring script permissions and cron job

Make the monitoring script executable and schedule it to run every hour.

sudo chmod 755 /usr/local/bin/security-monitor.sh
sudo chown root:root /usr/local/bin/security-monitor.sh

Add cron job for hourly monitoring
echo "0     /usr/local/bin/security-monitor.sh" | sudo crontab -

Configure automated response escalation

Set up different response levels based on threat severity and implement escalation policies.

  fail2ban-ban
  local
  10
  3600




  fail2ban-ban
  local
  12
  86400




  fail2ban-ban
  local
  15
  604800

Create threat intelligence integration

Set up a script that cross-references detected IPs with threat intelligence feeds for enhanced detection. You can extend this further using techniques from our OSSEC active response tutorial.

#!/bin/bash
threat-intel.sh - Threat intelligence integration for OSSEC and fail2ban
Author: Binadit Infrastructure Team

THREAT_DB="/var/ossec/threat-intelligence.db"
LOGFILE="/var/log/threat-intel.log"
ALERT_EMAIL="admin@example.com"

Function to log messages
log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a $LOGFILE
}

Check if IP is in known threat lists
check_threat_intel() {
    local IP=$1
    local THREAT_FOUND=0
    
    # Check against AbuseIPDB (requires API key)
    if [ -n "$ABUSEIPDB_API_KEY" ]; then
        ABUSE_RESULT=$(curl -s -H "Key: $ABUSEIPDB_API_KEY" \
            "https://api.abuseipdb.com/api/v2/check?ipAddress=$IP&maxAgeInDays=90" | \
            jq -r '.data.abuseConfidencePercentage')
        
        if [ "$ABUSE_RESULT" != "null" ] && [ "$ABUSE_RESULT" -gt 50 ]; then
            log "Threat detected: $IP has $ABUSE_RESULT% abuse confidence"
            THREAT_FOUND=1
        fi
    fi
    
    # Check local blacklist
    if grep -q "$IP" "$THREAT_DB" 2>/dev/null; then
        log "Threat detected: $IP found in local threat database"
        THREAT_FOUND=1
    fi
    
    return $THREAT_FOUND
}

Process OSSEC alerts for threat intelligence
process_alerts() {
    # Monitor OSSEC alerts log for new IPs
    tail -f /var/ossec/logs/alerts/alerts.log | while read -r line; do
        if echo "$line" | grep -q "Src IP:"; then
            IP=$(echo "$line" | grep -o 'Src IP: [0-9.]*' | cut -d' ' -f3)
            if [ -n "$IP" ]; then
                if check_threat_intel "$IP"; then
                    log "High-priority threat detected from $IP"
                    # Immediately ban high-threat IPs
                    /usr/bin/fail2ban-client set ossec banip "$IP"
                    # Send alert email
                    echo "High-priority threat detected: $IP" | \
                        mail -s "Critical Security Alert" "$ALERT_EMAIL"
                fi
            fi
        fi
    done
}

Update threat intelligence database
update_threat_db() {
    log "Updating threat intelligence database"
    
    # Download threat feeds (example sources)
    curl -s "https://raw.githubusercontent.com/stamparm/ipsum/master/levels/3.txt" > "$THREAT_DB.tmp"
    
    if [ -s "$THREAT_DB.tmp" ]; then
        mv "$THREAT_DB.tmp" "$THREAT_DB"
        log "Threat database updated with $(wc -l < $THREAT_DB) entries"
    else
        log "Failed to update threat database"
        rm -f "$THREAT_DB.tmp"
    fi
}

case "$1" in
    process)
        process_alerts
        ;;
    update)
        update_threat_db
        ;;
    check)
        if [ -n "$2" ]; then
            check_threat_intel "$2"
        else
            echo "Usage: $0 check "
        fi
        ;;
    *)
        echo "Usage: $0 {process|update|check}"
        ;;
esac

Set up automated reporting

Create a comprehensive reporting system that generates daily security summaries.

#!/bin/bash
security-report.sh - Generate daily security reports
Author: Binadit Infrastructure Team

REPORT_EMAIL="admin@example.com"
REPORT_DATE=$(date '+%Y-%m-%d')
REPORT_FILE="/tmp/security-report-$REPORT_DATE.txt"

Generate comprehensive security report
generate_report() {
    cat > "$REPORT_FILE" << EOF
Daily Security Report for $(hostname)
Generated: $(date '+%Y-%m-%d %H:%M:%S')

=== OSSEC Statistics ===
OSSEC Service Status: $(systemctl is-active ossec.service 2>/dev/null || echo "Unknown")
Alerts Today: $(grep "$(date '+%Y %b %d')" /var/ossec/logs/alerts/alerts.log 2>/dev/null | wc -l)
High Priority Alerts (Level 10+): $(grep "$(date '+%Y %b %d')" /var/ossec/logs/alerts/alerts.log 2>/dev/null | grep -E 'Rule: [0-9]+ \(level 1[0-5]\)' | wc -l)

Top Alert Rules Today:
$(grep "$(date '+%Y %b %d')" /var/ossec/logs/alerts/alerts.log 2>/dev/null | grep -o 'Rule: [0-9]*' | sort | uniq -c | sort -rn | head -5)

=== Fail2ban Statistics ===
Fail2ban Service Status: $(systemctl is-active fail2ban 2>/dev/null || echo "Unknown")
Active Jails: $(/usr/bin/fail2ban-client status 2>/dev/null | grep "Jail list" | cut -d: -f2 | xargs || echo "Unknown")
Currently Banned IPs: $(/usr/bin/fail2ban-client status 2>/dev/null | grep "Currently banned" | awk '{print $4}' || echo "0")

Banned IPs Today:
$(/usr/bin/fail2ban-client status 2>/dev/null | grep "Jail list" | cut -d: -f2 | xargs -n1 /usr/bin/fail2ban-client status | grep "Currently banned" || echo "None")

=== System Health ===
Disk Usage (/var/log): $(df -h /var/log | tail -1 | awk '{print $5}')
Memory Usage: $(free | grep Mem | awk '{printf "%.1f%%", $3/$2 * 100.0}')
Load Average: $(uptime | awk '{print $10" "$11" "$12}')

=== Log File Sizes ===
OSSEC Alerts: $(ls -lh /var/ossec/logs/alerts/alerts.log 2>/dev/null | awk '{print $5}' || echo "N/A")
Fail2ban Log: $(ls -lh /var/log/fail2ban.log 2>/dev/null | awk '{print $5}' || echo "N/A")
Auth Log: $(ls -lh /var/log/auth.log 2>/dev/null | awk '{print $5}' || echo "N/A")

=== Recent Critical Events ===
$(grep "$(date '+%Y %b %d')" /var/log/auth.log 2>/dev/null | grep -i "failed\|error\|invalid" | tail -10 || echo "No critical events found")

EOF
}

Send the report via email
send_report() {
    if command -v mail >/dev/null 2>&1; then
        mail -s "Daily Security Report - $(hostname) - $REPORT_DATE" "$REPORT_EMAIL" < "$REPORT_FILE"
        echo "Security report sent to $REPORT_EMAIL"
    else
        echo "Mail command not available. Report saved to $REPORT_FILE"
    fi
}

Main execution
generate_report
send_report

Cleanup
rm -f "$REPORT_FILE"

Schedule automated tasks

Set up cron jobs for automated monitoring, threat intelligence updates, and reporting.

sudo chmod 755 /usr/local/bin/threat-intel.sh
sudo chmod 755 /usr/local/bin/security-report.sh

Add cron jobs
sudo crontab -l > /tmp/current_cron
cat >> /tmp/current_cron << 'EOF'
Security monitoring tasks
0     /usr/local/bin/security-monitor.sh
0 6   * /usr/local/bin/threat-intel.sh update
0 8   * /usr/local/bin/security-report.sh
EOF

sudo crontab /tmp/current_cron
rm /tmp/current_cron