dstreefkerk · October 1, 2025 02:06 · Oct 1, 2025 · Oct 1, 2025 · Aug 4, 2025 · Jun 3, 2025
diff --git a/40-fortianalyzer-filtered-cef.conf b/40-fortianalyzer-filtered-cef.conf
@@ -32,7 +32,7 @@
 #
 # FILTERING LOGIC:
 # ----------------
-# FORWARDED: ALL messages containing type="utm"
+# FORWARDED: ALL messages containing cat="utm"
 # DROPPED:   Non-CEF messages and non-UTM CEF messages
 #
 # IMPORTANT - FORTIANALYZER CONNECTION MODE:

diff --git a/40-fortianalyzer-filtered-cef.conf b/40-fortianalyzer-filtered-cef.conf
@@ -75,7 +75,7 @@ ruleset(name="forti-force-pri-cef") {
 
     # Forward ALL UTM logs regardless of severity level
     # This ensures complete security visibility in Sentinel
-    if ($rawmsg contains 'type="utm"') then {
+    if ($rawmsg contains 'cat="utm"') then {
         # FORCE PRI header addition for AMA compliance
         action(type="omfwd"
                template="FortiCEF_ForcePRI"

diff --git a/40-fortianalyzer-filtered-cef.conf b/40-fortianalyzer-filtered-cef.conf
@@ -1,83 +1,82 @@
 #####################################################################
-# FortiAnalyzer CEF Integration with Severity-Based Noise Reduction
+# FortiAnalyzer CEF Integration - Production Configuration
 #####################################################################
 #
 # PURPOSE:
 # --------
-# This configuration solves two critical problems with FortiAnalyzer
-# CEF log forwarding to Microsoft Sentinel:
+# This configuration solves the critical protocol compliance issue with 
+# FortiAnalyzer CEF log forwarding to Microsoft Sentinel via Azure Monitor Agent.
 #
-# 1. PROTOCOL COMPLIANCE: FortiAnalyzer sends raw CEF messages without
-#    RFC3164/RFC5424 compliant syslog PRI headers. Azure Monitor Agent
-#    (AMA) requires properly formatted syslog messages and will reject
-#    or misprocess headerless CEF data.
-#
-# 2. LOG VOLUME REDUCTION: FortiAnalyzer generates high volumes of
-#    informational and notice-level logs that create noise in Sentinel,
-#    increase ingestion costs, and reduce alert-to-noise ratio.
+# PROBLEM:
+# --------
+# FortiAnalyzer sends CEF messages in partial syslog format:
+#   SENDS:     "MMM dd HH:mm:ss hostname CEF:0|..."
+#   EXPECTED:  "<164>MMM dd HH:mm:ss hostname CEF:0|..."
+# 
+# Azure Monitor Agent (AMA) strictly requires RFC3164/RFC5424 compliant 
+# syslog messages with PRI headers and will reject headerless messages entirely.
 #
 # SOLUTION:
 # ---------
-# - Receives raw CEF messages from FortiAnalyzer on UDP port 1514
+# - Receives CEF messages from FortiAnalyzer on TCP port 1514
 # - Adds mandatory syslog PRI header <164> (local4.warning) for AMA compliance
-# - Implements severity-based filtering at rsyslog level (before AMA)
-# - Only forwards security-relevant events (critical/high/medium severity)
-# - Filters out noise (notice/info/debug severity) before network transmission
+# - Forwards ALL UTM logs to ensure complete security visibility
+# - No severity-based filtering to prevent missing critical security events
 #
-# BENEFITS:
-# ---------
-# - 60-80% reduction in Sentinel ingestion volume and costs
-# - Improved security alert-to-noise ratio
-# - Reduced bandwidth usage between log collector and Azure
-# - Maintains full compliance with Azure Monitor Agent requirements
-# - Preserves all security-critical events while eliminating operational noise
+# KEY LEARNINGS:
+# --------------
+# Initial attempts at severity-based filtering (deviceSeverity=critical|high|medium)
+# resulted in missing logs in Sentinel. Customer requirements prioritised log 
+# completeness over volume reduction. The simple type="utm" filter ensures all
+# Unified Threat Management logs are captured.
 #
 # FILTERING LOGIC:
 # ----------------
-# FORWARDED: deviceSeverity=critical, high, medium
-# DROPPED: deviceSeverity=notice, info, debug
+# FORWARDED: ALL messages containing type="utm"
+# DROPPED:   Non-CEF messages and non-UTM CEF messages
 #
-# IMPORTANT - FORTIANALYZER RELIABLE CONNECTION MODE:
-# --------------------------------------------------
-# By default, FortiAnalyzer sends logs via UDP (unreliable, best-effort delivery).
-# When "Reliable Connection" is enabled in FortiAnalyzer, it switches to TCP.
-# 
-# This configuration ONLY listens on UDP port 1514 by default.
-# If you enable "Reliable Connection" in FortiAnalyzer, logs will stop flowing!
+# IMPORTANT - FORTIANALYZER CONNECTION MODE:
+# ------------------------------------------
+# FortiAnalyzer can send logs via UDP (default) or TCP ("Reliable Connection" mode).
+# This configuration is set up for TCP on port 1514, which is recommended for
+# production environments to ensure reliable log delivery.
 #
-# To support TCP mode, uncomment the TCP INPUT SECTION below.
+# To support UDP mode (if needed), uncomment the UDP INPUT line at the bottom.
 # You can safely enable both UDP and TCP listeners simultaneously.
 #
 # DEPLOYMENT:
 # -----------
 # 1. Configure FortiAnalyzer to forward CEF logs to this server:1514
+#    - Enable "Reliable Connection" for TCP mode (recommended)
 # 2. Ensure Azure Monitor Agent is configured with appropriate DCR
-# 3. This setup assumes that the DCR is listening for LOG_LOCAL4:LOG_WARNING
-# 4. If FortiAnalyzer "Reliable Connection" is enabled, uncomment TCP section
-# 5. If debugging, uncomment the line that starts with #action(type="omfile"
-#    and monitor /var/log/forti-force-pri.log for all received messages
-# 6. Validate filtered logs appear in Sentinel CommonSecurityLog table
+# 3. DCR should be configured for LOG_LOCAL4:LOG_WARNING
+# 4. For debugging, uncomment the action(type="omfile" line and monitor
+#    /var/log/forti-force-pri.log to see all received messages
+# 5. Validate logs appear in Sentinel CommonSecurityLog table
+#    - Note: Ingestion delay can be 10-15 minutes
 #
-# VALIDATION TESTED: 2025-05-23
+# VALIDATION COMPLETED: 2025-05-23
+# UPDATED: 2025-08-04 - Changed to accept all UTM logs based on production feedback
 #####################################################################
 
-# Template that ALWAYS adds PRI header - no conditional logic
-template(name="FortiCEF_ForcePRI" type="string"
-  string="<164>%TIMESTAMP% %HOSTNAME% %rawmsg-after-pri%\n")
+# Template that ALWAYS adds PRI header - no conditional logic needed
+template(name="FortiCEF_ForcePRI" type="string" 
+         string="<164>%TIMESTAMP% %HOSTNAME% %rawmsg-after-pri%\n")
 
 # Main processing ruleset - handles both UDP and TCP inputs
 ruleset(name="forti-force-pri-cef") {
-    # Debug logging
+    # Debug logging - uncomment to troubleshoot
     #action(type="omfile" file="/var/log/forti-force-pri.log")
 
-    # CEF filtering
+    # CEF filtering - only process CEF formatted messages
     if not ($rawmsg contains "CEF:") then {
         stop
     }
 
-    # SEVERITY FILTERING - Only forward Warning+ severity messages
-    if ($rawmsg contains "deviceSeverity=critical" or $rawmsg contains "deviceSeverity=high" or $rawmsg contains "deviceSeverity=medium") then {
-        # FORCE PRI header addition - unconditional
+    # Forward ALL UTM logs regardless of severity level
+    # This ensures complete security visibility in Sentinel
+    if ($rawmsg contains 'type="utm"') then {
+        # FORCE PRI header addition for AMA compliance
         action(type="omfwd"
                template="FortiCEF_ForcePRI"
                target="127.0.0.1"
@@ -89,21 +88,18 @@ ruleset(name="forti-force-pri-cef") {
                queue.maxDiskSpace="1g"
                action.resumeRetryCount="-1"
                action.resumeInterval="5"
-               queue.size="25000"
+               queue.size="25000" 
                queue.workerThreads="100"
                queue.saveonshutdown="on")
     }
+
     stop
 }
 
-# UDP INPUT (Default - always enabled)
-input(type="imudp" port="1514" ruleset="forti-force-pri-cef")
+# TCP INPUT (Recommended for production - reliable delivery)
+module(load="imtcp")  # Load TCP input module
+input(type="imtcp" port="1514" ruleset="forti-force-pri-cef")
 
-# TCP INPUT SECTION - UNCOMMENT IF FORTIANALYZER "RELIABLE CONNECTION" IS ENABLED
-# -------------------------------------------------------------------------------
-# FortiAnalyzer switches from UDP to TCP when "Reliable Connection" is checked.
-# TCP provides guaranteed delivery with retransmission but adds overhead.
-# You can have both UDP and TCP listeners active simultaneously.
-#
-#module(load="imtcp")  # Load TCP input module
-#input(type="imtcp" port="1514" ruleset="forti-force-pri-cef")
+# UDP INPUT (Optional - uncomment if FortiAnalyzer uses default UDP mode)
+# Note: UDP is best-effort delivery and may lose logs under heavy load
+#input(type="imudp" port="1514" ruleset="forti-force-pri-cef")
diff --git a/40-fortianalyzer-filtered-cef.conf b/40-fortianalyzer-filtered-cef.conf
@@ -4,16 +4,16 @@
 #
 # PURPOSE:
 # --------
-# This configuration solves two critical problems with FortiAnalyzer 
+# This configuration solves two critical problems with FortiAnalyzer
 # CEF log forwarding to Microsoft Sentinel:
 #
-# 1. PROTOCOL COMPLIANCE: FortiAnalyzer sends raw CEF messages without 
-#    RFC3164/RFC5424 compliant syslog PRI headers. Azure Monitor Agent 
-#    (AMA) requires properly formatted syslog messages and will reject 
+# 1. PROTOCOL COMPLIANCE: FortiAnalyzer sends raw CEF messages without
+#    RFC3164/RFC5424 compliant syslog PRI headers. Azure Monitor Agent
+#    (AMA) requires properly formatted syslog messages and will reject
 #    or misprocess headerless CEF data.
 #
-# 2. LOG VOLUME REDUCTION: FortiAnalyzer generates high volumes of 
-#    informational and notice-level logs that create noise in Sentinel, 
+# 2. LOG VOLUME REDUCTION: FortiAnalyzer generates high volumes of
+#    informational and notice-level logs that create noise in Sentinel,
 #    increase ingestion costs, and reduce alert-to-noise ratio.
 #
 # SOLUTION:
@@ -34,30 +34,43 @@
 #
 # FILTERING LOGIC:
 # ----------------
-# FORWARDED:  deviceSeverity=critical, high, medium
-# DROPPED:    deviceSeverity=notice, info, debug
+# FORWARDED: deviceSeverity=critical, high, medium
+# DROPPED: deviceSeverity=notice, info, debug
+#
+# IMPORTANT - FORTIANALYZER RELIABLE CONNECTION MODE:
+# --------------------------------------------------
+# By default, FortiAnalyzer sends logs via UDP (unreliable, best-effort delivery).
+# When "Reliable Connection" is enabled in FortiAnalyzer, it switches to TCP.
+# 
+# This configuration ONLY listens on UDP port 1514 by default.
+# If you enable "Reliable Connection" in FortiAnalyzer, logs will stop flowing!
+#
+# To support TCP mode, uncomment the TCP INPUT SECTION below.
+# You can safely enable both UDP and TCP listeners simultaneously.
 #
 # DEPLOYMENT:
 # -----------
 # 1. Configure FortiAnalyzer to forward CEF logs to this server:1514
 # 2. Ensure Azure Monitor Agent is configured with appropriate DCR
 # 3. This setup assumes that the DCR is listening for LOG_LOCAL4:LOG_WARNING
-# 3. If debugging, uncomment the line that starts with #action(type="omfile" 
+# 4. If FortiAnalyzer "Reliable Connection" is enabled, uncomment TCP section
+# 5. If debugging, uncomment the line that starts with #action(type="omfile"
 #    and monitor /var/log/forti-force-pri.log for all received messages
-# 4. Validate filtered logs appear in Sentinel CommonSecurityLog table
+# 6. Validate filtered logs appear in Sentinel CommonSecurityLog table
 #
 # VALIDATION TESTED: 2025-05-23
 #####################################################################
 
 # Template that ALWAYS adds PRI header - no conditional logic
-template(name="FortiCEF_ForcePRI" type="string" 
-         string="<164>%TIMESTAMP% %HOSTNAME% %rawmsg-after-pri%\n")
+template(name="FortiCEF_ForcePRI" type="string"
+  string="<164>%TIMESTAMP% %HOSTNAME% %rawmsg-after-pri%\n")
 
+# Main processing ruleset - handles both UDP and TCP inputs
 ruleset(name="forti-force-pri-cef") {
     # Debug logging
-    # action(type="omfile" file="/var/log/forti-force-pri.log")
+    #action(type="omfile" file="/var/log/forti-force-pri.log")
 
-    # CEF filtering  
+    # CEF filtering
     if not ($rawmsg contains "CEF:") then {
         stop
     }
@@ -76,12 +89,21 @@ ruleset(name="forti-force-pri-cef") {
                queue.maxDiskSpace="1g"
                action.resumeRetryCount="-1"
                action.resumeInterval="5"
-               queue.size="25000" 
+               queue.size="25000"
                queue.workerThreads="100"
                queue.saveonshutdown="on")
     }
-
     stop
 }
 
+# UDP INPUT (Default - always enabled)
 input(type="imudp" port="1514" ruleset="forti-force-pri-cef")
+
+# TCP INPUT SECTION - UNCOMMENT IF FORTIANALYZER "RELIABLE CONNECTION" IS ENABLED
+# -------------------------------------------------------------------------------
+# FortiAnalyzer switches from UDP to TCP when "Reliable Connection" is checked.
+# TCP provides guaranteed delivery with retransmission but adds overhead.
+# You can have both UDP and TCP listeners active simultaneously.
+#
+#module(load="imtcp")  # Load TCP input module
+#input(type="imtcp" port="1514" ruleset="forti-force-pri-cef")
diff --git a/40-fortianalyzer-filtered-cef.conf b/40-fortianalyzer-filtered-cef.conf
@@ -51,7 +51,7 @@
 
 # Template that ALWAYS adds PRI header - no conditional logic
 template(name="FortiCEF_ForcePRI" type="string" 
-         string="<164>%TIMESTAMP% %HOSTNAME% FortiAnalyzer: %rawmsg-after-pri%\n")
+         string="<164>%TIMESTAMP% %HOSTNAME% %rawmsg-after-pri%\n")
 
 ruleset(name="forti-force-pri-cef") {
     # Debug logging

diff --git a/40-fortianalyzer-filtered-cef.conf b/40-fortianalyzer-filtered-cef.conf
@@ -0,0 +1,87 @@
+#####################################################################
+# FortiAnalyzer CEF Integration with Severity-Based Noise Reduction
+#####################################################################
+#
+# PURPOSE:
+# --------
+# This configuration solves two critical problems with FortiAnalyzer 
+# CEF log forwarding to Microsoft Sentinel:
+#
+# 1. PROTOCOL COMPLIANCE: FortiAnalyzer sends raw CEF messages without 
+#    RFC3164/RFC5424 compliant syslog PRI headers. Azure Monitor Agent 
+#    (AMA) requires properly formatted syslog messages and will reject 
+#    or misprocess headerless CEF data.
+#
+# 2. LOG VOLUME REDUCTION: FortiAnalyzer generates high volumes of 
+#    informational and notice-level logs that create noise in Sentinel, 
+#    increase ingestion costs, and reduce alert-to-noise ratio.
+#
+# SOLUTION:
+# ---------
+# - Receives raw CEF messages from FortiAnalyzer on UDP port 1514
+# - Adds mandatory syslog PRI header <164> (local4.warning) for AMA compliance
+# - Implements severity-based filtering at rsyslog level (before AMA)
+# - Only forwards security-relevant events (critical/high/medium severity)
+# - Filters out noise (notice/info/debug severity) before network transmission
+#
+# BENEFITS:
+# ---------
+# - 60-80% reduction in Sentinel ingestion volume and costs
+# - Improved security alert-to-noise ratio
+# - Reduced bandwidth usage between log collector and Azure
+# - Maintains full compliance with Azure Monitor Agent requirements
+# - Preserves all security-critical events while eliminating operational noise
+#
+# FILTERING LOGIC:
+# ----------------
+# FORWARDED:  deviceSeverity=critical, high, medium
+# DROPPED:    deviceSeverity=notice, info, debug
+#
+# DEPLOYMENT:
+# -----------
+# 1. Configure FortiAnalyzer to forward CEF logs to this server:1514
+# 2. Ensure Azure Monitor Agent is configured with appropriate DCR
+# 3. This setup assumes that the DCR is listening for LOG_LOCAL4:LOG_WARNING
+# 3. If debugging, uncomment the line that starts with #action(type="omfile" 
+#    and monitor /var/log/forti-force-pri.log for all received messages
+# 4. Validate filtered logs appear in Sentinel CommonSecurityLog table
+#
+# VALIDATION TESTED: 2025-05-23
+#####################################################################
+
+# Template that ALWAYS adds PRI header - no conditional logic
+template(name="FortiCEF_ForcePRI" type="string" 
+         string="<164>%TIMESTAMP% %HOSTNAME% FortiAnalyzer: %rawmsg-after-pri%\n")
+
+ruleset(name="forti-force-pri-cef") {
+    # Debug logging
+    # action(type="omfile" file="/var/log/forti-force-pri.log")
+
+    # CEF filtering  
+    if not ($rawmsg contains "CEF:") then {
+        stop
+    }
+
+    # SEVERITY FILTERING - Only forward Warning+ severity messages
+    if ($rawmsg contains "deviceSeverity=critical" or $rawmsg contains "deviceSeverity=high" or $rawmsg contains "deviceSeverity=medium") then {
+        # FORCE PRI header addition - unconditional
+        action(type="omfwd"
+               template="FortiCEF_ForcePRI"
+               target="127.0.0.1"
+               port="28330"
+               protocol="tcp"
+               queue.type="LinkedList"
+               queue.filename="omfwd-forti-force"
+               queue.maxFileSize="32m"
+               queue.maxDiskSpace="1g"
+               action.resumeRetryCount="-1"
+               action.resumeInterval="5"
+               queue.size="25000" 
+               queue.workerThreads="100"
+               queue.saveonshutdown="on")
+    }
+
+    stop
+}
+
+input(type="imudp" port="1514" ruleset="forti-force-pri-cef")