Add heurisitic debug message filtering

- This helps address forwarded message volume problems
  caused by UAA's verbose, on-by-default debug logs,
  while still allowing those logs to be left on the local disk
  so they will be available when needed for support purposes.
- Integrators/mainifest maintainers have had to maintain this rule,
  making it available as a property simplifies their manifests.

[Finishes #160691113]

Author: anEXPer

jobs/syslog_forwarder/spec

@@ -2,17 +2,18 @@
 name: syslog_forwarder
 
 templates:
+  blackbox_ctl.erb: bin/blackbox_ctl
+  blackbox_config.yml.erb: config/blackbox_config.yml
+  ca_cert.pem.erb: config/ca_cert.pem
+  drain.erb: bin/drain
   pre-start.erb: bin/pre-start
   syslog-release.conf.erb: config/syslog-release.conf
-  syslog-release-vcap-filter.conf.erb: config/syslog-release-vcap-filter.conf
   syslog-release-custom-rules.conf.erb: config/syslog-release-custom-rules.conf
+  syslog-release-debug-filter.conf.erb: config/syslog-release-debug-filter.conf
+  syslog-release-file-exclusion.conf.erb: config/syslog-release-file-exclusion.conf
   syslog-release-forwarding-rules.conf.erb: config/syslog-release-forwarding-rules.conf
   syslog-release-forwarding-setup.conf.erb: config/syslog-release-forwarding-setup.conf
-  syslog-release-file-exclusion.conf.erb: config/syslog-release-file-exclusion.conf
-  ca_cert.pem.erb: config/ca_cert.pem
-  blackbox_ctl.erb: bin/blackbox_ctl
-  blackbox_config.yml.erb: config/blackbox_config.yml
-  drain.erb: bin/drain
+  syslog-release-vcap-filter.conf.erb: config/syslog-release-vcap-filter.conf
 
 packages:
   - blackbox
@@ -85,6 +86,21 @@ properties:
       This may be on by default in the future,
       though this would be a breaking/major version change.
     default: false
+  syslog.heuristically_filter_debug_messages:
+    description: >
+      Drop messages with an msg that start with "DEBUG".
+      This is intended to prevent high-volume,
+      low-value debug logs from overwhelming syslog receivers,
+      while still allowing the UAA job
+      to log its debug messages to disk for support-enablement purposes.
+      While this may impact other logs,
+      most other jobs are not generally configured to emit debug logs,
+      and anyone who wants to filter out UAA's debug volume
+      likely doesn't mind losing the other debug volume, too.
+      This filter is necessarily heuristic/string-based
+      because syslog PRI information is not meaningful
+      in logs produced by blackbox.
+    default: false
 
   syslog.tls_enabled:
     description: Set this to true to enable TLS.

jobs/syslog_forwarder/templates/pre-start.erb

@@ -32,6 +32,10 @@ rsyslogd -N1 || (echo 'Custom rule configuration invalid' && rm /etc/rsyslog.d/3
 cp $(dirname $0)/../config/syslog-release-vcap-filter.conf /etc/rsyslog.d/32-syslog-release-vcap-filter.conf
 chmod 0644 /etc/rsyslog.d/32-syslog-release-vcap-filter.conf
 
+
+cp $(dirname $0)/../config/syslog-release-debug-filter.conf /etc/rsyslog.d/33-syslog-release-debug-filter.conf
+chmod 0644 /etc/rsyslog.d/33-syslog-release-debug-filter.conf
+
 cp $(dirname $0)/../config/syslog-release-forwarding-rules.conf /etc/rsyslog.d/35-syslog-release-forwarding-rules.conf
 chmod 0644 /etc/rsyslog.d/35-syslog-release-forwarding-rules.conf
 

jobs/syslog_forwarder/templates/syslog-release-debug-filter.conf.erb

@@ -0,0 +1,3 @@
+<% unless p('syslog.migration.disabled') || !p('syslog.heuristically_filter_debug_messages') %>
+if ($msg contains " DEBUG ") then stop
+<% end %>

tests/acceptance_test.go

@@ -225,6 +225,24 @@ var _ = Describe("Optional features to reduce CF log volume", func() {
 			}).ShouldNot(ContainSubstring(loggerMessage))
 		})
 	})
+	Context("when DEBUG filtering is enabled to reduce volume", func() {
+		BeforeEach(func() {
+			Cleanup()
+			Deploy("manifests/debug-filtering.yml")
+		})
+		It("filters logs that start with DEBUG while forwarding other logs", func() {
+			By("continuing to forward logs from the filesystem")
+			normalMessage := "INFO is not debug or DEBUG"
+			Eventually(WriteToTestFile(normalMessage)).Should(gbytes.Say(normalMessage))
+
+			By("not forwarding logs that start with DEBUG")
+			debugMessage := "DEBUG is debug, however"
+			SendLogMessage(debugMessage)
+			Consistently(func() string {
+				return ForwardedLogs()
+			}).ShouldNot(ContainSubstring(debugMessage))
+		})
+	})
 })
 
 var _ = Describe("When syslog is configured to run in unprivileged mode", func() {

tests/manifests/debug-filtering.yml

@@ -0,0 +1,40 @@
+---
+name: ((deployment))
+releases:
+  - name: syslog
+    version: latest
+stemcells:
+  - alias: default
+    os: ((stemcell-os))
+    version: latest
+instance_groups:
+  - name: forwarder
+    instances: 1
+    vm_type: default
+    stemcell: default
+    networks:
+      - name: default
+    azs:
+      - z1
+    jobs:
+      - name: syslog_forwarder
+        release: syslog
+    properties:
+      syslog:
+        heuristically_filter_debug_messages: true
+  - name: storer
+    instances: 1
+    vm_type: default
+    stemcell: default
+    networks:
+      - name: default
+    azs:
+      - z1
+    jobs:
+      - name: syslog_storer
+        release: syslog
+update:
+  canaries: 1
+  max_in_flight: 1
+  canary_watch_time: 1000-60000
+  update_watch_time: 1000-60000