Add check empty logfiles scripts and some cleaning

roles/rsyslog/defaults/main.yml

@@ -31,3 +31,12 @@ rsyslog_queue_dir: "/var/spool/rsyslog"
 
 # rsyslog_imjournal_statefile # default is imjournal.state which means imjournal.state relative to the rsyslog workdir
 # rsyslog_workdirectory # default /var/spool/rsyslog
+
+# Empty log check script, optional
+rsyslog_enable_warn_empty_script: false
+rsyslog_warn_empty_log_recipient: admin@example.com
+rsyslog_monitor_for_emptylogs_path: "{{ rsyslog_dir }}/apps/prod_sc"
+rsyslog_checkemptylogs_cron_minute: "0"
+rsyslog_checkemptylogs_cron_hour: "9"
+rsyslog_checkemptylogs_cron_weekdays: "1-5"
+rsyslog_checkemptylogs_dir: "/usr/local/bin"

roles/rsyslog/tasks/rsyslog_central.yml

@@ -77,9 +77,25 @@
     src: centralsyslog.j2
     dest: /etc/logrotate.d/centralsyslog
 
-- name: put ryslog config file
+- name: Put ryslog config file
   template:
     src: "rsyslog.conf.j2"
     dest: "/etc/rsyslog.conf"
   notify:
     - "restart rsyslog"
+
+- name: Put log empty warn script and cronjob
+  when: rsyslog_enable_warn_empty_script
+  block:
+    - name: Put log empty script
+      ansible.builtin.template:
+        src: warn-empty-log.sh.j2
+        dest: "{{ rsyslog_checkemptylogs_dir }}/warn-empty-log.sh"
+        backup: True
+    - name: Create cronjob
+      ansible.builtin.cron:
+        name: "check empty logs"
+        minute: "{{ rsyslog_checkemptylogs_cron_minute }}"
+        hour: "{{ rsyslog_checkemptylogs_cron_hour }}"
+        weekday: "{{ rsyslog_checkemptylogs_cron_weekdays }}"
+        job: "{{ rsyslog_checkemptylogs_dir }}/warn-empty-log.sh -m"

roles/rsyslog/templates/sc_ruleset.conf.j2

@@ -16,10 +16,7 @@ if $programname == "engineblock" and $msg contains '{"channel":"authentication"'
 :programname, isequal, "Apache-EBAPI" { action(type="omfile" DynaFile="apache-eb-api-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 :programname, isequal, "manageserver" { action(type="omfile" DynaFile="manage-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 :programname, isequal, "managegui" { action(type="omfile" DynaFile="apache-manage-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
-:programname, isequal, "PDPANALYTICS" { action(type="omfile" DynaFile="pdpanalytics-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 :programname, isequal, "pdp" { action(type="omfile" DynaFile="pdp-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
-:programname, isequal, "pdpserver" { action(type="omfile" DynaFile="pdp-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
-:programname, isequal, "pdpgui" { action(type="omfile" DynaFile="apache-pdp-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 if $programname == "profile" and $msg startswith "{" then { action(type="omfile" DynaFile="profile-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 :programname, isequal, "profile" { action(type="omfile" DynaFile="apache-profile-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 :programname, isequal, "teamsserver" { action(type="omfile" DynaFile="teams-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
@@ -55,8 +52,6 @@ if $programname == "gateway" and $msg contains '{"message":"Intrinsic Loa Reques
 if $programname == "{{ stepupapp }}" and $msg startswith "{{ stepupapp }}" then { action(type="omfile" DynaFile="apache-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 :programname, isequal, "{{ stepupapp }}" { action(type="omfile" DynaFile="stepup-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 :programname, isequal, "Apache-{{ stepupapp }}" { action(type="omfile" DynaFile="apache-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
-if $programname == "{{ stepupapp }}" and $msg startswith "{{ stepupapp }}" then { action(type="omfile" DynaFile="apache-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
-:programname, isequal, "{{ stepupapp }}" { action(type="omfile" DynaFile="stepup-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 {% endfor %}
 :programname, isequal, "Apache-azuremfa" { action(type="omfile" DynaFile="apache-azure-mfa-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
 if $programname == "spdashboard" and $msg startswith "spdashboard" then { action(type="omfile" DynaFile="apache-spdashboard-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }

roles/rsyslog/templates/sc_template.conf.j2

@@ -40,12 +40,12 @@ $template apache-servicedesk-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.na
 $template apache-eduid-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/eduid/eduid-apache.log"
 $template spdashboard-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/spdashboard/spdashboard.log"
 $template apache-spdashboard-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/spdashboard/apache.log"
-$template inviteclient-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/invite//inviteclient.log"
-$template invitewelcome-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/invite//invitewelcome.log"
-$template inviteserver-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/invite//inviteserver.log"
+$template inviteclient-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite//inviteclient.log"
+$template invitewelcome-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite//invitewelcome.log"
+$template inviteserver-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite//inviteserver.log"
 $template invitejson-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite/invitejson.log"
-$template inviteprovisioningmock-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/invite//inviteprovisioningmock.log"
-$template loadbalancer-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/traefik/traefik.log"
+$template inviteprovisioningmock-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite//inviteprovisioningmock.log"
+$template loadbalancer-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/traefik/traefik.log"
 {% for stepupapp in stepupapps %}
 $template stepup-{{ stepupapp }}-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/{{ stepupapp }}/{{ stepupapp }}.log
 $template apache-{{ stepupapp }}-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/{{ stepupapp }}/{{ stepupapp }}-apache.log

roles/rsyslog/templates/warn-empty-log.sh.j2

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Script to check for empty log files in {{ rsyslog_monitor_for_emptylogs_path }}
+# Usage: ./check_empty_logs.sh [-m]
+# -m: send mail if empty files are found
+
+set -e
+
+SEND_MAIL=false
+RECIPIENT="{{ rsyslog_warn_empty_log_recipient }}"
+HOSTNAME=$(hostname --fqdn)
+
+# Parse command line options
+while getopts "m" opt; do
+        case $opt in
+                m)
+                        SEND_MAIL=true
+                        ;;
+                \?)
+                        echo "Invalid option: -$OPTARG" >&2
+                        exit 1
+                        ;;
+        esac
+done
+
+# Find empty log files and read into array
+mapfile -t empty_files < <(find {{ rsyslog_monitor_for_emptylogs_path }} -maxdepth 2 -type f -size 0)
+
+# If no empty files found, exit successfully
+{% raw %}
+if [ ${#empty_files[@]} -eq 0 ]; then
+        exit 0
+fi
+{% endraw %}
+
+# Empty files were found
+{% raw %}
+file_count=${#empty_files[@]}
+{% endraw %}
+
+# Determine singular or plural
+if [ $file_count -eq 1 ]; then
+    file_word="file"
+    verb="was"
+else
+    file_word="files"
+    verb="were"
+fi
+
+# Create mail body
+mail_body=$(cat <<EOF
+Subject: Alert: Empty log files detected on ${HOSTNAME}
+
+Dear SURFconext team,
+
+This is an automated alert to inform you that ${file_count} empty log ${file_word} ${verb} found on ${HOSTNAME} in /opt/surfconext/logs/apps.
+
+Empty log files should not occur under normal operation and may indicate a configuration issue or a problem with the logging system.
+
+List of empty log files:
+
+$(printf '%s\n' "${empty_files[@]}")
+
+Please investigate this issue.
+
+--
+Your friendly automated monitoring system
+EOF
+)
+
+# Send mail if -m option was provided
+if [ "$SEND_MAIL" = true ]; then
+        echo "$mail_body" | mail -s "Alert: Empty log ${file_word} detected on ${HOSTNAME}" "$RECIPIENT"
+        echo "Mail sent to $RECIPIENT about $file_count empty log ${file_word}" | logger
+else
+        echo "Found $file_count empty log ${file_word}, but mail not sent (use -m to send):"
+        echo "$mail_body"
+fi
+
+exit 1