From 011852a4a1811df3b5773cc0564b4dc8fb8f9799 Mon Sep 17 00:00:00 2001 From: Rich Megginson Date: Fri, 8 Nov 2024 18:33:53 -0700 Subject: [PATCH] feat: support custom templates Feature: Allow users to specify custom templates to use for `files` and `forwards` outputs. Users can specify a list of templates to use with `logging_custom_templates`. Users can specify the default template to use for all `files` outputs with `logging_files_template_format`, or specify on a per output basis by using `template`. Users can specify the default template to use for all `forwards` outputs with `logging_forwards_template_format`, or specify on a per output basis by using `template`. Reason: Users need the ability to format log entries in different ways other than the built-in defaults, using custom templates. Result: Users can specify custom templates to use for files and forwards outputs. Signed-off-by: Rich Megginson --- .ansible-lint | 1 + README.md | 58 +++++++++++++++++-- defaults/main.yml | 8 +++ roles/rsyslog/defaults/main.yml | 8 +++ roles/rsyslog/tasks/main_core.yml | 5 ++ roles/rsyslog/templates/custom_templates.j2 | 3 + roles/rsyslog/templates/output_files.j2 | 13 ++++- roles/rsyslog/templates/output_forwards.j2 | 4 +- roles/rsyslog/templates/output_relp.j2 | 3 + .../rsyslog/templates/output_remote_files.j2 | 13 ++++- roles/rsyslog/vars/outputs/files/main.yml | 2 + roles/rsyslog/vars/outputs/forwards/main.yml | 2 + tasks/main.yml | 1 + tests/tests_basics_files.yml | 23 ++++++++ 14 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 roles/rsyslog/templates/custom_templates.j2 diff --git a/.ansible-lint b/.ansible-lint index ee1268db..6ffaf631 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,6 +1,7 @@ --- profile: production kinds: + - tasks: "**/tasks/*.yml" - yaml: "**/meta/collection-requirements.yml" - playbook: "**/tests/get_coverage.yml" - yaml: "**/tests/collection-requirements.yml" diff --git a/README.md b/README.md index 83d81c17..1dbe660b 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,42 @@ Available options: tcp_ports: [1514] ``` +### logging_custom_templates + +`logging_custom_templates`: A list of custom template definitions, for use with +`logging_outputs` `type` `files` and `type` `forwards`. You can specify the +template for a particular output to use by setting the `template` field in a +particular `logging_outputs` specification, or by setting the default for all +such outputs to use in `logging_files_template_format` and +`logging_forwards_template_format`. + +Specify custom templates like this, in either the legacy format or the new style +format: + +```yaml +logging_custom_templates: + - | + template(name="tpl1" type="list") { + constant(value="Syslog MSG is: '") + property(name="msg") + constant(value="', ") + property(name="timereported" dateFormat="rfc3339" caseConversion="lower") + constant(value="\n") + } + - >- + $template precise,"%syslogpriority%,%syslogfacility%,%timegenerated::fulltime%,%HOSTNAME%,%syslogtag%,%msg%\n" +``` + +Then use like this: + +```yaml +logging_outputs: + - name: custom_file_output + type: files + path: /var/log/custom_file_output.log + template: tpl1 # override logging_files_template_format if set +``` + ### Logging_outputs options `logging_outputs`: A list of following dictionary to configure outputs. @@ -285,8 +321,6 @@ Available options: * `property_op`: Operation in property-based filter; In case of not `!`, put the `property_op` value in quotes; default to `contains` * `property_value`: Value in property-based filter; default to `error` * `path`: Path to the output file. -* `logging_files_template_format`: Set default template for the files output. - Allowed values are `traditional`, `syslog`, and `modern`. Default to `modern`. * File/Directory properties - same as corresponding variables of the Ansible `file` module: * `mode` - sets the rsyslog `omfile` module `FileCreateMode` parameter * `owner` - sets the rsyslog `omfile` module `fileOwner` or `fileOwnerNum` parameter. If the value @@ -298,6 +332,15 @@ Available options: is an integer, set `dirOwnerNum`, otherwise, set `dirOwner`. * `dir_group` - sets the rsyslog `omfile` module `dirGroup` or `dirGroupNum` parameter. If the value is an integer, set `dirGroupNum`, otherwise, set `dirGroup`. +* `template`: Template format for the particular files output. Allowed values + are `traditional`, `syslog`, and `modern`, or one of the templates defined in + `logging_custom_templates`. Default to `modern`. + +Global options: + +`logging_files_template_format`: Set default template for the files output. +Allowed values are `traditional`, `syslog`, and `modern`, or one of the + templates defined in `logging_custom_templates`. Default to `modern`. **Note:** Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored. @@ -332,10 +375,15 @@ Available options: * `tls`: Set to `true` to encrypt the connection using the default TLS implementation used by the provider. Default to `false`. * `pki_authmode`: Specifying the default network driver authentication mode. `x509/name`, `x509/fingerprint`, or `anon` is accepted. Default to `x509/name`. * `permitted_server`: Hostname, IP address, fingerprint(sha1) or wildcard DNS domain of the server which this client will be allowed to connect and send logs over TLS. Default to `*.{{ logging_domain }}` -* `template`: Template format for the particular forwards output. Allowed values are `traditional`, `syslog`, and `modern`. Default to `modern`. +* `template`: Template format for the particular forwards output. Allowed values + are `traditional`, `syslog`, and `modern`, or one of the templates defined in + `logging_custom_templates`. Default to `modern`. + +Global options: -logging_forwards_template_format: Set default template for the forwards output. -Allowed values are `traditional`, `syslog`, and `modern`. Default to `modern`. +`logging_forwards_template_format`: Set default template for the forwards +output. Allowed values are `traditional`, `syslog`, and `modern`, or one of the + templates defined in `logging_custom_templates`. Default to `modern`. **Note:** Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored. diff --git a/defaults/main.yml b/defaults/main.yml index 774d3a85..19c06313 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -115,6 +115,14 @@ logging_custom_config_files: [] # ca: self-sign logging_certificates: [] +# logging_custom_templates +# +# List of custom templates to provide +# Each element is the string definition of +# an rsyslog output template as defined here: +# https://www.rsyslog.com/doc/configuration/templates.html +logging_custom_templates: [] + # ansible_facts required by the role __logging_required_facts: - distribution diff --git a/roles/rsyslog/defaults/main.yml b/roles/rsyslog/defaults/main.yml index f19140bd..26be4d45 100644 --- a/roles/rsyslog/defaults/main.yml +++ b/roles/rsyslog/defaults/main.yml @@ -31,3 +31,11 @@ rsyslog_extra_packages: [] # List of additional custom config files. # Each element: full paths to the files to be deployed. rsyslog_custom_config_files: [] + +# rsyslog_custom_templates +# +# List of custom templates to provide +# Each element is the string definition of +# an rsyslog output template as defined here: +# https://www.rsyslog.com/doc/configuration/templates.html +rsyslog_custom_templates: [] diff --git a/roles/rsyslog/tasks/main_core.yml b/roles/rsyslog/tasks/main_core.yml index deb6ac32..34ca9482 100644 --- a/roles/rsyslog/tasks/main_core.yml +++ b/roles/rsyslog/tasks/main_core.yml @@ -125,6 +125,11 @@ options: |- $RepeatedMsgReduction {{ "on" if rsyslog_message_reduction | bool else "off" }} + - name: 'templates' + type: 'templates' + sections: + - comment: 'User provided output templates' + options: "{{ lookup('template', 'custom_templates.j2') }}" set_fact: __rsyslog_common_rules: "{{ __rsyslog_global_common_rule }}" diff --git a/roles/rsyslog/templates/custom_templates.j2 b/roles/rsyslog/templates/custom_templates.j2 new file mode 100644 index 00000000..d2f6663f --- /dev/null +++ b/roles/rsyslog/templates/custom_templates.j2 @@ -0,0 +1,3 @@ +{% for template in rsyslog_custom_templates %} +{{ template }} +{% endfor %} diff --git a/roles/rsyslog/templates/output_files.j2 b/roles/rsyslog/templates/output_files.j2 index ce21484e..13c47b6d 100644 --- a/roles/rsyslog/templates/output_files.j2 +++ b/roles/rsyslog/templates/output_files.j2 @@ -42,12 +42,19 @@ ruleset(name="{{ __rsyslog_output.name }}" ruleset(name="{{ __rsyslog_output.name }}") { {% endif %} {{ print_file_attrs(__rsyslog_output) -}} +{% set template = " ;RSYSLOG_TraditionalFileFormat" + if __rsyslog_output.template | d("") == "traditional" + else " ;RSYSLOG_SyslogProtocol23Format" + if __rsyslog_output.template | d("") == "syslog" + else " ;" ~ __rsyslog_output.template + if __rsyslog_output.template | d("") not in ["", "modern"] + else "" %} {% if __rsyslog_output.property | d() %} - :{{ __rsyslog_output.property }}, {{ __rsyslog_output.property_op | d('contains') }}, "{{ __rsyslog_output.property_value | d('error') }}" {{ __rsyslog_output.path }} + :{{ __rsyslog_output.property }}, {{ __rsyslog_output.property_op | d('contains') }}, "{{ __rsyslog_output.property_value | d('error') }}" {{ __rsyslog_output.path }}{{ template }} {% elif __rsyslog_output.exclude | d([]) %} - {{ __rsyslog_output.facility | d('*') }}.{{ __rsyslog_output.severity | d('*') }};{{ __rsyslog_output.exclude | join(';') }} {{ __rsyslog_output.path }} + {{ __rsyslog_output.facility | d('*') }}.{{ __rsyslog_output.severity | d('*') }};{{ __rsyslog_output.exclude | join(';') }} {{ __rsyslog_output.path }}{{ template }} {% else %} - {{ __rsyslog_output.facility | d('*') }}.{{ __rsyslog_output.severity | d('*') }} {{ __rsyslog_output.path }} + {{ __rsyslog_output.facility | d('*') }}.{{ __rsyslog_output.severity | d('*') }} {{ __rsyslog_output.path }}{{ template }} {% endif %} } {% else %} diff --git a/roles/rsyslog/templates/output_forwards.j2 b/roles/rsyslog/templates/output_forwards.j2 index 4ff3bab1..d423f2c7 100644 --- a/roles/rsyslog/templates/output_forwards.j2 +++ b/roles/rsyslog/templates/output_forwards.j2 @@ -34,8 +34,10 @@ ruleset(name="{{ __rsyslog_output.name }}") { Template="RSYSLOG_TraditionalForwardFormat" {% elif __rsyslog_output.template | d('') == 'syslog' %} Template="RSYSLOG_SyslogProtocol23Format" -{% else %} +{% elif __rsyslog_output.template | d('modern') == 'modern' %} Template="RSYSLOG_ForwardFormat" +{% else %} + Template="{{ __rsyslog_output.template }}" {% endif %} {% if __rsyslog_output.action is defined %} {{ lookup('template', 'general_action_params.j2') | indent(8) | trim }} diff --git a/roles/rsyslog/templates/output_relp.j2 b/roles/rsyslog/templates/output_relp.j2 index 7aea466e..48eff80a 100644 --- a/roles/rsyslog/templates/output_relp.j2 +++ b/roles/rsyslog/templates/output_relp.j2 @@ -41,6 +41,9 @@ ruleset(name="{{ __rsyslog_output.name }}") { {% else %} tls.permittedpeer=["{{ '*.' + logging_domain }}"] {% endif %} +{% if __rsyslog_output.template | d("") | length > 0 %} + template="{{ __rsyslog_output.template }}" +{% endif %} {% endif %} ) } diff --git a/roles/rsyslog/templates/output_remote_files.j2 b/roles/rsyslog/templates/output_remote_files.j2 index 807bd3ed..958c7f2a 100644 --- a/roles/rsyslog/templates/output_remote_files.j2 +++ b/roles/rsyslog/templates/output_remote_files.j2 @@ -32,12 +32,19 @@ ruleset(name="{{ __rsyslog_output.name }}" {{ lookup('template', 'general_queue_params.j2') | indent(8) | trim }} {% endif %}) { # Store remote logs in separate logfiles +{% set template = ' template="RSYSLOG_TraditionalFileFormat"' + if __rsyslog_output.template | d("") == "traditional" + else ' template="RSYSLOG_SyslogProtocol23Format"' + if __rsyslog_output.template | d("") == "syslog" + else ' template="' ~ __rsyslog_output.template ~ '"' + if __rsyslog_output.template | d("") not in ["", "modern"] + else "" %} {% if __rsyslog_output.property | d() %} - :{{ __rsyslog_output.property }}, {{ __rsyslog_output.property_op | d('contains') }}, "{{ __rsyslog_output.property_value | d('error') }}" action(name="{{ __rsyslog_output.name }}" type="omfile" DynaFile="{{ __rsyslog_output.name }}_template" DynaFileCacheSize="{{ __rsyslog_output.client_count | d(10) }}" ioBufferSize="{{ __rsyslog_output.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if __rsyslog_output.async_writing | d(false) | bool else 'off' }}"{{ lookup('template', 'general_action_params.j2') | indent(1,true) | regex_replace("\s?\n","") }}) + :{{ __rsyslog_output.property }}, {{ __rsyslog_output.property_op | d('contains') }}, "{{ __rsyslog_output.property_value | d('error') }}" action(name="{{ __rsyslog_output.name }}" type="omfile"{{ template }} DynaFile="{{ __rsyslog_output.name }}_template" DynaFileCacheSize="{{ __rsyslog_output.client_count | d(10) }}" ioBufferSize="{{ __rsyslog_output.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if __rsyslog_output.async_writing | d(false) | bool else 'off' }}"{{ lookup('template', 'general_action_params.j2') | indent(1,true) | regex_replace("\s?\n","") }}) {% elif __rsyslog_output.exclude | d([]) %} - {{ __rsyslog_output.facility | d('*') }}.{{ __rsyslog_output.severity | d('*') }};{{ __rsyslog_output.exclude | join(';') }} action(name="{{ __rsyslog_output.name }}" type="omfile" DynaFile="{{ __rsyslog_output.name }}_template" DynaFileCacheSize="{{ __rsyslog_output.client_count | d(10) }}" ioBufferSize="{{ __rsyslog_output.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if __rsyslog_output.async_writing | d(false) | bool else 'off' }}"{{ lookup('template', 'general_action_params.j2') | indent(1,true) | regex_replace("\s?\n","") }}) + {{ __rsyslog_output.facility | d('*') }}.{{ __rsyslog_output.severity | d('*') }};{{ __rsyslog_output.exclude | join(';') }} action(name="{{ __rsyslog_output.name }}" type="omfile"{{ template }} DynaFile="{{ __rsyslog_output.name }}_template" DynaFileCacheSize="{{ __rsyslog_output.client_count | d(10) }}" ioBufferSize="{{ __rsyslog_output.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if __rsyslog_output.async_writing | d(false) | bool else 'off' }}"{{ lookup('template', 'general_action_params.j2') | indent(1,true) | regex_replace("\s?\n","") }}) {% else %} - {{ __rsyslog_output.facility | d('*') }}.{{ __rsyslog_output.severity | d('*') }} action(name="{{ __rsyslog_output.name }}" type="omfile" DynaFile="{{ __rsyslog_output.name }}_template" DynaFileCacheSize="{{ __rsyslog_output.client_count | d(10) }}" ioBufferSize="{{ __rsyslog_output.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if __rsyslog_output.async_writing | d(false) | bool else 'off' }}"{{ lookup('template', 'general_action_params.j2') | indent(1,true) | regex_replace("\s?\n","") }}) + {{ __rsyslog_output.facility | d('*') }}.{{ __rsyslog_output.severity | d('*') }} action(name="{{ __rsyslog_output.name }}" type="omfile"{{ template }} DynaFile="{{ __rsyslog_output.name }}_template" DynaFileCacheSize="{{ __rsyslog_output.client_count | d(10) }}" ioBufferSize="{{ __rsyslog_output.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if __rsyslog_output.async_writing | d(false) | bool else 'off' }}"{{ lookup('template', 'general_action_params.j2') | indent(1,true) | regex_replace("\s?\n","") }}) {% endif %} } {% else %} diff --git a/roles/rsyslog/vars/outputs/files/main.yml b/roles/rsyslog/vars/outputs/files/main.yml index a64a6452..c3b8c963 100644 --- a/roles/rsyslog/vars/outputs/files/main.yml +++ b/roles/rsyslog/vars/outputs/files/main.yml @@ -27,6 +27,8 @@ __rsyslog_conf_files_output_modules: module(load="builtin:omfile" Template="RSYSLOG_TraditionalFileFormat") {% elif logging_files_template_format == "syslog" %} module(load="builtin:omfile" Template="RSYSLOG_SyslogProtocol23Format") + {% elif logging_files_template_format not in ["", "modern"] %} + module(load="builtin:omfile" Template="{{ logging_files_template_format }}") {% else %} module(load="builtin:omfile") {% endif %} diff --git a/roles/rsyslog/vars/outputs/forwards/main.yml b/roles/rsyslog/vars/outputs/forwards/main.yml index 194c9f8d..aeed9e5b 100644 --- a/roles/rsyslog/vars/outputs/forwards/main.yml +++ b/roles/rsyslog/vars/outputs/forwards/main.yml @@ -27,6 +27,8 @@ __rsyslog_conf_forwards_output_modules: module(load="builtin:omfwd" Template="RSYSLOG_TraditionalForwardFormat") {% elif logging_forwards_template_format == "syslog" %} module(load="builtin:omfwd" Template="RSYSLOG_SyslogProtocol23Format") + {% elif logging_forwards_template_format | length > 0 and logging_forwards_template_format != "modern" %} + module(load="builtin:omfwd" Template="{{ logging_forwards_template_format }}") {% else %} module(load="builtin:omfwd") {% endif %} diff --git a/tasks/main.yml b/tasks/main.yml index 94f529a0..534b65f2 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -130,6 +130,7 @@ selectattr('type', 'defined') | selectattr('type', 'match', 'custom$') | selectattr('custom_config_files', 'defined') | map(attribute='custom_config_files') | flatten | list }}" + rsyslog_custom_templates: "{{ logging_custom_templates }}" include_role: name: "{{ role_path }}/roles/rsyslog" # noqa role-name[path] when: logging_provider == 'rsyslog' diff --git a/tests/tests_basics_files.yml b/tests/tests_basics_files.yml index 48ac3df8..1728df86 100644 --- a/tests/tests_basics_files.yml +++ b/tests/tests_basics_files.yml @@ -300,6 +300,10 @@ facility: local2 target: host.domain tcp_port: 2514 + - name: custom_template + type: files + path: /var/log/logging_custom_template.log + template: BasicTestFormat logging_inputs: - name: basic_input0 type: basics @@ -311,6 +315,13 @@ - files_output1 - forwards_severity_and_facility - forwards_facility_only + - custom_template + logging_custom_templates: + - | + template(name="BasicTestFormat" type="list") { + property(name="timestamp" dateFormat="rfc3339") + constant(value=" ThisIsBasicTestFormat\n") + } include_role: name: linux-system-roles.logging public: true @@ -373,6 +384,18 @@ - name: Check ports managed by firewall and selinux include_tasks: tasks/check_firewall_selinux.yml + - name: Check that the custom template is in the templates file + command: grep BasicTestFormat /etc/rsyslog.d/20-templates.conf + changed_when: false + + - name: Check that the custom template has an output action defined + command: cat /etc/rsyslog.d/30-output-files-custom_template.conf + changed_when: false + + - name: Check that the custom template is being used + command: grep ThisIsBasicTestFormat$ /var/log/logging_custom_template.log + changed_when: false + - name: Set firewall and selinux to false for cleanup set_fact: logging_manage_firewall: false