Added check in variable DataNumber parsing if the field_length is 0. … (#57)

mikemiles-dev · mikemiles-dev · Michael Mileusnich · web-flow · commit b5bee62af9d0 · 2024-05-26T22:30:39.000-05:00
IPFix flowsets now properly calculated.

---------

Co-authored-by: mikemiles-dev &lt;michaelmileusnich@Michaels-MacBook-Air-2.local&gt;
Co-authored-by: Michael Mileusnich &lt;mmileusnich@gmail.com&gt;
diff --git a/Cargo.toml b/Cargo.toml
@@ -16,8 +16,9 @@ mac_address = "1.1.5"
 serde = { version = "1.0.166", features = ["derive"] }
 
 [features]
-default = []
+default = ["parse_unknown_fields"]
 unix_timestamp = []
+parse_unknown_fields = []
 
 [dev-dependencies]
 insta = { version = "1.30.0", features = ["yaml"] }
diff --git a/README.md b/README.md
@@ -61,6 +61,11 @@ dbg!(parser.v9_parser.options_templates);
 
 To access templates flowset of a processed V9/IPFix flowset you can find the `flowsets` attribute on the Parsed Record.  In there you can find `Templates`, `Option Templates`, and `Data` Flowsets.
 
+## Features
+
+* unix_timestamp - Current count since 0000 UTC 1970 as Duration.
+* parse_unknown_fields - By default fields not listed in the netflow_parser library will not be parsed and an error is thrown for the packet.  When set to true the field will be parsed as a vector of bytes.
+
 ## Examples
 
 Some examples has been included mainly for those who want to use this parser to read from a Socket and parse netflow.  In those cases with V9/IPFix it is best to create a new parser for each router.  There are both single threaded and multi-threaded examples in the examples directory.
diff --git a/RELEASES.md b/RELEASES.md
@@ -1,5 +1,8 @@
 # 0.3.0
-  * Reworked V9 Parsing.  We enforce the length provided by the flow now.  Padding is now ignored.
+  * Reworked IPFIX + V9 Parsing.  Flowset length is now used.
+  * Flow data field Counts are now correctly calculated.
+  * Added `parse_unknown_fields` feature flag to attempt to parse unknown fields not supported by the library.
+  * `parse_unknown_fields` is enabled by default.
 
 # 0.2.9
   * Fixed parsing issue with V9 flow and padding.
diff --git a/src/lib.rs b/src/lib.rs
@@ -59,6 +59,11 @@
 //! ```
 //! To access templates flowset of a processed V9/IPFix flowset you can find the `flowsets` attribute on the Parsed Record.  In there you can find `Templates`, `Option Templates`, and `Data` Flowsets.
 //!
+//! ## Features
+//!
+//! * unix_timestamp - Current count since 0000 UTC 1970 as Duration.
+//! * parse_unknown_fields - By default fields not listed in the netflow_parser library will not be parsed and an error is thrown for the packet.  When set to true the field will be parsed as a vector of bytes.
+//!
 //! ## Examples
 //! Some examples has been included mainly for those who want to use this parser to read from a Socket and parse netflow.  In those cases with V9/IPFix it is best to create a new parser for each router.  There are both single threaded and multi-threaded examples in the examples directory.
 //!
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap b/src/snapshots/netflow_parser__tests__base_tests__it_doesnt_parse_0_length_fields_ipfix.snap
@@ -0,0 +1,56 @@
+---
+source: src/tests.rs
+expression: "NetflowParser::default().parse_bytes(&packet)"
+---
+- Error:
+    error_message: "Could not parse v10 packet: Parsing Error: Error { input: [], code: Fail }"
+    bytes:
+      - 0
+      - 10
+      - 0
+      - 48
+      - 1
+      - 2
+      - 3
+      - 4
+      - 0
+      - 0
+      - 0
+      - 0
+      - 1
+      - 2
+      - 3
+      - 4
+      - 0
+      - 2
+      - 0
+      - 20
+      - 1
+      - 0
+      - 0
+      - 3
+      - 0
+      - 8
+      - 0
+      - 4
+      - 0
+      - 12
+      - 0
+      - 4
+      - 0
+      - 65
+      - 0
+      - 0
+      - 1
+      - 0
+      - 0
+      - 12
+      - 1
+      - 2
+      - 3
+      - 4
+      - 1
+      - 2
+      - 3
+      - 4
+
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix.snap
@@ -12,34 +12,39 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
       sequence_number: 0
       observation_domain_id: 16909060
     sets:
-      - id: 2
-        length: 20
-        template:
-          template_id: 256
-          field_count: 3
-          fields:
-            - field_type_number: 8
-              field_type: SourceIpv4address
-              field_length: 4
-            - field_type_number: 12
-              field_type: DestinationIpv4address
-              field_length: 4
-            - field_type_number: 2
-              field_type: PacketDeltaCount
-              field_length: 4
-      - id: 256
-        length: 28
-        data:
-          data_fields:
-            - PacketDeltaCount:
-                DataNumber: 16909058
-              SourceIpv4address:
-                Ip4Addr: 1.2.3.4
-              DestinationIpv4address:
-                Ip4Addr: 1.2.3.3
-            - PacketDeltaCount:
-                DataNumber: 67438087
-              SourceIpv4address:
-                Ip4Addr: 0.2.0.2
-              DestinationIpv4address:
-                Ip4Addr: 0.1.2.3
+      - header:
+          id: 2
+          length: 20
+        body:
+          template:
+            template_id: 256
+            field_count: 3
+            fields:
+              - field_type_number: 8
+                field_type: SourceIpv4address
+                field_length: 4
+              - field_type_number: 12
+                field_type: DestinationIpv4address
+                field_length: 4
+              - field_type_number: 2
+                field_type: PacketDeltaCount
+                field_length: 4
+      - header:
+          id: 256
+          length: 28
+        body:
+          data:
+            data_fields:
+              - PacketDeltaCount:
+                  DataNumber: 16909058
+                SourceIpv4address:
+                  Ip4Addr: 1.2.3.4
+                DestinationIpv4address:
+                  Ip4Addr: 1.2.3.3
+              - PacketDeltaCount:
+                  DataNumber: 67438087
+                SourceIpv4address:
+                  Ip4Addr: 0.2.0.2
+                DestinationIpv4address:
+                  Ip4Addr: 0.1.2.3
+
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_data_cached_template.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_data_cached_template.snap
@@ -12,11 +12,14 @@ expression: parser.parse_bytes(&packet)
       sequence_number: 1
       observation_domain_id: 0
     sets:
-      - id: 258
-        length: 10
-        data:
-          data_fields:
-            - PacketDeltaCount:
-                DataNumber: 8
-              SourceIpv4address:
-                Ip4Addr: 0.0.1.1
+      - header:
+          id: 258
+          length: 10
+        body:
+          data:
+            data_fields:
+              - PacketDeltaCount:
+                  DataNumber: 8
+                SourceIpv4address:
+                  Ip4Addr: 0.0.1.1
+
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template.snap
@@ -12,20 +12,23 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
       sequence_number: 1
       observation_domain_id: 2
     sets:
-      - id: 3
-        length: 28
-        options_template:
-          template_id: 260
-          field_count: 3
-          scope_field_count: 1
-          fields:
-            - field_type_number: 123
-              field_type: Enterprise
-              field_length: 4
-              enterprise_number: 2
-            - field_type_number: 32809
-              field_type: ExportedMessageTotalCount
-              field_length: 2
-            - field_type_number: 32810
-              field_type: ExportedFlowRecordTotalCount
-              field_length: 2
+      - header:
+          id: 3
+          length: 28
+        body:
+          options_template:
+            template_id: 260
+            field_count: 3
+            scope_field_count: 1
+            fields:
+              - field_type_number: 123
+                field_type: Enterprise
+                field_length: 4
+                enterprise_number: 2
+              - field_type_number: 32809
+                field_type: ExportedMessageTotalCount
+                field_length: 2
+              - field_type_number: 32810
+                field_type: ExportedFlowRecordTotalCount
+                field_length: 2
+
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template_with_data.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_options_template_with_data.snap
@@ -12,36 +12,41 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
       sequence_number: 1
       observation_domain_id: 2
     sets:
-      - id: 3
-        length: 28
-        options_template:
-          template_id: 260
-          field_count: 3
-          scope_field_count: 1
-          fields:
-            - field_type_number: 123
-              field_type: Enterprise
-              field_length: 4
-              enterprise_number: 2
-            - field_type_number: 32809
-              field_type: ExportedMessageTotalCount
-              field_length: 2
-            - field_type_number: 32810
-              field_type: ExportedFlowRecordTotalCount
-              field_length: 2
-      - id: 260
-        length: 20
-        options_data:
-          data_fields:
-            - ExportedMessageTotalCount:
-                DataNumber: 276
-              ExportedFlowRecordTotalCount:
-                DataNumber: 5140
-              Enterprise:
-                DataNumber: 1
-            - ExportedMessageTotalCount:
-                DataNumber: 5140
-              ExportedFlowRecordTotalCount:
-                DataNumber: 7710
-              Enterprise:
-                DataNumber: 2
+      - header:
+          id: 3
+          length: 28
+        body:
+          options_template:
+            template_id: 260
+            field_count: 3
+            scope_field_count: 1
+            fields:
+              - field_type_number: 123
+                field_type: Enterprise
+                field_length: 4
+                enterprise_number: 2
+              - field_type_number: 32809
+                field_type: ExportedMessageTotalCount
+                field_length: 2
+              - field_type_number: 32810
+                field_type: ExportedFlowRecordTotalCount
+                field_length: 2
+      - header:
+          id: 260
+          length: 20
+        body:
+          options_data:
+            data_fields:
+              - ExportedMessageTotalCount:
+                  DataNumber: 276
+                ExportedFlowRecordTotalCount:
+                  DataNumber: 5140
+                Enterprise:
+                  DataNumber: 1
+              - ExportedMessageTotalCount:
+                  DataNumber: 5140
+                ExportedFlowRecordTotalCount:
+                  DataNumber: 7710
+                Enterprise:
+                  DataNumber: 2
+
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_with_no_template_fields_raises_error.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_with_no_template_fields_raises_error.snap
@@ -2,19 +2,25 @@
 source: src/tests.rs
 expression: parser.parse_bytes(&packet)
 ---
-- IPFix:
-    header:
-      version: 10
-      length: 26
-      export_time:
-        secs: 1
-        nanos: 0
-      sequence_number: 1
-      observation_domain_id: 0
-    sets: []
 - Error:
-    error_message: Unsupported Version
+    error_message: "Could not parse v10 packet: Parsing Error: Error { input: [0, 8, 0, 0, 1, 1], code: Fail }"
     bytes:
+      - 0
+      - 10
+      - 0
+      - 26
+      - 0
+      - 0
+      - 0
+      - 1
+      - 0
+      - 0
+      - 0
+      - 1
+      - 0
+      - 0
+      - 0
+      - 0
       - 1
       - 2
       - 0
@@ -25,3 +31,4 @@ expression: parser.parse_bytes(&packet)
       - 0
       - 1
       - 1
+
diff --git a/src/snapshots/netflow_parser__tests__base_tests__it_parses_multiple_packets.snap b/src/snapshots/netflow_parser__tests__base_tests__it_parses_multiple_packets.snap
diff --git a/src/tests.rs b/src/tests.rs
diff --git a/src/variable_versions/common.rs b/src/variable_versions/common.rs
diff --git a/src/variable_versions/ipfix.rs b/src/variable_versions/ipfix.rs
