Add API for reading message slices from a source.

This change will support:
- Generic statistics (esrlabs#31)
- Reading messages from streams (esrlabs#34)

Additionally it provides a cleanup of the crate's feature names
and removes the dependecy to the legacy buf_redux create.

Author: kruss

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.20.0] - 2025-02-25
+
+### Added
+
+- Support generic statistics from reading messages
+- Support parsing of DLT messages from streams
+
+### Changed
+
+- Cleanup feature names
+
 ## [0.19.2] - 2025-02-06
 
 ### Changed
@@ -36,7 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- Add feature "serde-support", which adds to crate's types Serialize/Deserialize
+- Add feature "serialization", which adds to crate's types Serialize/Deserialize
 
 ## [0.17.0] - 2024-10-04
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dlt-core"
-version = "0.19.2"
+version = "0.20.0"
 authors = ["esrlabs.com"]
 edition = "2021"
 description = """
@@ -21,13 +21,15 @@ rustc-hash = { version = "2.1", optional = true }
 serde = { version = "1.0", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }
 thiserror = "1.0"
+futures = "0.3"
 
 [features]
 default = []
 statistics = ["buf_redux", "rustc-hash"]
-fibex_parser = ["quick-xml"]
-debug_parser = []
-serde-support = ["serde", "serde_json"]
+fibex = ["quick-xml"]
+debug = []
+serialization = ["serde", "serde_json"]
+stream = []
 
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin_include)'] }
@@ -40,6 +42,9 @@ env_logger = "0.10"
 pretty_assertions = "1.3"
 proptest = "1.6"
 proptest-derive = "0.5"
+tokio = { version = "1", features = ["full"] }
+tokio-stream = "0.1"
+tokio-util = "0.7"
 
 [[bench]]
 name = "dlt_benchmarks"

README.md

@@ -150,11 +150,13 @@ Below is the revised and improved English version of the documentation:
 
 * **`statistics`**: Enables the `statistics` module, which scans the source data and provides a summary of its contents. This gives you an overview of the number of messages and their content.
 
-* **`fibex_parser`**: Enables the `fibex` module, which allows to parse configurations for non-verbose messages from a fibex model.
+* **`fibex`**: Enables the `fibex` module, which allows to parse configurations for non-verbose messages from a fibex model.
 
-* **`debug_parser`**: Adds additional log output for debugging purposes.
+* **`debug`**: Adds additional log output for debugging purposes.
 
-* **`serde-support`**: Adds `Serialize` and `Deserialize` implementations (via `serde`) to all public types. This feature is useful if you need to encode or decode these types for transmission or storage.
+* **`serialization`**: Adds `Serialize` and `Deserialize` implementations (via `serde`) to all public types. This feature is useful if you need to encode or decode these types for transmission or storage.
+
+* **`stream`**: Provides API for parsing DLT messages from streams.
 
 ## Example users
 

src/dlt.rs

@@ -42,7 +42,7 @@ pub enum Error {
 
 /// Used to express the byte order of DLT data type fields
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, PartialOrd, Ord)]
@@ -56,7 +56,7 @@ pub enum Endianness {
 
 /// represents a DLT message including all headers
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -69,7 +69,7 @@ pub struct Message {
 
 /// Storage header is used in case of dlt entries stored in file
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -82,7 +82,7 @@ pub struct StorageHeader {
 
 /// The Standard Header shall be in big endian format
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -107,7 +107,7 @@ pub struct StandardHeader {
 ///
 /// The Extended Header shall be in big endian format
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -144,7 +144,7 @@ pub struct ExtendedHeader {
 /// and payload. The payload contains of the Service ID and the contained parameters.
 ///
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -177,7 +177,7 @@ pub enum PayloadContent {
 }
 
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -343,7 +343,7 @@ impl StandardHeader {
 
 /// Representation of log levels used in DLT log messages
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, PartialOrd, Clone, Copy)]
@@ -378,7 +378,7 @@ impl AsRef<str> for LogLevel {
 /// In case the dlt message contains tracing information, the Trace-Type
 /// indicates different kinds of trace message types
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -414,7 +414,7 @@ impl AsRef<str> for ApplicationTraceType {
 /// In case the dlt message contains networking information,
 /// the Trace-Type indicates different kinds of network message types
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -457,7 +457,7 @@ const CTRL_TYPE_RESPONSE: u8 = 0x2;
 /// In case the dlt message contains control information,
 /// the Trace-Type indicates different kinds of control message types
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -498,7 +498,7 @@ impl ControlType {
 
 /// Part of the extended header, distinguishes log, trace and controll messages
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -559,7 +559,7 @@ impl ExtendedHeader {
 /// Fixed-Point representation. only supports 32 bit and 64 bit values
 /// according to the spec 128 bit are possible but we don't support it
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -578,7 +578,7 @@ pub(crate) fn fixed_point_value_width(v: &FixedPointValue) -> usize {
 
 /// Represents the value of an DLT argument
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -603,7 +603,7 @@ pub enum Value {
 /// Defines what string type is used, `ASCII` or `UTF8`
 #[allow(clippy::upper_case_acronyms)]
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -620,7 +620,7 @@ pub enum StringCoding {
 
 /// Represents the bit width of a floatingpoint value type
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq, Copy)]
@@ -639,7 +639,7 @@ pub(crate) fn float_width_to_type_length(width: FloatWidth) -> TypeLength {
 
 /// Represents the bit width of a value type
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq, Copy)]
@@ -677,7 +677,7 @@ impl TypeLength {
 ///
 /// the Array type is not yet supported and honestly I never saw anyone using it
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -724,7 +724,7 @@ pub enum TypeInfoKind {
 /// number of characters of the associated name or unit filed. The unit
 /// information is to add only in some data types.
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -910,7 +910,7 @@ impl TryFrom<u32> for TypeInfo {
 ///     * i64 bit if Type Length (TYLE) equals 4
 ///     * i128 bit if Type Length (TYLE) equals 5 (unsupported)
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -927,7 +927,7 @@ pub struct FixedPoint {
 /// itself, it is needed to provide information like size and type
 /// of the variable. This information is contained in the `type_info` field.
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -1528,7 +1528,7 @@ fn payload_content_len(content: &PayloadContent) -> usize {
 
 /// Configuration options for the extended header
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -1540,7 +1540,7 @@ pub struct ExtendedHeaderConfig {
 
 /// Configuration options for a DLT message, used when constructing a message
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -1557,7 +1557,7 @@ pub struct MessageConfig {
 
 #[inline]
 fn dbg_bytes_with_info(_name: &str, _bytes: &[u8], _info: Option<&str>) {
-    #[cfg(feature = "debug_parser")]
+    #[cfg(feature = "debug")]
     {
         trace!(
             "writing {}: {} {:02X?} {}",

src/fibex/mod.rs

@@ -55,7 +55,7 @@ pub enum Error {
 
 /// Contains all the paths of fibex files that should be combined into the model
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug)]

src/filtering.rs

@@ -25,7 +25,7 @@ use std::{collections::HashSet, iter::FromIterator};
 /// only this is possible:
 /// - `app-id is_one_of ["abc","foo"] AND log-level <= DEBUG`
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone)]
@@ -96,8 +96,8 @@ impl From<&DltFilterConfig> for ProcessedDltFilterConfig {
     }
 }
 
-/// Read filter config from a json file. Available only with feature "serde-support"
-#[cfg(feature = "serde-support")]
+/// Read filter config from a json file. Available only with feature "serialization"
+#[cfg(feature = "serialization")]
 pub fn read_filter_options(f: &mut std::fs::File) -> Option<DltFilterConfig> {
     use std::io::Read;
 

src/lib.rs

@@ -20,10 +20,11 @@
 extern crate log;
 
 pub mod dlt;
-#[cfg(feature = "fibex_parser")]
+#[cfg(feature = "fibex")]
 pub mod fibex;
 pub mod filtering;
 pub mod parse;
+pub mod read;
 
 #[cfg(not(tarpaulin_include))]
 pub mod service_id;

src/parse.rs

@@ -747,7 +747,7 @@ fn dlt_payload<T: NomByteOrder>(
 
 #[inline]
 fn dbg_parsed<T: std::fmt::Debug>(_name: &str, _before: &[u8], _after: &[u8], _value: &T) {
-    // #[cfg(feature = "debug_parser")]
+    // #[cfg(feature = "debug")]
     {
         let input_len = _before.len();
         let now_len = _after.len();