feat(remap, lua, codecs): add auto value to metric_tag_values

changelog.d/metric_tag_values_auto.feature.md

@@ -0,0 +1,5 @@
+The `metric_tag_values` option (used by the `remap` and `lua` transforms) now accepts an
+`auto` value that exposes single-value tags as strings and multi-value tags as arrays --
+preserving the underlying shape of each tag instead of forcing every tag into one form.
+
+authors: kaarolch

lib/codecs/src/decoding/format/vrl.rs

@@ -4,7 +4,7 @@ use vector_config_macros::configurable_component;
 use vector_core::{
     compile_vrl,
     config::{DataType, LogNamespace},
-    event::{Event, TargetEvents, VrlTarget},
+    event::{Event, MetricTagMode, TargetEvents, VrlTarget},
     schema,
 };
 use vrl::{
@@ -123,7 +123,7 @@ impl VrlDeserializer {
         log_namespace: LogNamespace,
     ) -> vector_common::Result<SmallVec<[Event; 1]>> {
         let mut runtime = Runtime::default();
-        let mut target = VrlTarget::new(event, self.program.info(), true);
+        let mut target = VrlTarget::new(event, self.program.info(), MetricTagMode::Full);
         match runtime.resolve(&mut target, &self.program, &self.timezone) {
             Ok(_) => match target.into_events(log_namespace) {
                 TargetEvents::One(event) => Ok(smallvec![event]),

lib/codecs/src/lib.rs

@@ -51,4 +51,25 @@ pub enum MetricTagValues {
     Single,
     /// All tags are exposed as arrays of either string or null values.
     Full,
+    /// Tag values are exposed using the shape that matches their underlying storage:
+    /// single-value tags are exposed as strings and multi-value tags are exposed as
+    /// arrays of either string or null values.
+    ///
+    /// Writes follow the same convention -- assigning a string or null to a tag stores
+    /// it as a single tag; assigning an array stores it as a multi-value tag.
+    ///
+    /// This preserves the on-the-wire shape of metrics that mix single- and multi-value
+    /// tags. Programs that consume tags in this mode must handle both string and array
+    /// shapes (for example, with `is_array(.tags.foo)` or by normalizing with `flatten`).
+    Auto,
+}
+
+impl From<MetricTagValues> for vector_core::event::MetricTagMode {
+    fn from(value: MetricTagValues) -> Self {
+        match value {
+            MetricTagValues::Single => Self::Single,
+            MetricTagValues::Full => Self::Full,
+            MetricTagValues::Auto => Self::Auto,
+        }
+    }
 }

lib/vector-core/src/event/lua/event.rs

@@ -4,10 +4,11 @@ use super::{
     super::{Event, LogEvent, Metric},
     metric::LuaMetric,
 };
+use crate::event::MetricTagMode;
 
 pub struct LuaEvent {
     pub event: Event,
-    pub metric_multi_value_tags: bool,
+    pub metric_tag_mode: MetricTagMode,
 }
 
 impl IntoLua for LuaEvent {
@@ -20,7 +21,7 @@ impl IntoLua for LuaEvent {
                 "metric",
                 LuaMetric {
                     metric,
-                    multi_value_tags: self.metric_multi_value_tags,
+                    tag_mode: self.metric_tag_mode,
                 }
                 .into_lua(lua)?,
             )?,
@@ -80,7 +81,7 @@ mod test {
                 "event",
                 LuaEvent {
                     event,
-                    metric_multi_value_tags: false,
+                    metric_tag_mode: MetricTagMode::Single,
                 },
             )
             .unwrap();

lib/vector-core/src/event/lua/metric.rs

@@ -9,16 +9,17 @@ use super::{
     },
     util::{table_to_timestamp, timestamp_to_table},
 };
+use crate::event::MetricTagMode;
 use crate::metrics::AgentDDSketch;
 
 pub struct LuaMetric {
     pub metric: Metric,
-    pub multi_value_tags: bool,
+    pub tag_mode: MetricTagMode,
 }
 
 pub struct LuaMetricTags {
     pub tags: MetricTags,
-    pub multi_value_tags: bool,
+    pub tag_mode: MetricTagMode,
 }
 
 impl IntoLua for MetricKind {
@@ -106,20 +107,44 @@ impl FromLua for MetricTags {
 
 impl IntoLua for LuaMetricTags {
     fn into_lua(self, lua: &Lua) -> LuaResult<LuaValue> {
-        if self.multi_value_tags {
-            Ok(LuaValue::Table(lua.create_table_from(
+        match self.tag_mode {
+            MetricTagMode::Full => Ok(LuaValue::Table(lua.create_table_from(
                 self.tags.0.into_iter().map(|(key, value)| {
                     let value: Vec<_> = value
                         .into_iter()
                         .filter_map(|tag_value| tag_value.into_option().into_lua(lua).ok())
                         .collect();
                     (key, value)
                 }),
-            )?))
-        } else {
-            Ok(LuaValue::Table(
+            )?)),
+            MetricTagMode::Single => Ok(LuaValue::Table(
                 lua.create_table_from(self.tags.iter_single())?,
-            ))
+            )),
+            // `Auto` exposes single-value tags as scalars and multi-value tags
+            // as arrays. Only a 1-element set scalarises -- empty sets keep
+            // the array shape so a noop Auto round-trip preserves them.
+            // (`as_single()` is unsuitable here because it deliberately
+            // collapses multi-value sets to their last value to power
+            // `Single` mode.)
+            MetricTagMode::Auto => {
+                let table = lua.create_table()?;
+                for (key, value) in self.tags.0 {
+                    if value.len() == 1 {
+                        let scalar =
+                            value.into_iter().next().and_then(TagValue::into_option);
+                        table.raw_set(key, scalar)?;
+                    } else {
+                        let arr: Vec<_> = value
+                            .into_iter()
+                            .filter_map(|tag_value| {
+                                tag_value.into_option().into_lua(lua).ok()
+                            })
+                            .collect();
+                        table.raw_set(key, arr)?;
+                    }
+                }
+                Ok(LuaValue::Table(table))
+            }
         }
     }
 }
@@ -144,7 +169,7 @@ impl IntoLua for LuaMetric {
                 "tags",
                 LuaMetricTags {
                     tags,
-                    multi_value_tags: self.multi_value_tags,
+                    tag_mode: self.tag_mode,
                 },
             )?;
         }
@@ -350,16 +375,10 @@ mod test {
 
     use super::*;
 
-    fn assert_metric(metric: Metric, multi_value_tags: bool, assertions: Vec<&'static str>) {
+    fn assert_metric(metric: Metric, tag_mode: MetricTagMode, assertions: Vec<&'static str>) {
         let lua = Lua::new();
         lua.globals()
-            .set(
-                "metric",
-                LuaMetric {
-                    metric,
-                    multi_value_tags,
-                },
-            )
+            .set("metric", LuaMetric { metric, tag_mode })
             .unwrap();
 
         for assertion in assertions {
@@ -389,7 +408,7 @@ mod test {
 
         assert_metric(
             metric.clone(),
-            false,
+            MetricTagMode::Single,
             vec![
                 "type(metric) == 'table'",
                 "metric.name == 'example counter'",
@@ -410,7 +429,7 @@ mod test {
         );
         assert_metric(
             metric,
-            true,
+            MetricTagMode::Full,
             vec![
                 "type(metric) == 'table'",
                 "metric.name == 'example counter'",
@@ -448,7 +467,7 @@ mod test {
 
         assert_metric(
             metric,
-            true,
+            MetricTagMode::Full,
             vec![
                 "type(metric.tags) == 'table'",
                 "metric.tags['example tag'][1] == 'a'",
@@ -457,6 +476,65 @@ mod test {
         );
     }
 
+    /// `Auto` exposes single-value tags as scalars and multi-value tags as
+    /// arrays, matching how each shape would naturally appear in Lua.
+    #[test]
+    fn auto_mode_preserves_tag_shapes() {
+        let mut tags = BTreeMap::new();
+        tags.insert(
+            "single".to_string(),
+            TagValueSet::from(vec!["one".to_string()]),
+        );
+        tags.insert(
+            "multi".to_string(),
+            TagValueSet::from(vec!["a".to_string(), "b".to_string()]),
+        );
+        let metric = Metric::new(
+            "example counter",
+            MetricKind::Incremental,
+            MetricValue::Counter { value: 1.0 },
+        )
+        .with_tags(Some(MetricTags(tags)));
+
+        assert_metric(
+            metric,
+            MetricTagMode::Auto,
+            vec![
+                "type(metric.tags) == 'table'",
+                "metric.tags['single'] == 'one'",
+                "type(metric.tags['multi']) == 'table'",
+                "metric.tags['multi'][1] == 'a'",
+                "metric.tags['multi'][2] == 'b'",
+            ],
+        );
+    }
+
+    /// `Auto` keeps an empty tag set as an array (Lua table). Without this,
+    /// an empty multi-value tag would silently morph into `nil` (and a Lua
+    /// `tags['empty'] = ...` assignment would then need to know the original
+    /// shape to round-trip correctly).
+    #[test]
+    fn auto_mode_empty_set_stays_array() {
+        let mut tags = BTreeMap::new();
+        tags.insert("empty".to_string(), TagValueSet::default());
+        let metric = Metric::new(
+            "example counter",
+            MetricKind::Incremental,
+            MetricValue::Counter { value: 1.0 },
+        )
+        .with_tags(Some(MetricTags(tags)));
+
+        assert_metric(
+            metric,
+            MetricTagMode::Auto,
+            vec![
+                "type(metric.tags) == 'table'",
+                "type(metric.tags['empty']) == 'table'",
+                "#metric.tags['empty'] == 0",
+            ],
+        );
+    }
+
     #[test]
     fn into_lua_counter_minimal() {
         let metric = Metric::new(
@@ -467,10 +545,10 @@ mod test {
             },
         );
 
-        for multi_value_tags in [false, true] {
+        for tag_mode in [MetricTagMode::Single, MetricTagMode::Full, MetricTagMode::Auto] {
             assert_metric(
                 metric.clone(),
-                multi_value_tags,
+                tag_mode,
                 vec![
                     "metric.timestamp == nil",
                     "metric.tags == nil",
@@ -490,7 +568,7 @@ mod test {
         );
         assert_metric(
             metric,
-            false,
+            MetricTagMode::Single,
             vec!["metric.gauge.value == 1.6180339", "metric.counter == nil"],
         );
     }
@@ -508,7 +586,7 @@ mod test {
         );
         assert_metric(
             metric,
-            false,
+            MetricTagMode::Single,
             vec![
                 "type(metric.set) == 'table'",
                 "type(metric.set.values) == 'table'",
@@ -531,7 +609,7 @@ mod test {
         );
         assert_metric(
             metric,
-            false,
+            MetricTagMode::Single,
             vec![
                 "type(metric.distribution) == 'table'",
                 "#metric.distribution.values == 2",
@@ -557,7 +635,7 @@ mod test {
         );
         assert_metric(
             metric,
-            false,
+            MetricTagMode::Single,
             vec![
                 "type(metric.aggregated_histogram) == 'table'",
                 "#metric.aggregated_histogram.buckets == 4",
@@ -588,7 +666,7 @@ mod test {
 
         assert_metric(
             metric,
-            false,
+            MetricTagMode::Single,
             vec![
                 "type(metric.aggregated_summary) == 'table'",
                 "#metric.aggregated_summary.quantiles == 7",

lib/vector-core/src/event/metric_tag_mode.rs

@@ -0,0 +1,27 @@
+//! How metric tags are exposed to and accepted from VRL or Lua.
+//!
+//! This enum lives in its own always-compiled module (rather than inside
+//! `vrl_target`, which is gated on the `vrl` feature) so that the `lua`
+//! feature can depend on the same type without being forced to also enable
+//! `vrl`. The `vrl_target` and `lua` modules both re-export it.
+//!
+//! It mirrors `codecs::MetricTagValues`, but lives in `vector-core` so that
+//! the crate dependency direction (`codecs -> vector-core`) is preserved.
+//! Callers at the `codecs::MetricTagValues` boundary translate using the
+//! `From<MetricTagValues>` impl on the codecs side.
+
+/// How metric tags are exposed to and accepted from VRL or Lua.
+#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
+pub enum MetricTagMode {
+    /// Tags are exposed as single strings (last value wins for multi-value
+    /// tags); writes always produce single-value tags.
+    #[default]
+    Single,
+    /// Tags are always exposed as arrays; writes always produce multi-value
+    /// tags regardless of whether the assigned value is scalar or array.
+    Full,
+    /// Tags are exposed using their underlying shape: single-value tags as
+    /// strings, multi-value tags as arrays. Writes follow the same rule --
+    /// scalar values produce single tags, arrays produce multi-value tags.
+    Auto,
+}

lib/vector-core/src/event/mod.rs

@@ -17,6 +17,7 @@ use vector_common::{
     EventDataEq, byte_size_of::ByteSizeOf, config::ComponentKey, finalization,
     internal_event::TaggedEventsSent, json_size::JsonSize, request_metadata::GetEventCountTags,
 };
+pub use metric_tag_mode::MetricTagMode;
 pub use vrl::value::{KeyString, ObjectMap, Value};
 #[cfg(feature = "vrl")]
 pub use vrl_target::{TargetEvents, VrlTarget};
@@ -34,6 +35,7 @@ pub mod lua;
 pub mod merge_state;
 mod metadata;
 pub mod metric;
+mod metric_tag_mode;
 pub mod proto;
 mod r#ref;
 mod ser;