Skip to content

Commit 4998c01

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add support for Array Processor in Logs Pipelines (#2437)
Co-authored-by: ci.datadog-api-spec <[email protected]>
1 parent e2e6a9f commit 4998c01

File tree

24 files changed

+1103
-0
lines changed

24 files changed

+1103
-0
lines changed

‎.generator/schemas/v1/openapi.yaml

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5163,6 +5163,143 @@ components:
51635163
type: string
51645164
x-enum-varnames:
51655165
- ARITHMETIC_PROCESSOR
5166+
LogsArrayProcessor:
5167+
description: 'A processor for extracting, aggregating, or transforming values
5168+
from JSON arrays within your logs.
5169+
5170+
Supported operations are:
5171+
5172+
- Select value from matching element
5173+
5174+
- Compute array length
5175+
5176+
- Append a value to an array'
5177+
properties:
5178+
is_enabled:
5179+
default: false
5180+
description: Whether or not the processor is enabled.
5181+
type: boolean
5182+
name:
5183+
description: Name of the processor.
5184+
type: string
5185+
operation:
5186+
$ref: '#/components/schemas/LogsArrayProcessorOperation'
5187+
type:
5188+
$ref: '#/components/schemas/LogsArrayProcessorType'
5189+
required:
5190+
- operation
5191+
- type
5192+
type: object
5193+
LogsArrayProcessorOperation:
5194+
description: Configuration of the array processor operation to perform.
5195+
oneOf:
5196+
- $ref: '#/components/schemas/LogsArrayProcessorOperationAppend'
5197+
- $ref: '#/components/schemas/LogsArrayProcessorOperationLength'
5198+
- $ref: '#/components/schemas/LogsArrayProcessorOperationSelect'
5199+
LogsArrayProcessorOperationAppend:
5200+
description: Operation that appends a value to a target array attribute.
5201+
properties:
5202+
preserve_source:
5203+
default: true
5204+
description: Remove or preserve the remapped source element.
5205+
type: boolean
5206+
source:
5207+
description: Attribute path containing the value to append.
5208+
example: network.client.ip
5209+
type: string
5210+
target:
5211+
description: Attribute path of the array to append to.
5212+
example: sourceIps
5213+
type: string
5214+
type:
5215+
$ref: '#/components/schemas/LogsArrayProcessorOperationAppendType'
5216+
required:
5217+
- type
5218+
- source
5219+
- target
5220+
type: object
5221+
LogsArrayProcessorOperationAppendType:
5222+
description: Operation type.
5223+
enum:
5224+
- append
5225+
example: append
5226+
type: string
5227+
x-enum-varnames:
5228+
- APPEND
5229+
LogsArrayProcessorOperationLength:
5230+
description: Operation that computes the length of a `source` array and stores
5231+
the result in the `target` attribute.
5232+
properties:
5233+
source:
5234+
description: Attribute path of the array to measure.
5235+
example: tags
5236+
type: string
5237+
target:
5238+
description: Attribute that receives the computed length.
5239+
example: tagCount
5240+
type: string
5241+
type:
5242+
$ref: '#/components/schemas/LogsArrayProcessorOperationLengthType'
5243+
required:
5244+
- type
5245+
- source
5246+
- target
5247+
type: object
5248+
LogsArrayProcessorOperationLengthType:
5249+
description: Operation type.
5250+
enum:
5251+
- length
5252+
example: length
5253+
type: string
5254+
x-enum-varnames:
5255+
- LENGTH
5256+
LogsArrayProcessorOperationSelect:
5257+
description: Operation that finds an object in a `source` array using a `filter`,
5258+
and then extracts a specific value into the `target` attribute.
5259+
properties:
5260+
filter:
5261+
description: Filter condition expressed as `key:value` used to find the
5262+
matching element.
5263+
example: name:Referrer
5264+
type: string
5265+
source:
5266+
description: Attribute path of the array to search into.
5267+
example: httpRequest.headers
5268+
type: string
5269+
target:
5270+
description: Attribute that receives the extracted value.
5271+
example: referrer
5272+
type: string
5273+
type:
5274+
$ref: '#/components/schemas/LogsArrayProcessorOperationSelectType'
5275+
value_to_extract:
5276+
description: Key of the value to extract from the matching element.
5277+
example: value
5278+
type: string
5279+
required:
5280+
- type
5281+
- source
5282+
- target
5283+
- filter
5284+
- value_to_extract
5285+
type: object
5286+
LogsArrayProcessorOperationSelectType:
5287+
description: Operation type.
5288+
enum:
5289+
- select
5290+
example: select
5291+
type: string
5292+
x-enum-varnames:
5293+
- SELECT
5294+
LogsArrayProcessorType:
5295+
default: array-processor
5296+
description: Type of logs array processor.
5297+
enum:
5298+
- array-processor
5299+
example: array-processor
5300+
type: string
5301+
x-enum-varnames:
5302+
- ARRAY_PROCESSOR
51665303
LogsAttributeRemapper:
51675304
description: 'The remapper processor remaps any source attribute(s) or tag to
51685305
another target attribute or tag.
@@ -6060,6 +6197,7 @@ components:
60606197
- $ref: '#/components/schemas/ReferenceTableLogsLookupProcessor'
60616198
- $ref: '#/components/schemas/LogsTraceRemapper'
60626199
- $ref: '#/components/schemas/LogsSpanRemapper'
6200+
- $ref: '#/components/schemas/LogsArrayProcessor'
60636201
LogsQueryCompute:
60646202
description: Define computation for a log query.
60656203
properties:

‎cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/frozen.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2025-06-30T15:45:40.994Z"

‎cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-returns-OK-response_2598677476/recording.har

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
{
2+
"log": {
3+
"_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Append Operation returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "69ee9de755f2e0b61cbbe11dbe56dc06",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 247,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 574,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArrayAppend\",\"processors\":[{\"is_enabled\":true,\"name\":\"append_ip_to_array\",\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v1/logs/config/pipelines"
39+
},
40+
"response": {
41+
"bodySize": 359,
42+
"content": {
43+
"mimeType": "application/json",
44+
"size": 359,
45+
"text": "{\"id\":\"s_cPqdnkQVaU6PwbPPt2ZQ\",\"type\":\"pipeline\",\"name\":\"testPipelineArrayAppend\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"append_ip_to_array\",\"is_enabled\":true,\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"preserve_source\":true,\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/json"
52+
}
53+
],
54+
"headersSize": 702,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2025-06-30T15:45:41.001Z",
61+
"time": 398
62+
},
63+
{
64+
"_id": "2d231c72b566a1746dfa7066c5e6b025",
65+
"_order": 0,
66+
"cache": {},
67+
"request": {
68+
"bodySize": 0,
69+
"cookies": [],
70+
"headers": [
71+
{
72+
"_fromType": "array",
73+
"name": "accept",
74+
"value": "*/*"
75+
}
76+
],
77+
"headersSize": 533,
78+
"httpVersion": "HTTP/1.1",
79+
"method": "DELETE",
80+
"queryString": [],
81+
"url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/s_cPqdnkQVaU6PwbPPt2ZQ"
82+
},
83+
"response": {
84+
"bodySize": 3,
85+
"content": {
86+
"mimeType": "application/json",
87+
"size": 3,
88+
"text": "{}\n"
89+
},
90+
"cookies": [],
91+
"headers": [
92+
{
93+
"name": "content-type",
94+
"value": "application/json"
95+
}
96+
],
97+
"headersSize": 677,
98+
"httpVersion": "HTTP/1.1",
99+
"redirectURL": "",
100+
"status": 200,
101+
"statusText": "OK"
102+
},
103+
"startedDateTime": "2025-06-30T15:45:41.406Z",
104+
"time": 432
105+
}
106+
],
107+
"pages": [],
108+
"version": "1.2"
109+
}
110+
}

‎cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/frozen.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2025-06-30T15:45:41.844Z"

‎cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-false-return_3111560159/recording.har

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
{
2+
"log": {
3+
"_recordingName": "Logs Pipelines/Create a pipeline with Array Processor Append Operation with preserve_source false returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "c5d4e470e0dfe8eb6dfd55ebe4d07d64",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 290,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 574,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testPipelineArrayAppendNoPreserve\",\"processors\":[{\"is_enabled\":true,\"name\":\"append_ip_and_remove_source\",\"operation\":{\"preserve_source\":false,\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v1/logs/config/pipelines"
39+
},
40+
"response": {
41+
"bodySize": 379,
42+
"content": {
43+
"mimeType": "application/json",
44+
"size": 379,
45+
"text": "{\"id\":\"B91fO94kQnCeZ4bOoZQOWg\",\"type\":\"pipeline\",\"name\":\"testPipelineArrayAppendNoPreserve\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"append_ip_and_remove_source\",\"is_enabled\":true,\"operation\":{\"source\":\"network.client.ip\",\"target\":\"sourceIps\",\"preserve_source\":false,\"type\":\"append\"},\"type\":\"array-processor\"}],\"tags\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/json"
52+
}
53+
],
54+
"headersSize": 702,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2025-06-30T15:45:41.847Z",
61+
"time": 409
62+
},
63+
{
64+
"_id": "b152c85178b71b06e1b12177b9404b61",
65+
"_order": 0,
66+
"cache": {},
67+
"request": {
68+
"bodySize": 0,
69+
"cookies": [],
70+
"headers": [
71+
{
72+
"_fromType": "array",
73+
"name": "accept",
74+
"value": "*/*"
75+
}
76+
],
77+
"headersSize": 533,
78+
"httpVersion": "HTTP/1.1",
79+
"method": "DELETE",
80+
"queryString": [],
81+
"url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/B91fO94kQnCeZ4bOoZQOWg"
82+
},
83+
"response": {
84+
"bodySize": 3,
85+
"content": {
86+
"mimeType": "application/json",
87+
"size": 3,
88+
"text": "{}\n"
89+
},
90+
"cookies": [],
91+
"headers": [
92+
{
93+
"name": "content-type",
94+
"value": "application/json"
95+
}
96+
],
97+
"headersSize": 677,
98+
"httpVersion": "HTTP/1.1",
99+
"redirectURL": "",
100+
"status": 200,
101+
"statusText": "OK"
102+
},
103+
"startedDateTime": "2025-06-30T15:45:42.260Z",
104+
"time": 391
105+
}
106+
],
107+
"pages": [],
108+
"version": "1.2"
109+
}
110+
}

‎cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Array-Processor-Append-Operation-with-preserve_source-true-returns-_558178392/frozen.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2025-06-30T15:45:42.655Z"

0 commit comments

Comments
 (0)