Skip to content

Commit 02797fc

Browse files
authored
feat!: Add support for filtering with hierarchal and auto-generated keys. (#199)
1 parent cbb1fa2 commit 02797fc

File tree

8 files changed

+126
-40
lines changed

8 files changed

+126
-40
lines changed

package-lock.json

Lines changed: 5 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"@mui/icons-material": "^6.4.1",
3131
"@mui/joy": "^5.0.0-beta.51",
3232
"axios": "^1.7.9",
33-
"clp-ffi-js": "^0.4.0",
33+
"clp-ffi-js": "^0.5.0",
3434
"dayjs": "^1.11.13",
3535
"monaco-editor": "0.50.0",
3636
"react": "^19.0.0",

src/services/decoders/ClpIrDecoder/index.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
convertToDayjsTimestamp,
2222
isJsonObject,
2323
} from "../JsonlDecoder/utils";
24+
import {parseFilterKeys} from "../utils";
2425
import {
2526
CLP_IR_STREAM_TYPE,
2627
getStructuredIrNamespaceKeys,
@@ -42,7 +43,8 @@ class ClpIrDecoder implements Decoder {
4243
dataArray: Uint8Array,
4344
decoderOptions: DecoderOptions
4445
) {
45-
this.#streamReader = new ffiModule.ClpStreamReader(dataArray, decoderOptions);
46+
const readerOptions = parseFilterKeys(decoderOptions, true);
47+
this.#streamReader = new ffiModule.ClpStreamReader(dataArray, readerOptions);
4648
this.#streamType =
4749
this.#streamReader.getIrStreamType() === ffiModule.IrStreamType.STRUCTURED ?
4850
CLP_IR_STREAM_TYPE.STRUCTURED :

src/services/decoders/JsonlDecoder/index.ts

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@ import {
1616
LogEvent,
1717
LogLevelFilter,
1818
} from "../../../typings/logs";
19+
import {getNestedJsonValue} from "../../../utils/js";
1920
import YscopeFormatter from "../../formatters/YscopeFormatter";
2021
import {postFormatPopup} from "../../MainWorker";
22+
import {parseFilterKeys} from "../utils";
2123
import {
2224
convertToDayjsTimestamp,
2325
convertToLogLevelValue,
@@ -34,9 +36,9 @@ class JsonlDecoder implements Decoder {
3436

3537
#dataArray: Nullable<Uint8Array>;
3638

37-
#logLevelKey: string;
39+
#logLevelKeyParts: string[];
3840

39-
#timestampKey: string;
41+
#timestampKeyParts: string[];
4042

4143
#logEvents: LogEvent[] = [];
4244

@@ -52,8 +54,11 @@ class JsonlDecoder implements Decoder {
5254
*/
5355
constructor (dataArray: Uint8Array, decoderOptions: DecoderOptions) {
5456
this.#dataArray = dataArray;
55-
this.#logLevelKey = decoderOptions.logLevelKey;
56-
this.#timestampKey = decoderOptions.timestampKey;
57+
58+
const filterKeys = parseFilterKeys(decoderOptions, false);
59+
this.#logLevelKeyParts = filterKeys.logLevelKey.parts;
60+
this.#timestampKeyParts = filterKeys.timestampKey.parts;
61+
5762
this.#formatter = new YscopeFormatter({formatString: decoderOptions.formatString});
5863
if (0 === decoderOptions.formatString.length) {
5964
postFormatPopup();
@@ -165,8 +170,14 @@ class JsonlDecoder implements Decoder {
165170
if (false === isJsonObject(fields)) {
166171
throw new Error("Unexpected non-object.");
167172
}
168-
level = convertToLogLevelValue(fields[this.#logLevelKey]);
169-
timestamp = convertToDayjsTimestamp(fields[this.#timestampKey]);
173+
level = convertToLogLevelValue(getNestedJsonValue(
174+
fields,
175+
this.#logLevelKeyParts
176+
));
177+
timestamp = convertToDayjsTimestamp(getNestedJsonValue(
178+
fields,
179+
this.#timestampKeyParts
180+
));
170181
} catch (e) {
171182
if (0 === line.length) {
172183
return;

src/services/decoders/utils.ts

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import {DecoderOptions} from "../../typings/decoders";
2+
import {
3+
ParsedKey,
4+
REPLACEMENT_CHARACTER,
5+
} from "../../typings/formatters";
6+
import {
7+
EXISTING_REPLACEMENT_CHARACTER_WARNING,
8+
parseKey,
9+
replaceDoubleBacklash,
10+
UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE,
11+
} from "../formatters/YscopeFormatter/utils";
12+
13+
14+
/**
15+
* Preprocesses filter key by removing escaped backlash to facilitate simpler parsing, then parses
16+
* the key.
17+
*
18+
* @param filterKey
19+
* @return The parsed key.
20+
*/
21+
const preprocessThenParseFilterKey = (filterKey: string): ParsedKey => {
22+
if (filterKey.includes(REPLACEMENT_CHARACTER)) {
23+
console.warn(EXISTING_REPLACEMENT_CHARACTER_WARNING);
24+
}
25+
26+
return parseKey(replaceDoubleBacklash(filterKey));
27+
};
28+
29+
/**
30+
* Parses the log level key and timestamp key from the decoder options.
31+
*
32+
* @param decoderOptions
33+
* @param supportsAutoGeneratedKeys
34+
* @return An object containing the parsed log level key and timestamp key.
35+
* @throws {Error} If the keys contain reserved symbols.
36+
*/
37+
const parseFilterKeys = (decoderOptions: DecoderOptions, supportsAutoGeneratedKeys: boolean): {
38+
logLevelKey: ParsedKey;
39+
timestampKey: ParsedKey;
40+
} => {
41+
const parsedLogLevelKey = preprocessThenParseFilterKey(decoderOptions.logLevelKey);
42+
const parsedTimestampKey = preprocessThenParseFilterKey(decoderOptions.timestampKey);
43+
44+
if (false === supportsAutoGeneratedKeys &&
45+
(parsedLogLevelKey.isAutoGenerated || parsedTimestampKey.isAutoGenerated)) {
46+
throw new Error(UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE);
47+
}
48+
49+
return {
50+
logLevelKey: parsedLogLevelKey,
51+
timestampKey: parsedTimestampKey,
52+
};
53+
};
54+
55+
export {
56+
parseFilterKeys,
57+
preprocessThenParseFilterKey,
58+
};

src/services/formatters/YscopeFormatter/index.ts

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
FIELD_PLACEHOLDER_REGEX,
44
Formatter,
55
FormatterOptionsType,
6+
ParsedKey,
67
REPLACEMENT_CHARACTER,
78
YscopeFieldFormatter,
89
YscopeFieldPlaceholder,
@@ -11,10 +12,13 @@ import {LogEvent} from "../../../typings/logs";
1112
import {jsonValueToString} from "../../../utils/js";
1213
import {StructuredIrNamespaceKeys} from "../../decoders/ClpIrDecoder/utils";
1314
import {
15+
EXISTING_REPLACEMENT_CHARACTER_WARNING,
1416
getFormattedField,
17+
parseKey,
1518
removeEscapeCharacters,
1619
replaceDoubleBacklash,
1720
splitFieldPlaceholder,
21+
UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE,
1822
YSCOPE_FIELD_FORMATTER_MAP,
1923
} from "./utils";
2024

@@ -34,8 +38,7 @@ class YscopeFormatter implements Formatter {
3438
this.#structuredIrNamespaceKeys = options.structuredIrNamespaceKeys ?? null;
3539

3640
if (options.formatString.includes(REPLACEMENT_CHARACTER)) {
37-
console.warn("Unicode replacement character `U+FFFD` found in format string; " +
38-
"it will be replaced with \"\\\"");
41+
console.warn(EXISTING_REPLACEMENT_CHARACTER_WARNING);
3942
}
4043

4144
this.#processedFormatString = replaceDoubleBacklash(options.formatString);
@@ -90,8 +93,13 @@ class YscopeFormatter implements Formatter {
9093
throw Error("Field placeholder regex is invalid and does not have a capture group");
9194
}
9295

93-
const {parsedFieldName, formatterName, formatterOptions} =
94-
splitFieldPlaceholder(groupMatch, this.#structuredIrNamespaceKeys);
96+
const {fieldName, formatterName, formatterOptions} =
97+
splitFieldPlaceholder(groupMatch);
98+
99+
const parsedFieldName: ParsedKey = parseKey(fieldName);
100+
if (null === this.#structuredIrNamespaceKeys && parsedFieldName.isAutoGenerated) {
101+
throw new Error(UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE);
102+
}
95103

96104
let fieldFormatter: Nullable<YscopeFieldFormatter> = null;
97105
if (null !== formatterName) {

src/services/formatters/YscopeFormatter/utils.ts

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import {
33
AUTO_GENERATED_KEY_PREFIX,
44
COLON_REGEX,
55
DOUBLE_BACKSLASH,
6-
ParsedFieldName,
6+
ParsedKey,
77
PERIOD_REGEX,
88
REPLACEMENT_CHARACTER,
99
SINGLE_BACKSLASH,
@@ -22,6 +22,22 @@ import RoundFormatter from "./FieldFormatters/RoundFormatter";
2222
import TimestampFormatter from "./FieldFormatters/TimestampFormatter";
2323

2424

25+
/**
26+
* Error message for when a key is prefixed with the `@` symbol in the format string or
27+
* filter key for JSON logs.
28+
*/
29+
const UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE =
30+
"`@` is a reserved symbol for CLP IR logs and must be escaped with `\\` for JSONL logs.";
31+
32+
33+
/**
34+
* Warning message for when Unicode replacement character is found in format string or filter
35+
* key.
36+
*/
37+
const EXISTING_REPLACEMENT_CHARACTER_WARNING =
38+
"Unicode replacement character `U+FFFD` found in format string or filter key; " +
39+
"it will be replaced with a double backlash";
40+
2541
/**
2642
* List of currently supported field formatters.
2743
*/
@@ -81,7 +97,7 @@ const replaceDoubleBacklash = (str: string): string => {
8197

8298
/**
8399
* Retrieves fields from auto-generated or user-generated namespace of a structured IR log
84-
* event based on the prefix of the parsed key.
100+
* event based on the prefix of the parsed field name.
85101
*
86102
* @param logEvent
87103
* @param structuredIrNamespaceKeys
@@ -93,7 +109,7 @@ const replaceDoubleBacklash = (str: string): string => {
93109
const getFieldsByNamespace = (
94110
logEvent: LogEvent,
95111
structuredIrNamespaceKeys: StructuredIrNamespaceKeys,
96-
parsedFieldName: ParsedFieldName
112+
parsedFieldName: ParsedKey
97113
): JsonObject => {
98114
const namespaceKey = parsedFieldName.isAutoGenerated ?
99115
structuredIrNamespaceKeys.autoGenerated :
@@ -162,7 +178,7 @@ const validateComponent = (component: string | undefined): Nullable<string> => {
162178
* @param key The key to be parsed.
163179
* @return The parsed key.
164180
*/
165-
const parseKey = (key: string): ParsedFieldName => {
181+
const parseKey = (key: string): ParsedKey => {
166182
const isAutoGenerated = AUTO_GENERATED_KEY_PREFIX === key.charAt(0);
167183
const keyWithoutAutoPrefix = isAutoGenerated ?
168184
key.substring(1) :
@@ -176,22 +192,20 @@ const parseKey = (key: string): ParsedFieldName => {
176192
};
177193

178194
/**
179-
* Splits a field placeholder string into its components: parsed field name, formatter name, and
195+
* Splits a field placeholder string into its components: field name, formatter name, and
180196
* formatter options.
181197
*
182198
* @param placeholderString
183-
* @param structuredIrNamespaceKeys
184199
* @return - An object containing:
185-
* - parsedFieldName: The parsed field name.
200+
* - fieldName: The field name.
186201
* - formatterName: The formatter name, or `null` if not provided.
187202
* - formatterOptions: The formatter options, or `null` if not provided.
188203
* @throws {Error} If the field name could not be parsed.
189204
*/
190205
const splitFieldPlaceholder = (
191206
placeholderString: string,
192-
structuredIrNamespaceKeys: Nullable<StructuredIrNamespaceKeys>
193207
): {
194-
parsedFieldName: ParsedFieldName;
208+
fieldName: string;
195209
formatterName: Nullable<string>;
196210
formatterOptions: Nullable<string>;
197211
} => {
@@ -207,14 +221,6 @@ const splitFieldPlaceholder = (
207221
throw Error("Field name could not be parsed");
208222
}
209223

210-
const parsedFieldName: ParsedFieldName = parseKey(fieldName);
211-
if (null === structuredIrNamespaceKeys && parsedFieldName.isAutoGenerated) {
212-
throw new Error(
213-
"`@` is a reserved symbol and must be escaped with `\\` " +
214-
"for JSONL logs."
215-
);
216-
}
217-
218224
formatterName = validateComponent(formatterName);
219225
if (null !== formatterName) {
220226
formatterName = removeEscapeCharacters(formatterName);
@@ -225,15 +231,17 @@ const splitFieldPlaceholder = (
225231
formatterOptions = removeEscapeCharacters(formatterOptions);
226232
}
227233

228-
return {parsedFieldName, formatterName, formatterOptions};
234+
return {fieldName, formatterName, formatterOptions};
229235
};
230236

231237

232238
export {
239+
EXISTING_REPLACEMENT_CHARACTER_WARNING,
233240
getFormattedField,
234241
parseKey,
235242
removeEscapeCharacters,
236243
replaceDoubleBacklash,
237244
splitFieldPlaceholder,
245+
UNEXPECTED_AUTOGENERATED_SYMBOL_ERROR_MESSAGE,
238246
YSCOPE_FIELD_FORMATTER_MAP,
239247
};

src/typings/formatters.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,12 @@ interface YscopeFieldFormatterMap {
6060

6161

6262
/**
63-
* Parsed field name from YScope format string.
63+
* Parsed key from YScope format string or filter keys.
6464
*
6565
* @property isAutoGenerated whether the key is prefixed with `AUTO_GENERATED_KEY_PREFIX`.
6666
* @property parts The key split into its hierarchical components.
6767
*/
68-
type ParsedFieldName = {
68+
type ParsedKey = {
6969
isAutoGenerated: boolean;
7070
parts: string[];
7171
};
@@ -74,7 +74,7 @@ type ParsedFieldName = {
7474
* Parsed field placeholder from a YScope format string.
7575
*/
7676
type YscopeFieldPlaceholder = {
77-
parsedFieldName: ParsedFieldName;
77+
parsedFieldName: ParsedKey;
7878
fieldFormatter: Nullable<YscopeFieldFormatter>;
7979

8080
// Location of field placeholder in format string including braces.
@@ -124,7 +124,7 @@ const PERIOD_REGEX = Object.freeze(/(?<!\\)\./);
124124
export type {
125125
Formatter,
126126
FormatterOptionsType,
127-
ParsedFieldName,
127+
ParsedKey,
128128
YscopeFieldFormatter,
129129
YscopeFieldFormatterMap,
130130
YscopeFieldPlaceholder,

0 commit comments

Comments
 (0)