feat!: Add support for viewing auto-generated keys in Structured IR files. (#192)

Author: davemarco

package-lock.json

@@ -30,7 +30,7 @@
     "@mui/icons-material": "^6.4.1",
     "@mui/joy": "^5.0.0-beta.51",
     "axios": "^1.7.9",
-    "clp-ffi-js": "^0.3.4",
+    "clp-ffi-js": "^0.4.0",
     "dayjs": "^1.11.13",
     "monaco-editor": "0.50.0",
     "react": "^19.0.0",
@@ -50,11 +50,11 @@
     "copy-webpack-plugin": "^12.0.2",
     "css-loader": "^7.1.2",
     "eslint-config-yscope": "latest",
-    "html-webpack-plugin": "^5.6.3",
     "globals": "^15.14.0",
+    "html-webpack-plugin": "^5.6.3",
     "jest": "^29.7.0",
-    "mini-css-extract-plugin": "^2.9.2",
     "jest-environment-jsdom": "^29.7.0",
+    "mini-css-extract-plugin": "^2.9.2",
     "monaco-editor-webpack-plugin": "^7.1.0",
     "npm-run-all": "^4.1.5",
     "prettier": "^3.4.2",

package.json

@@ -1,46 +1,59 @@
-import clpFfiJsModuleInit, {ClpStreamReader} from "clp-ffi-js";
+import clpFfiJsModuleInit, {
+    ClpStreamReader,
+    MainModule,
+} from "clp-ffi-js";
 import {Dayjs} from "dayjs";
 
-import {Nullable} from "../../typings/common";
+import {Nullable} from "../../../typings/common";
 import {
     Decoder,
     DecodeResult,
     DecoderOptions,
     FilteredLogEventMap,
     LogEventCount,
-} from "../../typings/decoders";
-import {Formatter} from "../../typings/formatters";
-import {JsonObject} from "../../typings/js";
-import {LogLevelFilter} from "../../typings/logs";
-import YscopeFormatter from "../formatters/YscopeFormatter";
-import {postFormatPopup} from "../MainWorker";
+} from "../../../typings/decoders";
+import {Formatter} from "../../../typings/formatters";
+import {JsonObject} from "../../../typings/js";
+import {LogLevelFilter} from "../../../typings/logs";
+import YscopeFormatter from "../../formatters/YscopeFormatter";
+import {postFormatPopup} from "../../MainWorker";
 import {
     convertToDayjsTimestamp,
     isJsonObject,
-} from "./JsonlDecoder/utils";
-
+} from "../JsonlDecoder/utils";
+import {
+    CLP_IR_STREAM_TYPE,
+    getStructuredIrNamespaceKeys,
+    StructuredIrNamespaceKeys,
+} from "./utils";
 
-enum CLP_IR_STREAM_TYPE {
-    STRUCTURED = "structured",
-    UNSTRUCTURED = "unstructured",
-}
 
 class ClpIrDecoder implements Decoder {
     #streamReader: ClpStreamReader;
 
     readonly #streamType: CLP_IR_STREAM_TYPE;
 
+    readonly #structuredIrNamespaceKeys: StructuredIrNamespaceKeys;
+
     #formatter: Nullable<Formatter> = null;
 
     constructor (
-        streamType: CLP_IR_STREAM_TYPE,
-        streamReader: ClpStreamReader,
+        ffiModule: MainModule,
+        dataArray: Uint8Array,
         decoderOptions: DecoderOptions
     ) {
-        this.#streamType = streamType;
-        this.#streamReader = streamReader;
-        if (streamType === CLP_IR_STREAM_TYPE.STRUCTURED) {
-            this.#formatter = new YscopeFormatter({formatString: decoderOptions.formatString});
+        this.#streamReader = new ffiModule.ClpStreamReader(dataArray, decoderOptions);
+        this.#streamType =
+            this.#streamReader.getIrStreamType() === ffiModule.IrStreamType.STRUCTURED ?
+                CLP_IR_STREAM_TYPE.STRUCTURED :
+                CLP_IR_STREAM_TYPE.UNSTRUCTURED;
+        this.#structuredIrNamespaceKeys = getStructuredIrNamespaceKeys(ffiModule);
+
+        if (this.#streamType === CLP_IR_STREAM_TYPE.STRUCTURED) {
+            this.#formatter = new YscopeFormatter({
+                formatString: decoderOptions.formatString,
+                structuredIrNamespaceKeys: this.#structuredIrNamespaceKeys,
+            });
             if (0 === decoderOptions.formatString.length) {
                 postFormatPopup();
             }
@@ -61,12 +74,7 @@ class ClpIrDecoder implements Decoder {
         decoderOptions: DecoderOptions
     ): Promise<ClpIrDecoder> {
         const module = await clpFfiJsModuleInit();
-        const streamReader = new module.ClpStreamReader(dataArray, decoderOptions);
-        const streamType = streamReader.getIrStreamType() === module.IrStreamType.STRUCTURED ?
-            CLP_IR_STREAM_TYPE.STRUCTURED :
-            CLP_IR_STREAM_TYPE.UNSTRUCTURED;
-
-        return new ClpIrDecoder(streamType, streamReader, decoderOptions);
+        return new ClpIrDecoder(module, dataArray, decoderOptions);
     }
 
     getEstimatedNumEvents (): number {
@@ -91,7 +99,10 @@ class ClpIrDecoder implements Decoder {
     }
 
     setFormatterOptions (options: DecoderOptions): boolean {
-        this.#formatter = new YscopeFormatter({formatString: options.formatString});
+        this.#formatter = new YscopeFormatter({
+            formatString: options.formatString,
+            structuredIrNamespaceKeys: this.#structuredIrNamespaceKeys,
+        });
 
         return true;
     }

src/services/decoders/ClpIrDecoder.ts renamed to src/services/decoders/ClpIrDecoder/index.ts

@@ -0,0 +1,46 @@
+import {MainModule} from "clp-ffi-js";
+
+
+enum CLP_IR_STREAM_TYPE {
+    STRUCTURED = "structured",
+    UNSTRUCTURED = "unstructured",
+}
+
+/**
+ * Keys that delineate the auto-generated and user-generated namespaces in structured IR log events.
+ */
+interface StructuredIrNamespaceKeys {
+    autoGenerated: string;
+    userGenerated: string;
+}
+
+/**
+ * Retrieves the structured IR namespace keys from the "clp-ffi-js" module.
+ *
+ * @param ffiModule The main module from "clp-ffi-js".
+ * @return
+ * @throws {Error} If the keys are of invalid type.
+ */
+const getStructuredIrNamespaceKeys = (ffiModule: MainModule): StructuredIrNamespaceKeys => {
+    const {MERGED_KV_PAIRS_AUTO_GENERATED_KEY, MERGED_KV_PAIRS_USER_GENERATED_KEY} = ffiModule;
+
+    if ("string" !== typeof MERGED_KV_PAIRS_AUTO_GENERATED_KEY) {
+        throw new Error("Invalid type for MERGED_KV_PAIRS_AUTO_GENERATED_KEY.");
+    }
+
+    if ("string" !== typeof MERGED_KV_PAIRS_USER_GENERATED_KEY) {
+        throw new Error("Invalid type for MERGED_KV_PAIRS_USER_GENERATED_KEY.");
+    }
+
+    return {
+        autoGenerated: MERGED_KV_PAIRS_AUTO_GENERATED_KEY,
+        userGenerated: MERGED_KV_PAIRS_USER_GENERATED_KEY,
+    };
+};
+
+
+export type {StructuredIrNamespaceKeys};
+export {
+    CLP_IR_STREAM_TYPE,
+    getStructuredIrNamespaceKeys,
+};

src/services/decoders/ClpIrDecoder/utils.ts

@@ -1,7 +1,7 @@
 import {Nullable} from "../../../../typings/common";
 import {YscopeFieldFormatter} from "../../../../typings/formatters";
 import {JsonValue} from "../../../../typings/js";
-import {jsonValueToString} from "../utils";
+import {jsonValueToString} from "../../../../utils/js";
 
 
 /**

src/services/formatters/YscopeFormatter/FieldFormatters/RoundFormatter.ts

@@ -8,9 +8,10 @@ import {
     YscopeFieldPlaceholder,
 } from "../../../typings/formatters";
 import {LogEvent} from "../../../typings/logs";
+import {jsonValueToString} from "../../../utils/js";
+import {StructuredIrNamespaceKeys} from "../../decoders/ClpIrDecoder/utils";
 import {
     getFormattedField,
-    jsonValueToString,
     removeEscapeCharacters,
     replaceDoubleBacklash,
     splitFieldPlaceholder,
@@ -25,12 +26,16 @@ import {
 class YscopeFormatter implements Formatter {
     readonly #processedFormatString: string;
 
+    readonly #structuredIrNamespaceKeys: Nullable<StructuredIrNamespaceKeys>;
+
     #fieldPlaceholders: YscopeFieldPlaceholder[] = [];
 
     constructor (options: FormatterOptionsType) {
+        this.#structuredIrNamespaceKeys = options.structuredIrNamespaceKeys ?? null;
+
         if (options.formatString.includes(REPLACEMENT_CHARACTER)) {
-            console.warn("Unicode replacement character `U+FFFD` is found in Decoder Format" +
-            ' String, which will appear as "\\".');
+            console.warn("Unicode replacement character `U+FFFD` found in format string; " +
+                         "it will be replaced with \"\\\"");
         }
 
         this.#processedFormatString = replaceDoubleBacklash(options.formatString);
@@ -51,7 +56,11 @@ class YscopeFormatter implements Formatter {
                 this.#processedFormatString.slice(lastIndex, fieldPlaceholder.range.start);
 
             formattedLogFragments.push(removeEscapeCharacters(formatStringFragment));
-            formattedLogFragments.push(getFormattedField(logEvent, fieldPlaceholder));
+            formattedLogFragments.push(getFormattedField(
+                this.#structuredIrNamespaceKeys,
+                logEvent,
+                fieldPlaceholder
+            ));
             lastIndex = fieldPlaceholder.range.end;
         }
 
@@ -62,9 +71,9 @@ class YscopeFormatter implements Formatter {
     }
 
     /**
-     * Parses field placeholders in format string. For each field placeholder, creates a
-     * corresponding `YscopeFieldFormatter` using the placeholder's field name, formatter type,
-     * and formatter options. Each `YscopeFieldFormatter` is then stored on the
+     * Parses field placeholders in format string. For each field, creates a corresponding
+     * `YscopeFieldPlaceholder` using the placeholder's parsed field name, formatter type,
+     * and formatter options. Each `YscopeFieldPlaceholder` is then stored on the
      * class-level array `#fieldPlaceholders`.
      *
      * @throws Error if `FIELD_PLACEHOLDER_REGEX` does not contain a capture group.
@@ -81,8 +90,8 @@ class YscopeFormatter implements Formatter {
                 throw Error("Field placeholder regex is invalid and does not have a capture group");
             }
 
-            const {fieldNameKeys, formatterName, formatterOptions} =
-                splitFieldPlaceholder(groupMatch);
+            const {parsedFieldName, formatterName, formatterOptions} =
+                splitFieldPlaceholder(groupMatch, this.#structuredIrNamespaceKeys);
 
             let fieldFormatter: Nullable<YscopeFieldFormatter> = null;
             if (null !== formatterName) {
@@ -94,7 +103,7 @@ class YscopeFormatter implements Formatter {
             }
 
             this.#fieldPlaceholders.push({
-                fieldNameKeys: fieldNameKeys,
+                parsedFieldName: parsedFieldName,
                 fieldFormatter: fieldFormatter,
                 range: {
                     start: match.index,