feat(formatter): Add YScope formatter for structured logs and remove Logback-style formatter. (#123)

Co-authored-by: Junhao Liao <[email protected]>

Author: davemarco

src/components/modals/SettingsModal/SettingsDialog.tsx

@@ -33,8 +33,12 @@ import ThemeSwitchToggle from "./ThemeSwitchToggle";
 
 const CONFIG_FORM_FIELDS = [
     {
-        helperText: "[JSON] Log messages conversion pattern. The current syntax is similar to" +
-            " Logback conversion patterns but will change in a future release.",
+        helperText: `[JSON] Log message conversion pattern: use field placeholders to insert
+        values from JSON log events. The syntax is
+        \`{<field-name>[:<formatter-name>[:<formatter-options>]]}\`, where \`field-name\` is
+        required, while \`formatter-name\` and \`formatter-options\` are optional. For example,
+        the following placeholder would format a timestamp field with name \`@timestamp\`:
+        \`{@timestamp:timestamp:YYYY-MM-DD HH\\:mm\\:ss.SSS}\`.`,
         initialValue: getConfig(CONFIG_KEY.DECODER_OPTIONS).formatString,
         label: "Decoder: Format string",
         name: LOCAL_STORAGE_KEY.DECODER_OPTIONS_FORMAT_STRING,

src/services/decoders/ClpIrDecoder.ts

@@ -12,7 +12,7 @@ import {
 import {Formatter} from "../../typings/formatters";
 import {JsonObject} from "../../typings/js";
 import {LogLevelFilter} from "../../typings/logs";
-import LogbackFormatter from "../formatters/LogbackFormatter";
+import YscopeFormatter from "../formatters/YscopeFormatter";
 import {
     convertToDayjsTimestamp,
     isJsonObject,
@@ -39,7 +39,7 @@ class ClpIrDecoder implements Decoder {
         this.#streamType = streamType;
         this.#streamReader = streamReader;
         this.#formatter = (streamType === CLP_IR_STREAM_TYPE.STRUCTURED) ?
-            new LogbackFormatter({formatString: decoderOptions.formatString}) :
+            new YscopeFormatter({formatString: decoderOptions.formatString}) :
             null;
     }
 
@@ -87,7 +87,7 @@ class ClpIrDecoder implements Decoder {
     }
 
     setFormatterOptions (options: DecoderOptions): boolean {
-        this.#formatter = new LogbackFormatter({formatString: options.formatString});
+        this.#formatter = new YscopeFormatter({formatString: options.formatString});
 
         return true;
     }

src/services/decoders/JsonlDecoder/index.ts

@@ -16,7 +16,7 @@ import {
     LogEvent,
     LogLevelFilter,
 } from "../../../typings/logs";
-import LogbackFormatter from "../../formatters/LogbackFormatter";
+import YscopeFormatter from "../../formatters/YscopeFormatter";
 import {
     convertToDayjsTimestamp,
     convertToLogLevelValue,
@@ -53,7 +53,7 @@ class JsonlDecoder implements Decoder {
         this.#dataArray = dataArray;
         this.#logLevelKey = decoderOptions.logLevelKey;
         this.#timestampKey = decoderOptions.timestampKey;
-        this.#formatter = new LogbackFormatter({formatString: decoderOptions.formatString});
+        this.#formatter = new YscopeFormatter({formatString: decoderOptions.formatString});
     }
 
     getEstimatedNumEvents (): number {
@@ -82,7 +82,7 @@ class JsonlDecoder implements Decoder {
     }
 
     setFormatterOptions (options: DecoderOptions): boolean {
-        this.#formatter = new LogbackFormatter({formatString: options.formatString});
+        this.#formatter = new YscopeFormatter({formatString: options.formatString});
 
         return true;
     }

src/services/formatters/LogbackFormatter.ts

@@ -0,0 +1,29 @@
+import {Nullable} from "../../../../typings/common";
+import {YscopeFieldFormatter} from "../../../../typings/formatters";
+import {JsonValue} from "../../../../typings/js";
+import {jsonValueToString} from "../utils";
+
+
+/**
+ * A field formatter that rounds numerical values to the nearest integer.
+ * For non-numerical values, the field's value is converted to a string then returned as-is.
+ * Options: None.
+ */
+class RoundFormatter implements YscopeFieldFormatter {
+    constructor (options: Nullable<string>) {
+        if (null !== options) {
+            throw Error(`RoundFormatter does not support options "${options}"`);
+        }
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    formatField (field: JsonValue): string {
+        if ("number" === typeof field) {
+            field = Math.round(field);
+        }
+
+        return jsonValueToString(field);
+    }
+}
+
+export default RoundFormatter;

src/services/formatters/YscopeFormatter/FieldFormatters/RoundFormatter.ts

@@ -0,0 +1,35 @@
+import {Dayjs} from "dayjs";
+
+import {Nullable} from "../../../../typings/common";
+import {YscopeFieldFormatter} from "../../../../typings/formatters";
+import {JsonValue} from "../../../../typings/js";
+import {convertToDayjsTimestamp} from "../../../decoders/JsonlDecoder/utils";
+
+
+/**
+ * A formatter for timestamp values, using a specified date-time pattern.
+ * Options: If no pattern is provided, defaults to ISO 8601 format.
+ */
+class TimestampFormatter implements YscopeFieldFormatter {
+    #dateFormat: Nullable<string> = null;
+
+    constructor (options: Nullable<string>) {
+        this.#dateFormat = options;
+    }
+
+    formatField (field: JsonValue): string {
+        // eslint-disable-next-line no-warning-comments
+        // TODO: We already parsed the timestamp during deserialization so this is perhaps
+        // inefficient. However, this field formatter can be used for multiple keys, so using
+        // the single parsed timestamp by itself would not work. Perhaps in future we can check
+        // if the key is the same as timestamp key and avoid parsing again.
+        const timestamp: Dayjs = convertToDayjsTimestamp(field);
+        if (null === this.#dateFormat) {
+            return timestamp.format();
+        }
+
+        return timestamp.format(this.#dateFormat);
+    }
+}
+
+export default TimestampFormatter;

src/services/formatters/YscopeFormatter/FieldFormatters/TimestampFormatter.ts

@@ -0,0 +1,102 @@
+import {Nullable} from "../../../typings/common";
+import {
+    FIELD_PLACEHOLDER_REGEX,
+    Formatter,
+    FormatterOptionsType,
+    REPLACEMENT_CHARACTER,
+    YscopeFieldFormatter,
+    YscopeFieldPlaceholder,
+} from "../../../typings/formatters";
+import {LogEvent} from "../../../typings/logs";
+import {
+    getFormattedField,
+    removeEscapeCharacters,
+    replaceDoubleBacklash,
+    splitFieldPlaceholder,
+    YSCOPE_FIELD_FORMATTER_MAP,
+} from "./utils";
+
+
+/**
+ * A formatter that uses a YScope format string to format log events into a string. See
+ * `YscopeFormatterOptionsType` for details about the format string.
+ */
+class YscopeFormatter implements Formatter {
+    readonly #processedFormatString: string;
+
+    #fieldPlaceholders: YscopeFieldPlaceholder[] = [];
+
+    constructor (options: FormatterOptionsType) {
+        if (options.formatString.includes(REPLACEMENT_CHARACTER)) {
+            console.warn("Unicode replacement character `U+FFFD` is found in Decoder Format" +
+            ' String, which will appear as "\\".');
+        }
+
+        this.#processedFormatString = replaceDoubleBacklash(options.formatString);
+        this.#parseFieldPlaceholder();
+    }
+
+    formatLogEvent (logEvent: LogEvent): string {
+        const formattedLogFragments: string[] = [];
+        let lastIndex = 0;
+
+        for (const fieldPlaceholder of this.#fieldPlaceholders) {
+            const formatStringFragment =
+                this.#processedFormatString.slice(lastIndex, fieldPlaceholder.range.start);
+
+            formattedLogFragments.push(removeEscapeCharacters(formatStringFragment));
+            formattedLogFragments.push(getFormattedField(logEvent, fieldPlaceholder));
+            lastIndex = fieldPlaceholder.range.end;
+        }
+
+        const remainder = this.#processedFormatString.slice(lastIndex);
+        formattedLogFragments.push(removeEscapeCharacters(remainder));
+
+        return `${formattedLogFragments.join("")}\n`;
+    }
+
+    /**
+     * Parses field placeholders in format string. For each field placeholder, creates a
+     * corresponding `YscopeFieldFormatter` using the placeholder's field name, formatter type,
+     * and formatter options. Each `YscopeFieldFormatter` is then stored on the
+     * class-level array `#fieldPlaceholders`.
+     *
+     * @throws Error if `FIELD_PLACEHOLDER_REGEX` does not contain a capture group.
+     * @throws Error if a formatter type is not supported.
+     */
+    #parseFieldPlaceholder () {
+        const placeholderPattern = new RegExp(FIELD_PLACEHOLDER_REGEX, "g");
+        const it = this.#processedFormatString.matchAll(placeholderPattern);
+        for (const match of it) {
+            // `fullMatch` includes braces and `groupMatch` excludes them.
+            const [fullMatch, groupMatch]: (string | undefined) [] = match;
+
+            if ("undefined" === typeof groupMatch) {
+                throw Error("Field placeholder regex is invalid and does not have a capture group");
+            }
+
+            const {fieldNameKeys, formatterName, formatterOptions} =
+                splitFieldPlaceholder(groupMatch);
+
+            let fieldFormatter: Nullable<YscopeFieldFormatter> = null;
+            if (null !== formatterName) {
+                const FieldFormatterConstructor = YSCOPE_FIELD_FORMATTER_MAP[formatterName];
+                if ("undefined" === typeof FieldFormatterConstructor) {
+                    throw Error(`Formatter ${formatterName} is not currently supported`);
+                }
+                fieldFormatter = new FieldFormatterConstructor(formatterOptions);
+            }
+
+            this.#fieldPlaceholders.push({
+                fieldNameKeys: fieldNameKeys,
+                fieldFormatter: fieldFormatter,
+                range: {
+                    start: match.index,
+                    end: match.index + fullMatch.length,
+                },
+            });
+        }
+    }
+}
+
+export default YscopeFormatter;