Skip to content

Commit 978b0af

Browse files
authored
feat(decoder): Add support for Structured CLP IR streams but without log-level filtering. (#85)
1 parent a168a2a commit 978b0af

File tree

11 files changed

+123
-50
lines changed

11 files changed

+123
-50
lines changed

package-lock.json

Lines changed: 4 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"@mui/icons-material": "^6.1.0",
3131
"@mui/joy": "^5.0.0-beta.48",
3232
"axios": "^1.7.2",
33-
"clp-ffi-js": "^0.2.0",
33+
"clp-ffi-js": "^0.3.0",
3434
"dayjs": "^1.11.11",
3535
"monaco-editor": "^0.50.0",
3636
"react": "^18.3.1",

src/services/LogFileManager/index.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/* eslint max-lines: ["error", 400] */
22
import {
33
Decoder,
4-
DecoderOptionsType,
4+
DecoderOptions,
55
} from "../../typings/decoders";
66
import {MAX_V8_STRING_LENGTH} from "../../typings/js";
77
import {LogLevelFilter} from "../../typings/logs";
@@ -110,7 +110,7 @@ class LogFileManager {
110110
static async create (
111111
fileSrc: FileSrcType,
112112
pageSize: number,
113-
decoderOptions: DecoderOptionsType,
113+
decoderOptions: DecoderOptions,
114114
onQueryResults: (queryResults: QueryResults) => void,
115115
): Promise<LogFileManager> {
116116
const {fileName, fileData} = await loadFile(fileSrc);
@@ -138,13 +138,13 @@ class LogFileManager {
138138
static async #initDecoder (
139139
fileName: string,
140140
fileData: Uint8Array,
141-
decoderOptions: DecoderOptionsType
141+
decoderOptions: DecoderOptions
142142
): Promise<Decoder> {
143143
let decoder: Decoder;
144144
if (fileName.endsWith(".jsonl")) {
145145
decoder = new JsonlDecoder(fileData, decoderOptions);
146146
} else if (fileName.endsWith(".clp.zst")) {
147-
decoder = await ClpIrDecoder.create(fileData);
147+
decoder = await ClpIrDecoder.create(fileData, decoderOptions);
148148
} else {
149149
throw new Error(`No decoder supports ${fileName}`);
150150
}
@@ -161,7 +161,7 @@ class LogFileManager {
161161
/* Sets any formatter options that exist in the decoder's options.
162162
* @param options
163163
*/
164-
setFormatterOptions (options: DecoderOptionsType) {
164+
setFormatterOptions (options: DecoderOptions) {
165165
this.#decoder.setFormatterOptions(options);
166166
}
167167

src/services/MainWorker.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import dayjs from "dayjs";
2+
import dayjsBigIntSupport from "dayjs/plugin/bigIntSupport";
23
import dayjsTimezone from "dayjs/plugin/timezone";
34
import dayjsUtc from "dayjs/plugin/utc";
45

@@ -17,6 +18,7 @@ import LogFileManager from "./LogFileManager";
1718
/* eslint-disable import/no-named-as-default-member */
1819
dayjs.extend(dayjsUtc);
1920
dayjs.extend(dayjsTimezone);
21+
dayjs.extend(dayjsBigIntSupport);
2022
/* eslint-enable import/no-named-as-default-member */
2123

2224
/**

src/services/decoders/ClpIrDecoder.ts

Lines changed: 87 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,68 @@
1-
import clpFfiJsModuleInit, {ClpIrStreamReader} from "clp-ffi-js";
1+
import clpFfiJsModuleInit, {ClpStreamReader} from "clp-ffi-js";
2+
import {Dayjs} from "dayjs";
23

34
import {Nullable} from "../../typings/common";
45
import {
56
Decoder,
6-
DecodeResultType,
7+
DecodeResult,
8+
DecoderOptions,
79
FilteredLogEventMap,
810
LogEventCount,
911
} from "../../typings/decoders";
12+
import {Formatter} from "../../typings/formatters";
13+
import {JsonObject} from "../../typings/js";
1014
import {LogLevelFilter} from "../../typings/logs";
15+
import LogbackFormatter from "../formatters/LogbackFormatter";
16+
import {
17+
convertToDayjsTimestamp,
18+
isJsonObject,
19+
} from "./JsonlDecoder/utils";
20+
1121

22+
enum CLP_IR_STREAM_TYPE {
23+
STRUCTURED = "structured",
24+
UNSTRUCTURED = "unstructured",
25+
}
1226

1327
class ClpIrDecoder implements Decoder {
14-
#streamReader: ClpIrStreamReader;
28+
#streamReader: ClpStreamReader;
29+
30+
readonly #streamType: CLP_IR_STREAM_TYPE;
1531

16-
constructor (streamReader: ClpIrStreamReader) {
32+
#formatter: Nullable<Formatter>;
33+
34+
constructor (
35+
streamType: CLP_IR_STREAM_TYPE,
36+
streamReader: ClpStreamReader,
37+
decoderOptions: DecoderOptions
38+
) {
39+
this.#streamType = streamType;
1740
this.#streamReader = streamReader;
41+
this.#formatter = (streamType === CLP_IR_STREAM_TYPE.STRUCTURED) ?
42+
new LogbackFormatter({formatString: decoderOptions.formatString}) :
43+
null;
1844
}
1945

2046
/**
2147
* Creates a new ClpIrDecoder instance.
48+
* NOTE: `decoderOptions` only affects decode results if the stream type is
49+
* {@link CLP_IR_STREAM_TYPE.STRUCTURED}.
2250
*
2351
* @param dataArray The input data array to be passed to the decoder.
52+
* @param decoderOptions
2453
* @return The created ClpIrDecoder instance.
2554
*/
26-
static async create (dataArray: Uint8Array): Promise<ClpIrDecoder> {
55+
static async create (
56+
dataArray: Uint8Array,
57+
decoderOptions: DecoderOptions
58+
): Promise<ClpIrDecoder> {
2759
const module = await clpFfiJsModuleInit();
28-
const streamReader = new module.ClpIrStreamReader(dataArray);
29-
return new ClpIrDecoder(streamReader);
60+
const streamReader = new module.ClpStreamReader(dataArray, decoderOptions);
61+
const streamType = streamReader.getIrStreamType() === module.IrStreamType.STRUCTURED ?
62+
CLP_IR_STREAM_TYPE.STRUCTURED :
63+
CLP_IR_STREAM_TYPE.UNSTRUCTURED;
64+
65+
return new ClpIrDecoder(streamType, streamReader, decoderOptions);
3066
}
3167

3268
getEstimatedNumEvents (): number {
@@ -50,18 +86,58 @@ class ClpIrDecoder implements Decoder {
5086
};
5187
}
5288

53-
// eslint-disable-next-line class-methods-use-this
54-
setFormatterOptions (): boolean {
89+
setFormatterOptions (options: DecoderOptions): boolean {
90+
this.#formatter = new LogbackFormatter({formatString: options.formatString});
91+
5592
return true;
5693
}
5794

5895
decodeRange (
5996
beginIdx: number,
6097
endIdx: number,
6198
useFilter: boolean
62-
): Nullable<DecodeResultType[]> {
63-
return this.#streamReader.decodeRange(beginIdx, endIdx, useFilter);
99+
): Nullable<DecodeResult[]> {
100+
const results: DecodeResult[] =
101+
this.#streamReader.decodeRange(beginIdx, endIdx, useFilter);
102+
103+
if (null === this.#formatter) {
104+
if (this.#streamType === CLP_IR_STREAM_TYPE.STRUCTURED) {
105+
// eslint-disable-next-line no-warning-comments
106+
// TODO: Revisit when we allow displaying structured logs without a formatter.
107+
console.error("Formatter is not set for structured logs.");
108+
}
109+
110+
return results;
111+
}
112+
113+
for (const r of results) {
114+
const [
115+
message,
116+
timestamp,
117+
level,
118+
] = r;
119+
const dayJsTimestamp: Dayjs = convertToDayjsTimestamp(timestamp);
120+
let fields: JsonObject = {};
121+
122+
try {
123+
fields = JSON.parse(message) as JsonObject;
124+
if (false === isJsonObject(fields)) {
125+
throw new Error("Unexpected non-object.");
126+
}
127+
} catch (e) {
128+
console.error(e, message);
129+
}
130+
131+
r[0] = this.#formatter.formatLogEvent({
132+
fields: fields,
133+
level: level,
134+
timestamp: dayJsTimestamp,
135+
});
136+
}
137+
138+
return results;
64139
}
65140
}
66141

142+
67143
export default ClpIrDecoder;

src/services/decoders/JsonlDecoder/index.ts

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ import {Dayjs} from "dayjs";
33
import {Nullable} from "../../../typings/common";
44
import {
55
Decoder,
6-
DecodeResultType,
6+
DecodeResult,
7+
DecoderOptions,
78
FilteredLogEventMap,
8-
JsonlDecoderOptionsType,
99
LogEventCount,
1010
} from "../../../typings/decoders";
1111
import {Formatter} from "../../../typings/formatters";
@@ -25,7 +25,7 @@ import {
2525

2626

2727
/**
28-
* A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptionsType` for
28+
* A decoder for JSONL (JSON lines) files that contain log events. See `DecoderOptions` for
2929
* properties that are specific to log events (compared to generic JSON records).
3030
*/
3131
class JsonlDecoder implements Decoder {
@@ -49,7 +49,7 @@ class JsonlDecoder implements Decoder {
4949
* @param dataArray
5050
* @param decoderOptions
5151
*/
52-
constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) {
52+
constructor (dataArray: Uint8Array, decoderOptions: DecoderOptions) {
5353
this.#dataArray = dataArray;
5454
this.#logLevelKey = decoderOptions.logLevelKey;
5555
this.#timestampKey = decoderOptions.timestampKey;
@@ -81,7 +81,7 @@ class JsonlDecoder implements Decoder {
8181
};
8282
}
8383

84-
setFormatterOptions (options: JsonlDecoderOptionsType): boolean {
84+
setFormatterOptions (options: DecoderOptions): boolean {
8585
this.#formatter = new LogbackFormatter({formatString: options.formatString});
8686

8787
return true;
@@ -91,7 +91,7 @@ class JsonlDecoder implements Decoder {
9191
beginIdx: number,
9292
endIdx: number,
9393
useFilter: boolean,
94-
): Nullable<DecodeResultType[]> {
94+
): Nullable<DecodeResult[]> {
9595
if (useFilter && null === this.#filteredLogEventMap) {
9696
return null;
9797
}
@@ -104,7 +104,7 @@ class JsonlDecoder implements Decoder {
104104
return null;
105105
}
106106

107-
const results: DecodeResultType[] = [];
107+
const results: DecodeResult[] = [];
108108
for (let i = beginIdx; i < endIdx; i++) {
109109
// Explicit cast since typescript thinks `#filteredLogEventMap[i]` can be undefined, but
110110
// it shouldn't be since we performed a bounds check at the beginning of the method.
@@ -204,12 +204,12 @@ class JsonlDecoder implements Decoder {
204204
}
205205

206206
/**
207-
* Decodes a log event into a `DecodeResultType`.
207+
* Decodes a log event into a `DecodeResult`.
208208
*
209209
* @param logEventIdx
210210
* @return The decoded log event.
211211
*/
212-
#decodeLogEvent = (logEventIdx: number): DecodeResultType => {
212+
#decodeLogEvent = (logEventIdx: number): DecodeResult => {
213213
let timestamp: number;
214214
let message: string;
215215
let logLevel: LOG_LEVEL;

src/services/decoders/JsonlDecoder/utils.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,13 @@ const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => {
5353
* - the timestamp's value is an unsupported type.
5454
* - the timestamp's value is not a valid dayjs timestamp.
5555
*/
56-
const convertToDayjsTimestamp = (field: JsonValue | undefined): dayjs.Dayjs => {
56+
const convertToDayjsTimestamp = (field: JsonValue | bigint | undefined): dayjs.Dayjs => {
5757
// If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`.
5858
// NOTE: dayjs surprisingly thinks `undefined` is a valid date. See
5959
// https://day.js.org/docs/en/parse/now#docsNav
6060
if (("string" !== typeof field &&
61-
"number" !== typeof field) ||
61+
"number" !== typeof field &&
62+
"bigint" !== typeof field) ||
6263
"undefined" === typeof field
6364
) {
6465
// `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is

src/typings/config.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import {JsonlDecoderOptionsType} from "./decoders";
1+
import {DecoderOptions} from "./decoders";
22
import {TAB_NAME} from "./tab";
33

44

@@ -27,7 +27,7 @@ enum LOCAL_STORAGE_KEY {
2727
/* eslint-enable @typescript-eslint/prefer-literal-enum-member */
2828

2929
type ConfigMap = {
30-
[CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptionsType,
30+
[CONFIG_KEY.DECODER_OPTIONS]: DecoderOptions,
3131
[CONFIG_KEY.INITIAL_TAB_NAME]: TAB_NAME,
3232
[CONFIG_KEY.THEME]: THEME_NAME,
3333
[CONFIG_KEY.PAGE_SIZE]: number,

src/typings/decoders.ts

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,16 @@ interface LogEventCount {
88
}
99

1010
/**
11-
* Options for the JSONL decoder.
12-
*
1311
* @property formatString The format string to use to serialize records as plain text.
1412
* @property logLevelKey The key of the kv-pair that contains the log level in every record.
1513
* @property timestampKey The key of the kv-pair that contains the timestamp in every record.
1614
*/
17-
interface JsonlDecoderOptionsType {
15+
interface DecoderOptions {
1816
formatString: string,
1917
logLevelKey: string,
2018
timestampKey: string,
2119
}
2220

23-
type DecoderOptionsType = JsonlDecoderOptionsType;
24-
2521
/**
2622
* Type of the decoded log event. We use an array rather than object so that it's easier to return
2723
* results from WASM-based decoders.
@@ -31,7 +27,7 @@ type DecoderOptionsType = JsonlDecoderOptionsType;
3127
* @property level
3228
* @property number
3329
*/
34-
type DecodeResultType = [string, number, number, number];
30+
type DecodeResult = [string, bigint, number, number];
3531

3632
/**
3733
* Mapping between an index in the filtered log events collection to an index in the unfiltered log
@@ -85,7 +81,7 @@ interface Decoder {
8581
* @param options
8682
* @return Whether the options were successfully set.
8783
*/
88-
setFormatterOptions(options: DecoderOptionsType): boolean;
84+
setFormatterOptions(options: DecoderOptions): boolean;
8985

9086
/**
9187
* Decodes log events in the range `[beginIdx, endIdx)` of the filtered or unfiltered
@@ -101,15 +97,14 @@ interface Decoder {
10197
beginIdx: number,
10298
endIdx: number,
10399
useFilter: boolean
104-
): Nullable<DecodeResultType[]>;
100+
): Nullable<DecodeResult[]>;
105101
}
106102

107103
export type {
108104
ActiveLogCollectionEventIdx,
109105
Decoder,
110-
DecodeResultType,
111-
DecoderOptionsType,
106+
DecodeResult,
107+
DecoderOptions,
112108
FilteredLogEventMap,
113-
JsonlDecoderOptionsType,
114109
LogEventCount,
115110
};

0 commit comments

Comments
 (0)