Skip to content

Commit ac1c1d6

Browse files
will-break-itwill-break-it
authored
data: add trace schema for haskell & rust (#194)
* data: add trace schema for haskell & rust * data: add example logs for all events for make validate * data: simplify haskell trace schema * data: remove verbose extract of message kind * data: updated rust schema * data: composed rust message & action types * Trace Schema Changes (#202) * fix: add cpu_time_s field to track total CPU time across cores * fix: consistent naming of RBs across both simulations * refactor: naming of block variants * - refactor(hs) - separate block lifecycle actions from network actions - use shorter block type names (IB/EB/RB/VT) * fix(hs): remove node_name * fix: remove duration_s * fix: remove redundant block kind; tag -> type * fix(hs-schema): add endorse_blocks; payload_bytes * fix(hs-schema): add slot number to base block evnet * fix(hs-schema): remove duplicate payload_bytes for sent & receive events
1 parent c5aaee8 commit ac1c1d6

7 files changed

+816
-3
lines changed

‎data/simulation/Makefile

+40-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
all: config.schema.json topology.schema.json
1+
all: config.schema.json topology.schema.json trace.haskell.schema.json trace.rust.schema.json
22

33
config.schema.json: config.d.ts
44
npx typescript-json-schema config.d.ts Config > config.schema.json
@@ -8,7 +8,44 @@ topology.schema.json: topology.d.ts
88
npx typescript-json-schema topology.d.ts Topology > topology.schema.json
99
npx prettier -w topology.schema.json
1010

11+
trace.haskell.schema.json: trace.haskell.d.ts
12+
npx typescript-json-schema trace.haskell.d.ts HaskellTraceEvent > trace.haskell.schema.json
13+
npx prettier -w trace.haskell.schema.json
14+
15+
trace.rust.schema.json: trace.rust.d.ts
16+
npx typescript-json-schema trace.rust.d.ts RustTraceEvent > trace.rust.schema.json
17+
npx prettier -w trace.rust.schema.json
18+
1119
.PHONY: validate
1220
validate:
13-
npx pajv -m schema.schema.json -s config.schema.json -d config.default.yaml
14-
npx pajv -m schema.schema.json -s topology.schema.json -d topology-dense-52.yaml
21+
@echo "Validating config..."
22+
@npx pajv -m schema.schema.json -s config.schema.json -d config.default.yaml > /dev/null && echo "config.default.yaml valid" || exit 1
23+
@echo "Validating topology..."
24+
@npx pajv -m schema.schema.json -s topology.schema.json -d topology-dense-52.yaml > /dev/null && echo "topology-dense-52.yaml valid" || exit 1
25+
@echo "Validating Haskell trace events..."
26+
@line_num=0; \
27+
while IFS= read -r line; do \
28+
line_num=$$((line_num + 1)); \
29+
echo "$$line" > temp.json; \
30+
if ! npx ajv-cli validate -s trace.haskell.schema.json -d temp.json 2>/dev/null >/dev/null; then \
31+
echo "Error in example.haskell.jsonl at line $$line_num:"; \
32+
echo "$$line"; \
33+
npx ajv-cli validate -s trace.haskell.schema.json -d temp.json; \
34+
rm -f temp.json; \
35+
exit 1; \
36+
fi; \
37+
done < example.haskell.jsonl && echo "example.haskell.jsonl valid"
38+
@echo "Validating Rust trace events..."
39+
@line_num=0; \
40+
while IFS= read -r line; do \
41+
line_num=$$((line_num + 1)); \
42+
echo "$$line" > temp.json; \
43+
if ! npx ajv-cli validate -s trace.rust.schema.json -d temp.json 2>/dev/null >/dev/null; then \
44+
echo "Error in example.rust.jsonl at line $$line_num:"; \
45+
echo "$$line"; \
46+
npx ajv-cli validate -s trace.rust.schema.json -d temp.json; \
47+
rm -f temp.json; \
48+
exit 1; \
49+
fi; \
50+
done < example.rust.jsonl && echo "example.rust.jsonl valid"
51+
@rm -f temp.json

‎data/simulation/example.haskell.jsonl

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{"time_s": 1.0, "event": {"tag": "Cpu", "node": 1, "node_name": "node1", "duration_s": 0.1, "task_label": "ValIB: 6-29"}}
2+
{"time_s": 1.2, "event": {"tag": "Cpu", "node": 1, "node_name": "node1", "duration_s": 0.05, "task_label": "ValIH: 6253064077348247640"}}
3+
{"time_s": 1.5, "event": {"tag": "Cpu", "node": 2, "node_name": "node2", "duration_s": 0.15, "task_label": "ValEB: 7-30"}}
4+
{"time_s": 1.8, "event": {"tag": "Cpu", "node": 2, "node_name": "node2", "duration_s": 0.08, "task_label": "ValRH: 7253064077348247641"}}
5+
{"time_s": 2.0, "event": {"tag": "Cpu", "node": 3, "node_name": "node3", "duration_s": 0.12, "task_label": "ValRB: 8-31"}}
6+
{"time_s": 2.3, "event": {"tag": "Cpu", "node": 3, "node_name": "node3", "duration_s": 0.03, "task_label": "ValVote: vote123"}}
7+
{"time_s": 3.0, "event": {"tag": "generated", "kind": "IB", "id": "block1", "node": 1, "node_name": "node1", "size_bytes": 1000, "slot": 1, "payload_bytes": 800, "rb_ref": "rb123"}}
8+
{"time_s": 3.1, "event": {"tag": "Sent", "sender": 1, "receipient": 2, "msg_size_bytes": 1000, "sending_s": 0.01, "kind": "IB", "ids": ["block1"]}}
9+
{"time_s": 3.2, "event": {"tag": "received", "kind": "IB", "id": "block1", "node": 2, "node_name": "node2"}}
10+
{"time_s": 3.3, "event": {"tag": "enteredstate", "kind": "IB", "id": "block1", "node": 2, "node_name": "node2"}}
11+
{"time_s": 4.0, "event": {"tag": "generated", "kind": "EB", "id": "eb1", "node": 2, "node_name": "node2", "size_bytes": 500, "input_blocks": ["block1"]}}
12+
{"time_s": 4.1, "event": {"tag": "Sent", "sender": 2, "receipient": 3, "msg_size_bytes": 500, "sending_s": 0.01, "kind": "EB", "ids": ["eb1"]}}
13+
{"time_s": 4.2, "event": {"tag": "received", "kind": "EB", "id": "eb1", "node": 3, "node_name": "node3"}}
14+
{"time_s": 4.3, "event": {"tag": "enteredstate", "kind": "EB", "id": "eb1", "node": 3, "node_name": "node3"}}
15+
{"time_s": 5.0, "event": {"tag": "generated", "kind": "RB", "id": "rb1", "node": 3, "node_name": "node3", "size_bytes": 300}}
16+
{"time_s": 5.1, "event": {"tag": "Sent", "sender": 3, "receipient": 1, "msg_size_bytes": 300, "sending_s": 0.01, "kind": "RB", "ids": ["rb1"]}}
17+
{"time_s": 5.2, "event": {"tag": "received", "kind": "RB", "id": "rb1", "node": 1, "node_name": "node1"}}
18+
{"time_s": 5.3, "event": {"tag": "enteredstate", "kind": "RB", "id": "rb1", "node": 1, "node_name": "node1"}}
19+
{"time_s": 6.0, "event": {"tag": "generated", "kind": "VT", "id": "vote1", "node": 1, "node_name": "node1", "size_bytes": 200, "votes": 1, "endorse_blocks": ["eb1"]}}
20+
{"time_s": 6.1, "event": {"tag": "Sent", "sender": 1, "receipient": 2, "msg_size_bytes": 200, "sending_s": 0.01, "kind": "VT", "ids": ["vote1"]}}
21+
{"time_s": 6.2, "event": {"tag": "received", "kind": "VT", "id": "vote1", "node": 2, "node_name": "node2"}}
22+
{"time_s": 6.3, "event": {"tag": "enteredstate", "kind": "VT", "id": "vote1", "node": 2, "node_name": "node2"}}

‎data/simulation/example.rust.jsonl

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{"time": 1000, "message": {"type": "Slot", "number": 1}}
2+
{"time": 2000, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 0}, "task_type": "PraosBlockValidated"}}
3+
{"time": 2100, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 0}, "task_type": "PraosBlockValidated"}}
4+
{"time": 2200, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 1}, "task_type": "EndorserBlockValidated"}}
5+
{"time": 2300, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 1}, "task_type": "EndorserBlockValidated"}}
6+
{"time": 2400, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 2}, "task_type": "VoteBundleValidated"}}
7+
{"time": 2500, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 2}, "task_type": "VoteBundleValidated"}}
8+
{"time": 2600, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 3}, "task_type": "InputBlockValidated"}}
9+
{"time": 2700, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 3}, "task_type": "InputBlockValidated"}}
10+
{"time": 2800, "message": {"type": "CpuTaskScheduled", "task": {"node": "node1", "index": 4}, "task_type": "TransactionValidated"}}
11+
{"time": 2900, "message": {"type": "CpuTaskFinished", "task": {"node": "node1", "index": 4}, "task_type": "TransactionValidated"}}
12+
{"time": 3000, "message": {"type": "CpuSubtaskStarted", "task": {"node": "node1", "index": 5}, "task_type": "VoteBundleValidated", "subtask_id": 0, "subtasks": 2}}
13+
{"time": 3100, "message": {"type": "CpuSubtaskFinished", "task": {"node": "node1", "index": 5}, "task_type": "VoteBundleValidated", "subtask_id": 0, "subtasks": 2}}
14+
{"time": 3200, "message": {"type": "CpuSubtaskStarted", "task": {"node": "node1", "index": 5}, "task_type": "VoteBundleValidated", "subtask_id": 1, "subtasks": 2}}
15+
{"time": 3300, "message": {"type": "CpuSubtaskFinished", "task": {"node": "node1", "index": 5}, "task_type": "VoteBundleValidated", "subtask_id": 1, "subtasks": 2}}
16+
{"time": 4000, "message": {"type": "InputBlockGenerated", "slot": 1, "producer": "node1", "index": 0, "header_bytes": 100, "transactions": ["tx1", "tx2"]}}
17+
{"time": 4100, "message": {"type": "InputBlockSent", "slot": 1, "producer": "node1", "sender": "node1", "recipient": "node2", "index": 0, "header_bytes": 100, "transactions": ["tx1", "tx2"]}}
18+
{"time": 4200, "message": {"type": "InputBlockReceived", "slot": 1, "producer": "node1", "sender": "node1", "recipient": "node2", "index": 0, "header_bytes": 100, "transactions": ["tx1", "tx2"]}}
19+
{"time": 4300, "message": {"type": "InputBlockLotteryWon", "slot": 1, "producer": "node1", "index": 0}}
20+
{"time": 5000, "message": {"type": "EndorserBlockGenerated", "slot": 1, "producer": "node2"}}
21+
{"time": 5100, "message": {"type": "EndorserBlockSent", "slot": 1, "producer": "node2", "sender": "node2", "recipient": "node3"}}
22+
{"time": 5200, "message": {"type": "EndorserBlockReceived", "slot": 1, "producer": "node2", "sender": "node2", "recipient": "node3"}}
23+
{"time": 5300, "message": {"type": "EndorserBlockLotteryWon", "slot": 1, "producer": "node2"}}
24+
{"time": 6000, "message": {"type": "PraosBlockGenerated", "slot": 1, "producer": "node3", "header_bytes": 200, "transactions": ["tx1", "tx2"], "vrf": 584785879898000}}
25+
{"time": 6100, "message": {"type": "PraosBlockSent", "slot": 1, "producer": "node3", "sender": "node3", "recipient": "node1", "header_bytes": 200, "transactions": ["tx1", "tx2"], "vrf": 584785879898836}}
26+
{"time": 6200, "message": {"type": "PraosBlockReceived", "slot": 1, "producer": "node3", "sender": "node3", "recipient": "node1", "header_bytes": 200, "transactions": ["tx1", "tx2"], "vrf": 584785870008836}}
27+
{"time": 6300, "message": {"type": "PraosBlockLotteryWon", "slot": 1, "producer": "node3", "vrf": 384785879898000}}
28+
{"time": 7000, "message": {"type": "TransactionSent", "id": "tx1", "sender": "node1", "recipient": "node2", "publisher": "node1", "bytes": 50}}
29+
{"time": 8000, "message": {"type": "VotesGenerated", "id": "vote1", "slot": 1, "producer": "node1", "votes": {"node1": 1}}}
30+
{"time": 8100, "message": {"type": "VotesSent", "id": "vote1", "slot": 1, "producer": "node1", "sender": "node1", "recipient": "node2", "votes": {"node1": 1}}}
31+
{"time": 8200, "message": {"type": "VotesReceived", "id": "vote1", "slot": 1, "producer": "node1", "sender": "node1", "recipient": "node2", "votes": {"node1": 1}}}
32+
{"time": 8300, "message": {"type": "VoteLotteryWon", "id": "vote1", "slot": 1, "producer": "node1"}}

‎data/simulation/trace.haskell.d.ts

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/** Haskell simulation trace format */
2+
export interface HaskellTraceEvent {
3+
time_s: number;
4+
event: HaskellEvent;
5+
}
6+
7+
type BlockKind = "IB" | "EB" | "RB" | "VT";
8+
type BlockAction = "Generated" | "EnteredState";
9+
10+
type HaskellEvent =
11+
| HaskellCpuEvent
12+
| HaskellBlockEvent // Unified block event type
13+
| HaskellNetworkEvent; // Combine Sent/Received into network events
14+
15+
interface HaskellCpuEvent {
16+
type: "Cpu";
17+
node: string;
18+
cpu_time_s: number;
19+
// CPU task types: Block validation (ValIB, ValEB, ValRB), Header validation (ValIH, ValRH), Vote validation (ValVote). Format: "<task_type>: <id>"
20+
task_label: string; // e.g., "ValIB: 6-29" or "ValRH: 6253064077348247640"
21+
}
22+
23+
// Base block event interface with just identification info
24+
interface BaseBlockEvent {
25+
type: `${BlockKind}${BlockAction}`;
26+
node: string;
27+
id: string;
28+
slot: number;
29+
}
30+
31+
// Additional fields for Generated events
32+
interface GeneratedBlockEvent extends BaseBlockEvent {
33+
size_bytes: number;
34+
}
35+
36+
interface GeneratedInputBlock extends GeneratedBlockEvent {
37+
payload_bytes: number;
38+
rb_ref: string;
39+
}
40+
41+
interface GeneratedEndorserBlock extends GeneratedBlockEvent {
42+
input_blocks: string[];
43+
}
44+
45+
interface GeneratedRankingBlock extends GeneratedBlockEvent {
46+
endorse_blocks: string[];
47+
payload_bytes: number;
48+
}
49+
50+
interface GeneratedVote extends GeneratedBlockEvent {
51+
votes: number;
52+
endorse_blocks: string[];
53+
}
54+
55+
// EnteredState events only need the base identification info
56+
type EnteredStateBlock = BaseBlockEvent;
57+
58+
type HaskellBlockEvent =
59+
| GeneratedInputBlock
60+
| GeneratedEndorserBlock
61+
| GeneratedRankingBlock
62+
| GeneratedVote
63+
| EnteredStateBlock;
64+
65+
interface HaskellNetworkEvent {
66+
type: "NetworkMessage";
67+
action: "Sent" | "Received"; // Added to distinguish direction
68+
sender: string;
69+
recipient: string;
70+
block_kind: BlockKind;
71+
msg_size_bytes: number;
72+
sending_s: number;
73+
ids: string[];
74+
}

0 commit comments

Comments
 (0)