-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* created architecture overview --------- Co-authored-by: dtrai2 <[email protected]>
- Loading branch information
Showing
31 changed files
with
4,645 additions
and
22 deletions.
There are no files selected for viewing
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
172 changes: 172 additions & 0 deletions
172
doc/source/development/architecture/diagramms/event.drawio
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,172 @@ | ||
| <mxfile host="65bd71144e"> | ||
| <diagram id="CprJfZTJ5oxMJRHnkWTq" name="Page-1"> | ||
| <mxGraphModel dx="924" dy="341" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0"> | ||
| <root> | ||
| <mxCell id="0"/> | ||
| <mxCell id="1" parent="0"/> | ||
| <mxCell id="24" value="" style="swimlane;html=1;childLayout=stackLayout;resizeParent=1;resizeParentMax=0;horizontal=1;startSize=0;horizontalStack=0;whiteSpace=wrap;" parent="1" vertex="1"> | ||
| <mxGeometry x="25" y="30" width="1250" height="1570" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="73" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=1;entryY=0.5;entryDx=0;entryDy=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="24" source="69" target="53"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="74" style="edgeStyle=orthogonalEdgeStyle;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="24" source="71"> | ||
| <mxGeometry relative="1" as="geometry"> | ||
| <mxPoint x="995" y="1480" as="targetPoint"/> | ||
| <Array as="points"> | ||
| <mxPoint x="1112" y="1480"/> | ||
| <mxPoint x="1000" y="1480"/> | ||
| </Array> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="25" value="Kafka TopicPartition" style="swimlane;html=1;startSize=20;horizontal=0;" parent="24" vertex="1"> | ||
| <mxGeometry width="1250" height="170" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="56" value="<font style="background-color: rgb(255, 255, 255);" color="#000000">event is written to kafka topic</font>" style="ellipse;whiteSpace=wrap;html=1;" parent="25" vertex="1"> | ||
| <mxGeometry x="56" y="45" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="34" value="librdkafka" style="swimlane;html=1;startSize=20;horizontal=0;" parent="24" vertex="1"> | ||
| <mxGeometry y="170" width="1250" height="300" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="40" value="" style="edgeStyle=none;html=1;" parent="34" source="35" target="39" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="42" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="34" source="35" target="78" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"> | ||
| <mxPoint x="233.0000000000001" y="107" as="targetPoint"/> | ||
| <Array as="points"> | ||
| <mxPoint x="234" y="54"/> | ||
| </Array> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="35" value="poll()" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="34" vertex="1"> | ||
| <mxGeometry x="56" y="14" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="39" value="consumer.poll()" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="34" vertex="1"> | ||
| <mxGeometry x="56" y="199" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="78" value="<span style="color: rgb(240, 240, 240);">message Queue per TopicPartition</span>" style="shape=dataStorage;whiteSpace=wrap;html=1;fixedSize=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;align=center;" vertex="1" parent="34"> | ||
| <mxGeometry x="156" y="111.5" width="155" height="77" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="26" value="ConfluentKafka Connector" style="swimlane;html=1;startSize=20;horizontal=0;" parent="24" vertex="1"> | ||
| <mxGeometry y="470" width="1250" height="190" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="3" value="_get_raw_event()<br>takes event from the topic" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="26" vertex="1"> | ||
| <mxGeometry x="56" y="55" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="45" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;" parent="26" source="5" target="44" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="5" value="_get_event()<br>parses raw event in json" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="26" vertex="1"> | ||
| <mxGeometry x="246" y="55" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="6" value="" style="edgeStyle=none;html=1;" parent="26" source="3" target="5" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="44" value="get_next()" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="26" vertex="1"> | ||
| <mxGeometry x="446" y="58" width="120" height="75" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="27" value="Pipeline" style="swimlane;html=1;startSize=20;horizontal=0;" parent="24" vertex="1"> | ||
| <mxGeometry y="660" width="1250" height="180" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="7" value="events goes into pipeline" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="27" vertex="1"> | ||
| <mxGeometry x="446" y="50" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="11" value="event given back to pipeline" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="27" vertex="1"> | ||
| <mxGeometry x="618" y="53" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="75" value="event is never returned<br>it is changed in place" style="shape=note;strokeWidth=2;fontSize=14;size=20;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;fontColor=#666600;opacity=50;" vertex="1" parent="27"> | ||
| <mxGeometry x="845" y="44" width="165" height="92" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="28" value="Processor" style="swimlane;html=1;startSize=20;horizontal=0;" parent="24" vertex="1"> | ||
| <mxGeometry y="840" width="1250" height="180" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="9" value="event being processed by processors" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="28" vertex="1"> | ||
| <mxGeometry x="447" y="63" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="29" value="Output Connector" style="swimlane;html=1;startSize=20;horizontal=0;" parent="24" vertex="1"> | ||
| <mxGeometry y="1020" width="1250" height="370" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="59" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="29" source="13"> | ||
| <mxGeometry relative="1" as="geometry"> | ||
| <mxPoint x="793" y="223" as="targetPoint"/> | ||
| <Array as="points"> | ||
| <mxPoint x="761" y="223"/> | ||
| </Array> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="62" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;" edge="1" parent="29" source="13" target="30"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="13" value="event is stored in message backlog" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="29" vertex="1"> | ||
| <mxGeometry x="701" y="63" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="66" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;" edge="1" parent="29" source="30" target="61"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="30" value="backlog full or flush timeout exceeded" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="29" vertex="1"> | ||
| <mxGeometry x="886" y="63" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="65" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;" edge="1" parent="29" source="61" target="63"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="61" value="write_backlog()" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" vertex="1" parent="29"> | ||
| <mxGeometry x="1056" y="65" width="114" height="77" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="63" value="parallel_bulk()" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" vertex="1" parent="29"> | ||
| <mxGeometry x="1056" y="194" width="114" height="60" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="69" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" vertex="1" parent="29"> | ||
| <mxGeometry x="1060" y="284" width="120" height="60" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="71" value="thread" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" vertex="1" parent="29"> | ||
| <mxGeometry x="1054" y="294" width="118" height="59" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="72" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;" edge="1" parent="29" source="63" target="71"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="77" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="29" target="63"> | ||
| <mxGeometry relative="1" as="geometry"> | ||
| <mxPoint x="915" y="224" as="sourcePoint"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="76" value="message backlog" style="shape=dataStorage;whiteSpace=wrap;html=1;fixedSize=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" vertex="1" parent="29"> | ||
| <mxGeometry x="795" y="185.5" width="155" height="77" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="48" value="Opensearch" style="swimlane;html=1;startSize=20;horizontal=0;" parent="24" vertex="1"> | ||
| <mxGeometry y="1390" width="1250" height="180" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="53" value="event is written to index in Opensearch" style="ellipse;whiteSpace=wrap;html=1;" parent="48" vertex="1"> | ||
| <mxGeometry x="881" y="63" width="120" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="8" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="24" source="44" target="7" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="10" value="" style="edgeStyle=none;html=1;" parent="24" source="7" target="9" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="12" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;" parent="24" source="9" target="11" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="14" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="24" source="11" target="13" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"> | ||
| <Array as="points"> | ||
| <mxPoint x="761" y="753"/> | ||
| </Array> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="38" value="" style="edgeStyle=none;html=1;" parent="24" source="56" target="35" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"> | ||
| <mxPoint x="116" y="145" as="sourcePoint"/> | ||
| <Array as="points"> | ||
| <mxPoint x="116" y="174"/> | ||
| </Array> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="43" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;" parent="24" source="39" target="3" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| </root> | ||
| </mxGraphModel> | ||
| </diagram> | ||
| </mxfile> |
11 changes: 11 additions & 0 deletions
11
doc/source/development/architecture/diagramms/event.drawio.html
Large diffs are not rendered by default.
Oops, something went wrong.
Oops, something went wrong.