[xbar,rtl] support fifo_depth>1

This commit adds the possibility to increase the fifo depth in the xbar
to values > 1 to support multiple outstanding transactions.

Why this is beneficial:
The ibex instruction cache issues two 32b requests to the flash controller.
Inside xbar_main a pipeline register is added to break the critical path
to the flash. The pipeline register is added with a fifo of depth=1 for
req and rsp data and effectively inserts a bubble after each request and
response because the fifo is immediately full. Ibex and flash_ctrl can
deal with up to 2 outstanding transactions.

The impact on performance is low because the instruction cache reads the
critical word first and hides the additional latency that is inserted by
the fifo with depth=1. Nonetheless, in phases with many cache misses,
the performance can be improved at the price of an additional fifo entry.

Signed-off-by: Michael Gautschi <[email protected]>

Author: gautschimi

hw/top_earlgrey/data/autogen/top_earlgrey.gen.hjson

@@ -11012,6 +11012,7 @@
           reset: rst_main_ni
           req_fifo_pass: false
           rsp_fifo_pass: false
+          fifo_depth: 2
           inst_type: flash_ctrl
           addr_range:
           [

hw/top_earlgrey/data/xbar_main.hjson

@@ -116,14 +116,15 @@
       req_fifo_pass: false,
       rsp_fifo_pass: false,
     },
-    // Return and examine whether this path
-    // latency can be improved.
+    // fifo depth is set to 2 to be able to support 2 outstanding transactions from the ibex
+    // instruction cache (2x32b=1x64b cacheline)
     { name:      "flash_ctrl.mem",
       type:      "device",
       clock:     "clk_main_i",
       reset:     "rst_main_ni",
       req_fifo_pass: false,
       rsp_fifo_pass: false,
+      fifo_depth: 2,
     },
     { name:      "hmac",
       type:      "device",

hw/top_earlgrey/ip/xbar_main/data/autogen/xbar_main.gen.hjson

@@ -397,6 +397,7 @@
       reset: rst_main_ni
       req_fifo_pass: false
       rsp_fifo_pass: false
+      fifo_depth: 2
       inst_type: flash_ctrl
       addr_range:
       [

hw/top_earlgrey/ip/xbar_main/rtl/autogen/xbar_main.sv

@@ -965,6 +965,8 @@ end
     .HRspDepth (12'h0),
     .DReqPass  (1'b0),
     .DRspPass  (1'b0),
+    .DReqDepth (4'h2),
+    .DRspDepth (4'h2),
     .M         (3)
   ) u_sm1_31 (
     .clk_i        (clk_main_i),

util/tlgen/elaborate.py

@@ -248,10 +248,10 @@ def process_pipeline(xbar: Xbar) -> None:
 
         elif isinstance(unode, SocketM1):
             if full_fifo:
-                log.info("Fifo present with no passthrough")
+                log.info("Fifo present with no passthrough and depth=2")
                 unode.dreq_pass = 0
                 unode.drsp_pass = 0
-                unode.ddepth = 1
+                unode.ddepth = device.fifo_depth
             elif fifo_passthru:
                 log.info("Fifo present with passthrough")
                 unode.dreq_pass = req_pass

util/tlgen/item.py

@@ -58,6 +58,12 @@ class Node:
     req_fifo_pass = True
     rsp_fifo_pass = True
 
+    # FIFO depth option. default is 1
+    # If pipeline is false or req/rsp_fifo_pass are true, this field has no meaning
+    # A fifo with depth > 1 can support multiple outstanding transactions without blocking the
+    # host
+    fifo_depth = 1
+
     def __init__(self,
                  name: str,
                  clock: str,

util/tlgen/validate.py

@@ -72,6 +72,7 @@
                           'If true, pipeline fifo has passthrough behavior on req'],
         'rsp_fifo_pass': ['pb',
                           'If true, pipeline fifo has passthrough behavior on rsp'],
+        'fifo_depth': ['d', 'depth of req/rsp fifo if pipeline is true and fifo_pass is false'],
         'inst_type': ['s', 'Instance type'],
         'xbar': ['pb', 'If true, the node is connected to another Xbar'],
         'addr_range': ['lg', addrs],
@@ -389,11 +390,14 @@ def validate(obj: Dict[Any, Any]) -> Optional[Xbar]:
         node.pipeline = False
         node.req_fifo_pass = False
         node.rsp_fifo_pass = False
+        node.fifo_depth = 1
 
         if isinstance(node, Device) or isinstance(node, Host):
             node.pipeline = nodeobj.get("pipeline", False)
             node.req_fifo_pass = nodeobj.get("req_fifo_pass", False)
             node.rsp_fifo_pass = nodeobj.get("rsp_fifo_pass", False)
+        if isinstance(node, Device):
+            node.fifo_depth = nodeobj.get("fifo_depth", 1)
 
         xbar.nodes.append(node)