Dynamic switch transport (#218)

* temporary solution, allowing users to specify a transport type at application boot

* update eval scripts

* format

Author: crazyboycjr

Cargo.lock

@@ -4,7 +4,7 @@
 
 Make sure you have libibverbs, librdmacm, libnuma, protoc, libclang, globally available.
 ```
-# apt install libclang-dev librdmacm-dev libibverbs-dev protobuf-compiler
+# apt install libclang-dev libnuma-dev librdmacm-dev libibverbs-dev protobuf-compiler
 ```
 
 Start the phoenix backend service on all servers.

README.md

@@ -10,8 +10,6 @@ This folder contains scripts for running large RPC bandwidth mircobenchmark.
    results under `/tmp/mrpc-eval`.
 ```
 ./start_traffic_rdma.sh [/tmp/mrpc-eval]
-./start_phoenix_rdma.sh [/tmp/mrpc-eval]
-./start_phoenix_tcp.sh [/tmp/mrpc-eval]
 ./start_traffic_tcp.sh [/tmp/mrpc-eval]
 ```
 

eval/micro-bench/bandwidth/README.md

@@ -71,6 +71,9 @@ transport = "Rdma"
 [[modules]]
 name = "RpcAdapter"
 lib_path = "/tmp/phoenix/plugins/libphoenix_rpc_adapter_plugin.so"
+config_string = '''
+enable_scheduler = false
+'''
 
 
 [[modules]]

eval/micro-bench/bandwidth/phoenix.toml

@@ -6,5 +6,4 @@ fi
 
 WORKDIR=`dirname $(realpath $0)`
 cd $WORKDIR
-sed -i 's/transport =\(.*\)/transport = "Tcp"/g' phoenix.toml
 cargo rr --bin launcher -- -o ${OD} --benchmark ./launch_phoenix.toml --configfile ./config.toml --timeout 600

eval/micro-bench/bandwidth/start_phoenix_tcp.sh eval/micro-bench/bandwidth/start_phoenix.sh

@@ -15,7 +15,8 @@ for concurrency in 32; do
     sed -i 's/--concurrency [0-9]*/--concurrency '"$concurrency"'/g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
     sed -i 's/-D [0-9]*/-D 10/g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
     sed -i 's/timeout_secs = [0-9]*/timeout_secs = 15/g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
-    sed -i 's/transport =\(.*\)/transport = "Rdma"/g' phoenix.toml
+	sed -i 's/--transport \(rdma\|tcp\)//g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
+    sed -i 's/args = "/args = "--transport rdma /g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
     timestamp=$(date +%s%N)
 
     for i in 2kb; do

eval/micro-bench/bandwidth/start_traffic_rdma.sh

@@ -15,7 +15,8 @@ for concurrency in 128; do
     sed -i 's/--concurrency [0-9]*/--concurrency '"$concurrency"'/g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
     # sed -i 's/-D [0-9]*/-D 10/g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
     # sed -i 's/timeout_secs = [0-9]*/timeout_secs = 15/g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
-    sed -i 's/transport =\(.*\)/transport = "Tcp"/g' phoenix.toml
+	sed -i 's/--transport \(rdma\|tcp\)//g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
+    sed -i 's/args = "/args = "--transport tcp /g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
     timestamp=$(date +%s%N)
 
     for i in 128b 512b 2kb 8kb 32kb 128kb 512kb 1mb 2mb 8mb; do

eval/micro-bench/bandwidth/start_traffic_tcp.sh

@@ -11,8 +11,6 @@ scalability.
    results under `/tmp/mrpc-eval`.
 ```
 ./start_traffic_rdma.sh [/tmp/mrpc-eval]
-./start_phoenix_rdma.sh [/tmp/mrpc-eval]
-./start_phoenix_tcp.sh [/tmp/mrpc-eval]
 ./start_traffic_tcp.sh [/tmp/mrpc-eval] [danyang-06]
 ```
 

eval/micro-bench/rate/README.md

@@ -71,7 +71,9 @@ transport = "Rdma"
 [[modules]]
 name = "RpcAdapter"
 lib_path = "/tmp/phoenix/plugins/libphoenix_rpc_adapter_plugin.so"
-
+config_string = '''
+enable_scheduler = false
+'''
 
 [[modules]]
 name = "TcpRpcAdapter"

eval/micro-bench/rate/phoenix.toml

@@ -14,8 +14,8 @@ cd $workdir
 concurrency=32
 sed -i 's/--concurrency [0-9]*/--concurrency '"$concurrency"'/g' ../../../benchmark/benchmark/rpc_bench_rate/*.toml
 sed -i 's/-D [0-9]*/-D 10/g' ../../../benchmark/benchmark/rpc_bench_rate/*.toml
-sed -i 's/timeout_secs = [0-9]*/timeout_secs = 15/g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
-sed -i 's/transport =\(.*\)/transport = "Rdma"/g' phoenix.toml
+sed -i 's/--transport \(rdma\|tcp\)//g' ../../../benchmark/benchmark/rpc_bench_rate/*.toml
+sed -i 's/args = "/args = "--transport rdma /g' ../../../benchmark/benchmark/rpc_bench_rate/*.toml
 timestamp=$(date +%s%N)
 
 for i in 1 2 4 8; do

eval/micro-bench/bandwidth/start_phoenix_rdma.sh eval/micro-bench/rate/start_phoenix.sh

@@ -14,8 +14,8 @@ cd $workdir
 concurrency=128
 sed -i 's/--concurrency [0-9]*/--concurrency '"$concurrency"'/g' ../../../benchmark/benchmark/rpc_bench_rate/*.toml
 sed -i 's/-D [0-9]*/-D 10/g' ../../../benchmark/benchmark/rpc_bench_rate/*.toml
-sed -i 's/timeout_secs = [0-9]*/timeout_secs = 15/g' ../../../benchmark/benchmark/rpc_bench_tput/*.toml
-sed -i 's/transport =\(.*\)/transport = "Tcp"/g' phoenix.toml
+sed -i 's/--transport \(rdma\|tcp\)//g' ../../../benchmark/benchmark/rpc_bench_rate/*.toml
+sed -i 's/args = "/args = "--transport tcp /g' ../../../benchmark/benchmark/rpc_bench_rate/*.toml
 timestamp=$(date +%s%N)
 
 for i in 1 2 4 8; do

eval/micro-bench/rate/start_traffic_rdma.sh

@@ -6,10 +6,10 @@ timeout_secs = 70
 [[worker]]
 host = "danyang-06"
 bin = "hotel_reservation_server"
-args = ""
+args = "--transport tcp"
 
 [[worker]]
 host = "danyang-05"
 bin = "hotel_reservation_client"
-args = "-c rdma0.danyang-06 --concurrency 128 -D 65 -i 1"
+args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 -D 65 -i 1"
 dependencies = [0]

eval/micro-bench/rate/start_traffic_tcp.sh

@@ -71,7 +71,9 @@ transport = "Tcp"
 [[modules]]
 name = "RpcAdapter"
 lib_path = "/tmp/phoenix/plugins/libphoenix_rpc_adapter_plugin.so"
-
+config_string = '''
+enable_scheduler = false
+'''
 
 [[modules]]
 name = "TcpRpcAdapter"

eval/policy/hotel-acl/hotel_reservation.toml

@@ -4,12 +4,13 @@ This folder contains scripts for null policy.
 1. Start receiver backend on `danyang-06`. Start sender backend on
    `danyang-05`.
 ```
-cjr@danyang-05 $ ./start_phoenix_tcp.sh [/tmp/mrpc-eval]
+cjr@danyang-05 $ ./start_phoenix.sh [/tmp/mrpc-eval]
 ```
 
 2. Start the rpc_bench_server and rpc_bench_client
 ```
 cjr@danyang-05 $ ./start_traffic_tcp.sh [/tmp/mrpc-eval]
+cjr@danyang-05 $ ./start_traffic_rdma.sh [/tmp/mrpc-eval]
 ```
 
 3. Attach policy at the client and the server

eval/policy/hotel-acl/phoenix.toml

@@ -72,7 +72,9 @@ nic_index = 0
 [[modules]]
 name = "RpcAdapter"
 lib_path = "/tmp/phoenix/plugins/libphoenix_rpc_adapter_plugin.so"
-
+config_string = '''
+enable_scheduler = false
+'''
 
 [[modules]]
 name = "TcpRpcAdapter"

eval/policy/null/README.md

@@ -8,6 +8,7 @@ WORKDIR=$(dirname $(realpath $0))
 cd $WORKDIR
 
 # concurrency = 1
-sed -i 's/transport =\(.*\)/transport = "Rdma"/g' phoenix.toml
+sed -i 's/--transport \(rdma\|tcp\)//g' ./rpc_bench_latency_64b.toml
+sed -i 's/--transport /--transport rdma /g' ./rpc_bench_latency_64b.toml
 sed -i 's/TcpRpcAdapterEngine/RpcAdapterEngine/g' attach.toml
 cargo rr --bin launcher -- --output-dir ${OD} --benchmark ./rpc_bench_latency_64b.toml --configfile ./config.toml

eval/policy/null/phoenix.toml

@@ -8,6 +8,7 @@ WORKDIR=$(dirname $(realpath $0))
 cd $WORKDIR
 
 # concurrency = 1
-sed -i 's/transport =\(.*\)/transport = "Tcp"/g' phoenix.toml
+sed -i 's/--transport \(rdma\|tcp\)//g' ./rpc_bench_latency_64b.toml
+sed -i 's/--transport /--transport rdma /g' ./rpc_bench_latency_64b.toml
 sed -i 's/\"RpcAdapterEngine\"/\"TcpRpcAdapterEngine\"/g' attach.toml
 cargo rr --bin launcher -- --output-dir ${OD} --benchmark ./rpc_bench_latency_64b.toml --configfile ./config.toml

eval/micro-bench/rate/start_phoenix_rdma.sh eval/policy/null/start_phoenix.sh

@@ -71,7 +71,9 @@ transport = "Rdma"
 [[modules]]
 name = "RpcAdapter"
 lib_path = "/tmp/phoenix/plugins/libphoenix_rpc_adapter_plugin.so"
-
+config_string = '''
+enable_scheduler = false
+'''
 
 [[modules]]
 name = "TcpRpcAdapter"

eval/policy/null/start_phoenix_rdma.sh

@@ -71,7 +71,9 @@ transport = "Rdma"
 [[modules]]
 name = "RpcAdapter"
 lib_path = "/tmp/phoenix/plugins/libphoenix_rpc_adapter_plugin.so"
-
+config_string = '''
+enable_scheduler = false
+'''
 
 [[modules]]
 name = "TcpRpcAdapter"
@@ -92,4 +94,4 @@ name = "Qos"
 lib_path = "/tmp/phoenix/plugins/libphoenix_qos_plugin.so"
 config_string = '''
 latency_budget_microsecs = 10
-'''
+'''

eval/policy/null/start_phoenix_tcp.sh

@@ -71,7 +71,9 @@ transport = "Rdma"
 [[modules]]
 name = "RpcAdapter"
 lib_path = "/tmp/phoenix/plugins/libphoenix_rpc_adapter_plugin.so"
-
+config_string = '''
+enable_scheduler = false
+'''
 
 [[modules]]
 name = "TcpRpcAdapter"

eval/policy/null/start_traffic_rdma.sh

@@ -6,5 +6,4 @@ fi
 
 WORKDIR=`dirname $(realpath $0)`
 cd $WORKDIR
-sed -i 's/transport =\(.*\)/transport = "Tcp"/g' phoenix.toml
 cargo rr --bin launcher -- -o ${OD} --benchmark ./launch_phoenix.toml --configfile ./config.toml --timeout 600

eval/policy/null/start_traffic_tcp.sh

@@ -2,14 +2,27 @@ use serde::{Deserialize, Serialize};
 
 type IResult<T> = Result<T, interface::Error>;
 
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
 pub enum TransportType {
     #[serde(alias = "Rdma")]
     Rdma,
+    #[default]
     #[serde(alias = "Tcp")]
     Tcp,
 }
 
+impl std::str::FromStr for TransportType {
+    type Err = &'static str;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.to_uppercase().as_str() {
+            "RDMA" => Ok(Self::Rdma),
+            "TCP" => Ok(Self::Tcp),
+            _ => Err("Expect RDMA or TCP"),
+        }
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum Request {}
 
@@ -20,7 +33,7 @@ pub enum ResponseKind {}
 pub struct Response(pub IResult<ResponseKind>);
 
 /// Service setting. Each use thread can have its own service setting.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
 pub struct Setting {
     /// The transport to use.
     pub transport: TransportType,
@@ -30,13 +43,3 @@ pub struct Setting {
     /// Set when the user thread binds to a CPU core.
     pub core_id: Option<usize>,
 }
-
-impl Default for Setting {
-    fn default() -> Self {
-        Setting {
-            transport: TransportType::Rdma,
-            nic_index: 0,
-            core_id: None,
-        }
-    }
-}

eval/policy/qos/phoenix_client.toml

@@ -58,3 +58,5 @@ unique = "0.9.1"
 sharded-slab = "0.1.4"
 libnuma = "0.0.4"
 libnuma-sys = "0.0.4"
+
+serde_json = "1.0.81"