Refactor Performance Testing

scripts/RemoteTests.json

@@ -12,7 +12,7 @@
                 "Tls": ["schannel", "openssl", "openssl3"],
                 "Arch": ["x64", "x86", "arm", "arm64"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -upload:12000 -stats:1 -sstats:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12000 -timed:1 -ptput:1 -pconn:1 -pstream:1 -tcp:0"
             },
             "Variables": [
                 {
@@ -46,11 +46,10 @@
                 "Tls": ["schannel", "openssl", "openssl3"],
                 "Arch": ["x64", "x86", "arm", "arm64"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -upload:12000 -tcp:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12000 -timed:1 -ptput:1 -pconn:1 -pstream:1 -tcp:1"
             },
             "Variables": [
             ],
-            "SkipKernel": true,
             "AllowLoopback": true,
             "Iterations": 5,
             "RemoteReadyMatcher": "Started!",
@@ -66,7 +65,7 @@
                 "Tls": ["openssl", "openssl3"],
                 "Arch": ["x64", "arm"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -upload:12000 -stats:1 -sstats:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12000 -timed:1 -ptput:1 -pconn:1 -pstream:1 -tcp:0"
             },
             "Variables": [
                 {
@@ -100,11 +99,10 @@
                 "Tls": ["openssl", "openssl3"],
                 "Arch": ["x64", "arm"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -upload:12000 -tcp:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12000 -timed:1 -ptput:1 -pconn:1 -pstream:1 -tcp:1"
             },
             "Variables": [
             ],
-            "SkipKernel": true,
             "AllowLoopback": true,
             "Iterations": 5,
             "RemoteReadyMatcher": "Started!",
@@ -120,7 +118,7 @@
                 "Tls": ["schannel", "openssl", "openssl3"],
                 "Arch": ["x64", "x86", "arm", "arm64"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -download:12000 -stats:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12000 -timed:1 -ptput:1 -pconn:1 -tcp:0"
             },
             "Variables": [
                 {
@@ -146,11 +144,10 @@
                 "Tls": ["schannel", "openssl", "openssl3"],
                 "Arch": ["x64", "x86", "arm", "arm64"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -download:12000 -tcp:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12000 -timed:1 -ptput:1 -pconn:1 -tcp:1"
             },
             "Variables": [
             ],
-            "SkipKernel": true,
             "AllowLoopback": true,
             "Iterations": 5,
             "RemoteReadyMatcher": "Started!",
@@ -166,7 +163,7 @@
                 "Tls": ["openssl", "openssl3"],
                 "Arch": ["x64", "arm"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -download:12000 -stats:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12000 -timed:1 -ptput:1 -pconn:1 -tcp:0"
             },
             "Variables": [
                 {
@@ -192,11 +189,10 @@
                 "Tls": ["openssl", "openssl3"],
                 "Arch": ["x64", "arm"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -download:12000 -tcp:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12000 -timed:1 -ptput:1 -pconn:1 -tcp:1"
             },
             "Variables": [
             ],
-            "SkipKernel": true,
             "AllowLoopback": true,
             "Iterations": 5,
             "RemoteReadyMatcher": "Started!",
@@ -212,16 +208,16 @@
                 "Tls": ["schannel", "openssl", "openssl3"],
                 "Arch": ["x64", "x86", "arm", "arm64"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:lowlat -test:RPS -target:$RemoteAddress"
+                "Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:0"
             },
             "Variables": [
                 {
                     "Name": "ConnectionCount",
                     "Local": {
                         "1": "-conns:1 -requests:1 -runtime:30000",
-                        "40": "-conns:40 -requests:20",
-                        "250": "-conns:250 -requests:7500",
-                        "1000": "-conns:1 -requests:1000"
+                        "40": "-conns:40 -requests:1 -runtime:10000",
+                        "250": "-conns:250 -requests:30 -runtime:10000",
+                        "1000": "-conns:1 -requests:1000 -runtime:10000"
                     },
                     "Default": "250"
                 },
@@ -246,8 +242,53 @@
             "AllowLoopback": false,
             "Iterations": 5,
             "RemoteReadyMatcher": "Started!",
-            "ResultsMatcher": "Result: (.*) RPS, Min: (.*), Max: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), StdErr: (.*)",
-            "Formats": ["{0} RPS", "Minimum: {0}", "Maximum: {0}", "Percentiles: 50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Standard Error: {0}"],
+            "ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
+            "Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
+            "RegressionThreshold": "-50.0"
+        },
+        {
+            "TestName": "TcpRPS",
+            "Local": {
+                "Platform": "Windows",
+                "Tls": ["schannel", "openssl", "openssl3"],
+                "Arch": ["x64", "x86", "arm", "arm64"],
+                "Exe": "secnetperf",
+                "Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:1"
+            },
+            "Variables": [
+                {
+                    "Name": "ConnectionCount",
+                    "Local": {
+                        "1": "-conns:1 -requests:1 -runtime:30000",
+                        "40": "-conns:40 -requests:1 -runtime:10000",
+                        "250": "-conns:250 -requests:30 -runtime:10000",
+                        "1000": "-conns:1 -requests:1000 -runtime:10000"
+                    },
+                    "Default": "250"
+                },
+                {
+                    "Name": "RequestSize",
+                    "Local": {
+                        "0": "-request:0"
+                    },
+                    "Default": "0"
+                },
+                {
+                    "Name": "ResponseSize",
+                    "Local": {
+                        "0": "-response:0",
+                        "512": "-response:512",
+                        "4096": "-response:4096",
+                        "16384": "-response:16384"
+                    },
+                    "Default": "4096"
+                }
+            ],
+            "AllowLoopback": false,
+            "Iterations": 5,
+            "RemoteReadyMatcher": "Started!",
+            "ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
+            "Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
             "RegressionThreshold": "-50.0"
         },
         {
@@ -257,16 +298,16 @@
                 "Tls": ["openssl", "openssl3"],
                 "Arch": ["x64", "arm"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:lowlat -test:RPS -target:$RemoteAddress"
+                "Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:0"
             },
             "Variables": [
                 {
                     "Name": "ConnectionCount",
                     "Local": {
-                        "1": "-conns:1 -requests:1",
-                        "40": "-conns:40 -requests:20",
-                        "250": "-conns:250 -requests:7500",
-                        "1000": "-conns:1 -requests:1000"
+                        "1": "-conns:1 -requests:1 -runtime:30000",
+                        "40": "-conns:40 -requests:1 -runtime:10000",
+                        "250": "-conns:250 -requests:30 -runtime:10000",
+                        "1000": "-conns:1 -requests:1000 -runtime:10000"
                     },
                     "Default": "250"
                 },
@@ -291,8 +332,53 @@
             "AllowLoopback": false,
             "Iterations": 5,
             "RemoteReadyMatcher": "Started!",
-            "ResultsMatcher": "Result: (.*) RPS, Min: (.*), Max: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), StdErr: (.*)",
-            "Formats": ["{0} RPS", "Minimum: {0}", "Maximum: {0}", "Percentiles: 50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Standard Error: {0}"],
+            "ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
+            "Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
+            "RegressionThreshold": "-75.0"
+        },
+        {
+            "TestName": "TcpRPS",
+            "Local": {
+                "Platform": "linux",
+                "Tls": ["openssl", "openssl3"],
+                "Arch": ["x64", "arm"],
+                "Exe": "secnetperf",
+                "Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:1"
+            },
+            "Variables": [
+                {
+                    "Name": "ConnectionCount",
+                    "Local": {
+                        "1": "-conns:1 -requests:1 -runtime:30000",
+                        "40": "-conns:40 -requests:1 -runtime:10000",
+                        "250": "-conns:250 -requests:30 -runtime:10000",
+                        "1000": "-conns:1 -requests:1000 -runtime:10000"
+                    },
+                    "Default": "250"
+                },
+                {
+                    "Name": "RequestSize",
+                    "Local": {
+                        "0": "-request:0"
+                    },
+                    "Default": "0"
+                },
+                {
+                    "Name": "ResponseSize",
+                    "Local": {
+                        "0": "-response:0",
+                        "512": "-response:512",
+                        "4096": "-response:4096",
+                        "16384": "-response:16384"
+                    },
+                    "Default": "4096"
+                }
+            ],
+            "AllowLoopback": false,
+            "Iterations": 5,
+            "RemoteReadyMatcher": "Started!",
+            "ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
+            "Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
             "RegressionThreshold": "-75.0"
         },
         {
@@ -302,7 +388,24 @@
                 "Tls": ["schannel", "openssl", "openssl3"],
                 "Arch": ["x64", "x86", "arm", "arm64"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:HPS -target:$RemoteAddress -incrementtarget:1"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -inctarget:1 -rconn:1 -conns:800 -share:1 -run:10000 -tcp:0 -prate:1"
+            },
+            "Variables": [],
+            "AllowLoopback": false,
+            "Iterations": 5,
+            "RemoteReadyMatcher": "Started!",
+            "ResultsMatcher": "Result: (.*) HPS.*",
+            "Formats": ["{0} HPS"],
+            "RegressionThreshold": "-40.0"
+        },
+        {
+            "TestName": "TcpHPS",
+            "Local": {
+                "Platform": "Windows",
+                "Tls": ["schannel", "openssl", "openssl3"],
+                "Arch": ["x64", "x86", "arm", "arm64"],
+                "Exe": "secnetperf",
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -inctarget:1 -rconn:1 -conns:800 -share:1 -run:10000 -tcp:1 -prate:1"
             },
             "Variables": [],
             "AllowLoopback": false,
@@ -319,7 +422,24 @@
                 "Tls": ["openssl", "openssl3"],
                 "Arch": ["x64", "arm"],
                 "Exe": "secnetperf",
-                "Arguments": "-exec:maxtput -test:HPS -target:$RemoteAddress"
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -inctarget:1 -rconn:1 -conns:800 -share:1 -run:10000 -tcp:0 -prate:1"
+            },
+            "Variables": [],
+            "AllowLoopback": false,
+            "Iterations": 5,
+            "RemoteReadyMatcher": "Started!",
+            "ResultsMatcher": "Result: (.*) HPS.*",
+            "Formats": ["{0} HPS"],
+            "RegressionThreshold": "-75.0"
+        },
+        {
+            "TestName": "TcpHPS",
+            "Local": {
+                "Platform": "linux",
+                "Tls": ["openssl", "openssl3"],
+                "Arch": ["x64", "arm"],
+                "Exe": "secnetperf",
+                "Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -inctarget:1 -rconn:1 -conns:800 -share:1 -run:10000 -tcp:1 -prate:1"
             },
             "Variables": [],
             "AllowLoopback": false,

scripts/clog.inputs

@@ -95,22 +95,10 @@
 ../src/test/lib/QuicDrill.cpp
 ../src/test/lib/DatagramTest.cpp
 ../src/test/lib/OwnershipTest.cpp
-../src/perf/lib/PerfHelpers.h
+../src/perf/lib/PerfClient.cpp
 ../src/perf/lib/PerfServer.cpp
-../src/perf/lib/SecNetPerfMain.cpp
-../src/perf/lib/HpsClient.h
-../src/perf/lib/PerfServer.h
 ../src/perf/lib/CMakeLists.txt
-../src/perf/lib/Tcp.h
-../src/perf/lib/RpsClient.h
 ../src/perf/lib/Tcp.cpp
-../src/perf/lib/ThroughputClient.h
-../src/perf/lib/RpsClient.cpp
-../src/perf/lib/PerfCommon.h
-../src/perf/lib/ThroughputClient.cpp
-../src/perf/lib/PerfBase.h
-../src/perf/lib/HpsClient.cpp
-../src/perf/lib/LatencyHelpers.h
 ../src/core/unittest/SettingsTest.cpp
 ../src/core/unittest/SpinFrame.cpp
 ../src/core/unittest/SlidingWindowExtremumTest.cpp
@@ -134,6 +122,5 @@
 ../src/perf/bin/histogram/hdr_histogram.h
 ../src/perf/bin/histogram/hdr_histogram.c
 ../src/perf/bin/drvmain.cpp
-../src/perf/bin/appmain.cpp
 ../src/tools/interop/interop.cpp
 ../src/tools/interop/interop.h