Skip to content

Commit 64f3f6d

Browse files
canepatcanepat
authored
ci: perf test light workflow (#2778)
1 parent b0fb9f4 commit 64f3f6d

File tree

3 files changed

+283
-217
lines changed

3 files changed

+283
-217
lines changed

‎.github/actions/perf-common-steps/action.yml

+255
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
name: QA - RPC Performance Common Steps
2+
description: Common steps for running Silkworm performance tests
3+
4+
inputs:
5+
activation_mode:
6+
description: 'Activation mode for performance tests (full,light)'
7+
default: 'light'
8+
9+
runs:
10+
using: "composite"
11+
12+
steps:
13+
- name: Checkout Silkworm repository
14+
uses: actions/checkout@v3
15+
with:
16+
submodules: recursive
17+
fetch-depth: "0"
18+
19+
- name: Checkout RPC Tests Repository & Install Requirements
20+
shell: bash
21+
run: |
22+
rm -rf ${{runner.workspace}}/rpc-tests
23+
git -c advice.detachedHead=false clone --depth 1 --branch v1.48.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
24+
cd ${{runner.workspace}}/rpc-tests
25+
pip3 install -r requirements.txt --break-system-packages
26+
27+
- name: Clean Build Directory
28+
shell: bash
29+
run: rm -rf ${{runner.workspace}}/silkworm/build
30+
31+
- name: Create Build Environment
32+
shell: bash
33+
run: cmake -E make_directory ${{runner.workspace}}/silkworm/build
34+
35+
- name: Configure CMake
36+
shell: bash
37+
working-directory: ${{runner.workspace}}/silkworm/build
38+
run: |
39+
cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=Release
40+
41+
- name: Build Silkworm RpcDaemon
42+
shell: bash
43+
working-directory: ${{runner.workspace}}/silkworm/build
44+
run: cmake --build . --config Release --target rpcdaemon -j 8
45+
46+
- name: Pause the Erigon instance dedicated to db maintenance
47+
shell: bash
48+
run: |
49+
python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
50+
51+
- name: Run Silkworm RpcDaemon
52+
shell: bash
53+
working-directory: ${{runner.workspace}}/silkworm/build/cmd
54+
run: |
55+
./rpcdaemon --datadir $ERIGON_DATA_DIR --api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --log.verbosity 1 --erigon_compatibility --jwt ./jwt.hex --skip_protocol_check --http.compression --eth.addr 127.0.0.1:51515 &
56+
SILKWORM_RPC_DAEMON_PID=$!
57+
echo "SILKWORM_RPC_DAEMON_PID=$SILKWORM_RPC_DAEMON_PID" >> $GITHUB_ENV
58+
59+
- name: Run Erigon RpcDaemon
60+
shell: bash
61+
run: |
62+
$ERIGON_DIR/rpcdaemon --datadir $ERIGON_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --verbosity 1 &
63+
ERIGON_RPC_DAEMON_PID=$!
64+
echo "ERIGON_RPC_DAEMON_PID=$ERIGON_RPC_DAEMON_PID" >> $GITHUB_ENV
65+
66+
- name: Run RPC Performance Tests
67+
id: test_step
68+
shell: bash
69+
run: |
70+
set +e # Disable exit on error
71+
failed_test=0
72+
commit=$(git -C ${{runner.workspace}}/silkworm rev-parse --short HEAD) # use ${{ github.sha }} or GITHUB_SHA
73+
past_test_dir=$RPC_PAST_TEST_DIR/mainnet_$(date +%Y%m%d_%H%M%S)_perf_$commit
74+
echo "past_test_dir=$past_test_dir" >> $GITHUB_ENV
75+
76+
# Prepare historical test results directory
77+
mkdir -p $past_test_dir
78+
rm -rf $RPC_PAST_TEST_DIR/mainnet_bin # we want only the latest binary files
79+
mkdir -p $RPC_PAST_TEST_DIR/mainnet_bin
80+
81+
run_perf () {
82+
servers=("silkworm" "erigon")
83+
for i in 1 2
84+
do
85+
network=$1
86+
method=$2
87+
pattern=$3
88+
sequence=$4
89+
90+
# clean temporary area
91+
cd ${{runner.workspace}}/rpc-tests/perf
92+
rm -rf ./reports/
93+
94+
python3 ./run_perf_tests.py --blockchain "$network" \
95+
--test-type "$method" \
96+
--pattern-file pattern/"$network"/"$pattern".tar \
97+
--test-sequence "$sequence" \
98+
--repetitions 5 \
99+
--silk-dir ${{runner.workspace}}/silkworm \
100+
--erigon-dir $ERIGON_DATA_DIR \
101+
--test-mode $i \
102+
--test-report \
103+
--json-report ./reports/mainnet/result.json \
104+
--testing-daemon ${servers[i-1]}
105+
106+
# Capture test runner script exit status
107+
perf_exit_status=$?
108+
109+
# Preserve test results
110+
mv ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/result.json ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/${servers[i-1]}-$method-result.json
111+
112+
# Detect the pre-built db version
113+
db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_DIR/production.ini production erigon_repo_commit)
114+
115+
# Check test runner script exit status
116+
if [ $perf_exit_status -eq 0 ]; then
117+
118+
# save all vegeta binary report
119+
echo "Save current vegeta binary files"
120+
cp -r ${{runner.workspace}}/rpc-tests/perf/reports/bin $RPC_PAST_TEST_DIR/mainnet_bin
121+
122+
echo "Save test result on DB"
123+
cd ${{runner.workspace}}/silkworm
124+
python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
125+
--repo silkworm \
126+
--branch ${{ github.ref_name }} \
127+
--commit $(git rev-parse HEAD) \
128+
--test_name rpc-performance-test-${servers[i-1]}-$method \
129+
--chain mainnet \
130+
--runner ${{ runner.name }} \
131+
--db_version $db_version \
132+
--outcome success \
133+
--result_file ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/${servers[i-1]}-$method-result.json
134+
135+
if [ $? -ne 0 ]; then
136+
failed_test=1
137+
echo "Failure saving test results on DB"
138+
fi
139+
140+
echo "Execute Latency Percentile HDR Analysis"
141+
cd ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/
142+
python3 $ERIGON_QA_PATH/test_system/qa-tests/rpc-tests/perf_hdr_analysis.py \
143+
--test_name ${servers[i-1]}-$method \
144+
--input_file ./result.json \
145+
--output_file ./${servers[i-1]}-${method}-latency_hdr_analysis.pdf
146+
else
147+
failed_test=1
148+
cd ${{runner.workspace}}/silkworm
149+
python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
150+
--repo silkworm \
151+
--branch ${{ github.ref_name }} \
152+
--commit $(git rev-parse HEAD) \
153+
--test_name rpc-performance-test-${servers[i-1]}-$method \
154+
--chain mainnet \
155+
--runner ${{ runner.name }} \
156+
--db_version $db_version \
157+
--outcome failure
158+
fi
159+
# Save test results to a directory with timestamp and commit hash
160+
cp -r ${{runner.workspace}}/rpc-tests/perf/reports/mainnet $past_test_dir
161+
done
162+
163+
}
164+
165+
# Launch the RPC performance test runner
166+
failed_test= 0
167+
if [[ ${{ inputs.activation_mode }}" == "light" ]]; then
168+
# Lightweight mode: narrower API coverage, shorter sequences
169+
run_perf mainnet eth_call stress_test_eth_call_20M 100:30,1000:20,10000:20,20000:20
170+
run_perf mainnet eth_getLogs stress_test_eth_getLogs_15M 100:30,1000:20,10000:20
171+
run_perf mainnet eth_getBalance stress_test_eth_getBalance_15M 100:30,1000:20,10000:20
172+
run_perf mainnet eth_getBlockByHash stress_test_eth_getBlockByHash_14M 100:30,1000:20
173+
run_perf mainnet eth_getTransactionByHash stress_test_eth_getTransactionByHash_13M 100:30,1000:20,10000:20
174+
run_perf mainnet eth_getTransactionReceipt stress_test_eth_getTransactionReceipt_14M 100:30
175+
else
176+
# Full mode: wider API coverage, longer sequences
177+
run_perf mainnet eth_call stress_test_eth_call_20M 1:1,100:30,1000:20,10000:20,20000:20
178+
run_perf mainnet eth_getLogs stress_test_eth_getLogs_15M 1:1,100:30,1000:20,10000:20,20000:20
179+
run_perf mainnet eth_getBalance stress_test_eth_getBalance_15M 1:1,100:30,1000:20,10000:20,20000:20
180+
run_perf mainnet eth_getBlockByHash stress_test_eth_getBlockByHash_14M 1:1,100:30,1000:20,10000:20
181+
run_perf mainnet eth_getBlockByNumber stress_test_eth_getBlockByNumber_13M 1:1,100:30,1000:20,5000:20
182+
run_perf mainnet eth_getTransactionByHash stress_test_eth_getTransactionByHash_13M 1:1,100:30,1000:20,10000:20
183+
run_perf mainnet eth_getTransactionReceipt stress_test_eth_getTransactionReceipt_14M 1:1,100:30,1000:20,5000:20,10000:20,20000:20
184+
run_perf mainnet eth_createAccessList stress_test_eth_createAccessList_16M 1:1,100:30,1000:20,10000:20,20000:20
185+
fi
186+
187+
if [ $failed_test -eq 0 ]; then
188+
echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
189+
echo "Tests completed successfully"
190+
else
191+
echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
192+
echo "Error detected during tests"
193+
fi
194+
195+
- name: Stop Silkworm RpcDaemon
196+
shell: bash
197+
working-directory: ${{runner.workspace}}/silkworm/build/cmd
198+
run: |
199+
# Clean up process if it's still running
200+
if kill -0 $SILKWORM_RPC_DAEMON_PID 2> /dev/null; then
201+
echo "Terminating Silkworm RpcDaemon"
202+
kill $SILKWORM_RPC_DAEMON_PID
203+
else
204+
echo "Silkworm RpcDaemon has already terminated"
205+
fi
206+
207+
- name: Stop Erigon RpcDaemon
208+
shell: bash
209+
run: |
210+
# Clean up process if it's still running
211+
if kill -0 $ERIGON_RPC_DAEMON_PID 2> /dev/null; then
212+
echo "Terminating Erigon RpcDaemon"
213+
kill $ERIGON_RPC_DAEMON_PID
214+
else
215+
echo "Erigon RpcDaemon has already terminated"
216+
fi
217+
218+
- name: Resume the Erigon instance dedicated to db maintenance
219+
shell: bash
220+
run: |
221+
python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
222+
223+
- name: Run change point analysis
224+
if: steps.test_step.outputs.TEST_RESULT == 'success'
225+
shell: bash
226+
working-directory: ${{runner.workspace}}/rpc-tests/perf/reports/mainnet
227+
run: |
228+
set +e # Disable exit on error
229+
open_change_points=0
230+
python3 $ERIGON_QA_PATH/test_system/qa-tests/change-points/change_point_analysis.py
231+
open_change_points=$?
232+
cp change_point_analysis.pdf $past_test_dir
233+
if [ $open_change_points -ne 0 ]; then
234+
echo "Change point analysis found points that need to be investigated"
235+
#echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" -- enable in the future
236+
fi
237+
238+
- name: Upload test results
239+
if: always()
240+
uses: actions/upload-artifact@v4
241+
with:
242+
name: test-results
243+
path: ${{ env.past_test_dir }}
244+
245+
- name: Action for Success
246+
if: steps.test_step.outputs.TEST_RESULT == 'success'
247+
shell: bash
248+
run: echo "::notice::Tests completed successfully"
249+
250+
- name: Action for Not Success
251+
if: steps.test_step.outputs.TEST_RESULT != 'success'
252+
shell: bash
253+
run: |
254+
echo "::error::Error detected during tests"
255+
exit 1

‎.github/workflows/rpc-performance-tests-light.yml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: QA - RPC Performance Tests Light
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
branches:
7+
- master
8+
types:
9+
- ready_for_review
10+
11+
jobs:
12+
performance-test-suite-light:
13+
strategy:
14+
matrix:
15+
backend: [ Erigon3 ]
16+
runs-on: [ self-hosted, "${{ matrix.backend }}" ]
17+
env:
18+
ERIGON_DIR: /opt/erigon-versions/reference-version
19+
ERIGON_DATA_DIR: /opt/erigon-versions/reference-version/datadir
20+
RPC_PAST_TEST_DIR: /opt/rpc-past-tests
21+
ERIGON_QA_PATH: /opt/erigon-qa
22+
23+
steps:
24+
- uses: ./.github/actions/perf-common-steps
25+
with:
26+
activation_mode: light

0 commit comments

Comments
 (0)