Skip to content

Commit 31226c8

Browse files
ZippeyKeys12ZippeyKeys12
authored
[CN-Test-Gen] Parallelize CI tests via make -j (#876)
1 parent ba4a383 commit 31226c8

File tree

5 files changed

+91
-93
lines changed

5 files changed

+91
-93
lines changed

‎backend/cn/lib/testGeneration/buildScript.ml

+2
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,8 @@ let run () =
213213
let coverage ~filename_base =
214214
string "# Coverage"
215215
^^ hardline
216+
^^ string "echo"
217+
^^ hardline
216218
^^ attempt
217219
("gcov \"" ^ filename_base ^ "_test.c\"")
218220
"Recorded coverage via gcov."

‎runtime/libcn/src/cn-testing/test.c

-1
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,6 @@ int cn_test_main(int argc, char* argv[]) {
303303
test_case->func(true, CN_TEST_GEN_PROGRESS_NONE, sizing_strategy, trap);
304304

305305
set_cn_logging_level(CN_LOGGING_NONE);
306-
printf("\n");
307306

308307
break;
309308
case CN_TEST_GEN_FAIL:

‎tests/cn-test-gen/Makefile

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
SRC_DIR := ./src
2+
3+
SRCS := $(shell find $(SRC_DIR) -name '*.c')
4+
TESTS := $(SRCS:%=testing-%)
5+
6+
test: $(TESTS)
7+
8+
$(TESTS): testing-%: %
9+
./run-single-test.sh $<

‎tests/cn-test-gen/run-single-test.sh

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/bin/bash
2+
3+
CN=$OPAM_SWITCH_PREFIX/bin/cn
4+
5+
DIRNAME=$(dirname "$0")
6+
TEST=$1
7+
8+
# Clean directory
9+
cd "$DIRNAME" || exit
10+
if [[ $TEST == *.pass.c ]]; then
11+
DIR=passing-$(basename $TEST .pass.c)
12+
else
13+
DIR=failing-$(basename $TEST .fail.c)
14+
fi
15+
rm -rf $DIR
16+
17+
# For UBSan
18+
export UBSAN_OPTIONS=halt_on_error=1
19+
20+
# Track failures
21+
NUM_FAILED=0
22+
FAILED=''
23+
24+
function separator() {
25+
OUTPUT="${OUTPUT}\n"
26+
for i in {1..60}; do
27+
OUTPUT="${OUTPUT}="
28+
done
29+
OUTPUT="${OUTPUT}\n\n"
30+
}
31+
32+
CONFIGS=("--sized-null --sizing-strategy 0" "--coverage --sizing-strategy 1" "--with-static-hack --coverage --sizing-strategy 2" "--random-size-splits" "--random-size-splits --allowed-size-split-backtracks=10")
33+
34+
OUTPUT=""
35+
36+
# For each configuration
37+
for CONFIG in "${CONFIGS[@]}"; do
38+
separator
39+
OUTPUT="${OUTPUT}Running CI with CLI config \"$CONFIG\"\n"
40+
separator
41+
42+
FULL_CONFIG="$CONFIG --input-timeout=1000 --progress-level=1 --sanitize=undefined"
43+
44+
if [[ $TEST == *.pass.c ]]; then
45+
CLEANUP="rm -rf ${DIR} run_tests.sh;separator"
46+
OUTPUT="${OUTPUT}$($CN test "$TEST" --output-dir="$DIR" $FULL_CONFIG 2>&1)"
47+
RET=$?
48+
if [[ "$RET" != 0 ]]; then
49+
OUTPUT="${OUTPUT}\n$TEST -- Tests failed unexpectedly\n"
50+
NUM_FAILED=$(($NUM_FAILED + 1))
51+
FAILED="$FAILED $CONFIG"
52+
else
53+
OUTPUT="${OUTPUT}\n$TEST -- Tests passed successfully\n"
54+
fi
55+
elif [[ $TEST == *.fail.c ]]; then
56+
CLEANUP="rm -rf ${DIR} run_tests.sh;separator"
57+
OUTPUT="${OUTPUT}$($CN test "$TEST" --output-dir="$DIR" $FULL_CONFIG 2>&1)"
58+
RET=$?
59+
if [[ "$RET" = 0 ]]; then
60+
OUTPUT="${OUTPUT}\n$TEST -- Tests passed unexpectedly\n"
61+
NUM_FAILED=$(($NUM_FAILED + 1))
62+
FAILED="$FAILED $CONFIG"
63+
else
64+
OUTPUT="${OUTPUT}\n$TEST -- Tests failed successfully"
65+
fi
66+
fi
67+
68+
eval "$CLEANUP"
69+
done
70+
71+
if [ -z "$FAILED" ]; then
72+
# echo "$TEST - all configs passed."
73+
exit 0
74+
else
75+
OUTPUT="${OUTPUT}$TEST - $NUM_FAILED configs failed:\n $FAILED"
76+
printf "%s\n" "${OUTPUT}"
77+
exit 1
78+
fi

‎tests/run-cn-test-gen.sh

+2-92
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,5 @@
11
#!/bin/bash
22

3-
# copying from run-ci.sh
4-
export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(ocamlfind query z3)
5-
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(ocamlfind query z3)
6-
CN=$OPAM_SWITCH_PREFIX/bin/cn
3+
cd cn-test-gen
74

8-
DIRNAME=$(dirname "$0")
9-
10-
# Clean directory
11-
cd "$DIRNAME"/cn-test-gen || exit
12-
rm -rf build decorated test
13-
mkdir build decorated test
14-
15-
# For UBSan
16-
export UBSAN_OPTIONS=halt_on_error=1
17-
18-
# Get `*.c` files
19-
FILES=$(find "$DIRNAME"/src -name '*.c')
20-
21-
# Track failures
22-
NUM_FAILED=0
23-
FAILED=''
24-
25-
function separator() {
26-
printf '\n'
27-
for i in {1..60}; do
28-
printf "="
29-
done
30-
printf '\n\n'
31-
}
32-
33-
CONFIGS=("--sized-null --sizing-strategy 0" "--coverage --sizing-strategy 1" "--with-static-hack --coverage --sanitize=undefined --sizing-strategy 2" "--random-size-splits" "--random-size-splits --allowed-size-split-backtracks=10")
34-
35-
# For each configuration
36-
for CONFIG in "${CONFIGS[@]}"; do
37-
separator
38-
echo "Running CI with CLI config \"$CONFIG\""
39-
separator
40-
41-
FULL_CONFIG="$CONFIG --input-timeout=1000 --progress-level=1"
42-
43-
# Test each `*.c` file
44-
for TEST in $FILES; do
45-
CLEANUP="rm -rf test/* run_tests.sh;separator"
46-
47-
# Run passing tests
48-
if [[ $TEST == *.pass.c ]]; then
49-
$CN test "$TEST" --output-dir="test" $FULL_CONFIG
50-
RET=$?
51-
if [[ "$RET" != 0 ]]; then
52-
echo
53-
echo "$TEST -- Tests failed unexpectedly"
54-
NUM_FAILED=$(($NUM_FAILED + 1))
55-
FAILED="$FAILED $TEST($CONFIG)"
56-
eval "$CLEANUP"
57-
continue
58-
else
59-
echo
60-
echo "$TEST -- Tests passed successfully"
61-
fi
62-
fi
63-
64-
# Run failing tests
65-
if [[ $TEST == *.fail.c ]]; then
66-
$CN test "$TEST" --output-dir="test" $FULL_CONFIG
67-
RET=$?
68-
if [[ "$RET" = 0 ]]; then
69-
echo
70-
echo "$TEST -- Tests passed unexpectedly"
71-
NUM_FAILED=$(($NUM_FAILED + 1))
72-
FAILED="$FAILED $TEST($CONFIG)"
73-
eval "$CLEANUP"
74-
continue
75-
else
76-
echo
77-
echo "$TEST -- Tests failed successfully"
78-
fi
79-
fi
80-
81-
eval "$CLEANUP"
82-
done
83-
done
84-
85-
echo 'Done running tests.'
86-
echo
87-
88-
if [ -z "$FAILED" ]; then
89-
echo "All tests passed."
90-
exit 0
91-
else
92-
echo "$NUM_FAILED tests failed:"
93-
echo " $FAILED"
94-
exit 1
95-
fi
5+
make -j

0 commit comments

Comments
 (0)