GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Commit dccc70c5 authored by Nick Mathewson's avatar Nick Mathewson 🥄

Parallelize src/test/test into chunks.

First, we introduce a flag to teach src/test/test to split its work
into chunks.  Then we replace our invocation of src/test/test in our
"make check" target with a set of 8 scripts that invoke the first
8th of the tests, the second 8th, and so on.

This change makes our "make -kj4 check" target in our hardened
gitlab build more than twice as fast, since src/test/test was taking
the longest to finish.

Closes 40098.
parent f5b94715
Pipeline #838 failed with stage
in 27 minutes and 23 seconds
o Minor features (tests):
- Our "make check" target now runs the unit tests in 8 parallel chunks.
Doing this speeds up hardened CI builds by more than a factor of two.
Closes ticket 40098.
......@@ -23,7 +23,15 @@ TESTSCRIPTS = \
src/test/test_workqueue_pipe.sh \
src/test/test_workqueue_pipe2.sh \
src/test/test_workqueue_socketpair.sh \
src/test/test_switch_id.sh
src/test/test_switch_id.sh \
src/test/unittest_part1.sh \
src/test/unittest_part2.sh \
src/test/unittest_part3.sh \
src/test/unittest_part4.sh \
src/test/unittest_part5.sh \
src/test/unittest_part6.sh \
src/test/unittest_part7.sh \
src/test/unittest_part8.sh
if USE_RUST
TESTSCRIPTS += \
......@@ -35,7 +43,7 @@ TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.s
TESTSCRIPTS += src/test/test_rebind.sh
endif
TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
TESTS += src/test/test-slow src/test/test-memwipe \
src/test/test_workqueue \
src/test/test_keygen.sh \
src/test/test_key_expiration.sh \
......
......@@ -278,6 +278,8 @@ main(int c, const char **v)
control_initialize_event_queue();
configure_backtrace_handler(get_version());
unsigned num=1, den=1;
for (i_out = i = 1; i < c; ++i) {
if (!strcmp(v[i], "--warn")) {
loglevel = LOG_WARN;
......@@ -289,6 +291,19 @@ main(int c, const char **v)
loglevel = LOG_DEBUG;
} else if (!strcmp(v[i], "--accel")) {
accel_crypto = 1;
} else if (!strcmp(v[i], "--fraction")) {
if (i+1 == c) {
printf("--fraction needs an argument.\n");
return 1;
}
const char *fracstr = v[++i];
char ch;
if (sscanf(fracstr, "%u/%u%c", &num, &den, &ch) != 2) {
printf("--fraction expects a fraction as an input.\n");
}
if (den == 0 || num == 0 || num > den) {
printf("--fraction expects a valid fraction as an input.\n");
}
} else {
v[i_out++] = v[i];
}
......@@ -363,6 +378,33 @@ main(int c, const char **v)
smartlist_free(skip);
}
if (den != 1) {
// count the tests. Linear but fast.
unsigned n_tests = 0;
struct testgroup_t *tg;
struct testcase_t *tc;
for (tg = testgroups; tg->prefix != NULL; ++tg) {
for (tc = tg->cases; tc->name != NULL; ++tc) {
++n_tests;
}
}
// Which tests should we run? This can give iffy results if den is huge
// but it doesn't actually matter in practice.
unsigned tests_per_chunk = CEIL_DIV(n_tests, den);
unsigned start_at = (num-1) * tests_per_chunk;
// Skip the tests that are outside of the range.
unsigned idx = 0;
for (tg = testgroups; tg->prefix != NULL; ++tg) {
for (tc = tg->cases; tc->name != NULL; ++tc) {
if (idx < start_at || idx >= start_at + tests_per_chunk) {
tc->flags |= TT_SKIP;
}
++idx;
}
}
}
int have_failed = (tinytest_main(c, v, testgroups) != 0);
free_pregenerated_keys();
......
#!/bin/sh
"${abs_top_builddir:-.}/src/test/test" --fraction 1/8
#!/bin/sh
"${abs_top_builddir:-.}/src/test/test" --fraction 2/8
#!/bin/sh
"${abs_top_builddir:-.}/src/test/test" --fraction 3/8
#!/bin/sh
"${abs_top_builddir:-.}/src/test/test" --fraction 4/8
#!/bin/sh
"${abs_top_builddir:-.}/src/test/test" --fraction 5/8
#!/bin/sh
"${abs_top_builddir:-.}/src/test/test" --fraction 6/8
#!/bin/sh
"${abs_top_builddir:-.}/src/test/test" --fraction 7/8
#!/bin/sh
"${abs_top_builddir:-.}/src/test/test" --fraction 8/8
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment