Commit 22690efcc2378222503cb8aaad26a6f4a539f5aa | zig

author	Andrew Kelley <andrew@ziglang.org> 2022-06-13 18:59:52-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2022-06-14 17:15:13-04:00
log	22690efcc2378222503cb8aaad26a6f4a539f5aa
tree	2dc41347d1db482beccbae41f8ecd078dc3ee9fe
parent	47c4d4450281bfddebf89cbad4f9c1fdcc7b0b65

multi-thread `zig build test-cases`

Instead of always using std.testing.allocator, the test harness now follows
the same logic as self-hosted for choosing an allocator - that is - it
uses C allocator when linking libc, std.testing.allocator otherwise, and
respects `-Dforce-gpa` to override the decision. I did this because
I found GeneralPurposeAllocator to be prohibitively slow when doing
multi-threading, even in the context of a debug build.

There is now a second thread pool which is used to spawn each
test case. The stage2 tests are passed the first thread pool. If it were
only multi-threading the stage1 tests then we could use the same thread
pool for everything. However, the problem with this strategy with stage2
is that stage2 wants to spawn tasks and then call wait() on the main
thread. If we use the same thread pool for everything, we get a deadlock
because all the threads end up all hanging at wait() and nothing is
getting done. So we use our second thread pool to simulate a "process pool"
of sorts.

I spent most of the time working on this commit scratching my head trying
to figure out why I was getting ETXTBSY when spawning the test cases.
Turns out it's a fundamental Unix design flaw, already a known, unsolved
issue by Go and Java maintainers:

https://github.com/golang/go/issues/22315
https://bugs.openjdk.org/browse/JDK-8068370

With this change, the following command, executed on my laptop, went from
6m24s to 1m44s:

```
stage1/bin/zig build test-cases -fqemu -fwasmtime -Denable-llvm
```

closes #11818

2 files changed, 137 insertions(+), 63 deletions(-)

build.zig
src
- test.zig

build.zig+1

...	...	@@ -398,6 +398,7 @@ pub fn build(b: *Builder) !void {
398	398	test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
399	399	test_cases_options.addOption(bool, "llvm_has_ve", llvm_has_ve);
400	400	test_cases_options.addOption(bool, "llvm_has_arc", llvm_has_arc);
	401	test_cases_options.addOption(bool, "force_gpa", force_gpa);
401	402	test_cases_options.addOption(bool, "enable_qemu", b.enable_qemu);
402	403	test_cases_options.addOption(bool, "enable_wine", b.enable_wine);
403	404	test_cases_options.addOption(bool, "enable_wasmtime", b.enable_wasmtime);

src/test.zig+136-63

...	...	@@ -1,11 +1,19 @@
1	1	const std = @import("std");
2	2	const builtin = @import("builtin");
	3	const Allocator = std.mem.Allocator;
	4	const CrossTarget = std.zig.CrossTarget;
	5	const print = std.debug.print;
	6	const assert = std.debug.assert;
	7
3	8	const link = @import("link.zig");
4	9	const Compilation = @import("Compilation.zig");
5		const Allocator = std.mem.Allocator;
6	10	const Package = @import("Package.zig");
7	11	const introspect = @import("introspect.zig");
8	12	const build_options = @import("build_options");
	13	const ThreadPool = @import("ThreadPool.zig");
	14	const WaitGroup = @import("WaitGroup.zig");
	15	const zig_h = link.File.C.zig_h;
	16
9	17	const enable_qemu: bool = build_options.enable_qemu;
10	18	const enable_wine: bool = build_options.enable_wine;
11	19	const enable_wasmtime: bool = build_options.enable_wasmtime;
...	...	@@ -13,12 +21,6 @@ const enable_darling: bool = build_options.enable_darling;
13	21	const enable_rosetta: bool = build_options.enable_rosetta;
14	22	const glibc_runtimes_dir: ?[]const u8 = build_options.glibc_runtimes_dir;
15	23	const skip_stage1 = build_options.skip_stage1;
16		const ThreadPool = @import("ThreadPool.zig");
17		const CrossTarget = std.zig.CrossTarget;
18		const print = std.debug.print;
19		const assert = std.debug.assert;
20
21		const zig_h = link.File.C.zig_h;
22	24
23	25	const hr = "=" ** 80;
24	26
...	...	@@ -27,11 +29,24 @@ test {
27	29	@import("stage1.zig").os_init();
28	30	}
29	31
30		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	32	const use_gpa = build_options.force_gpa or !builtin.link_libc;
	33	const gpa = gpa: {
	34	if (use_gpa) {
	35	break :gpa std.testing.allocator;
	36	}
	37	// We would prefer to use raw libc allocator here, but cannot
	38	// use it if it won't support the alignment we need.
	39	if (@alignOf(std.c.max_align_t) < @alignOf(i128)) {
	40	break :gpa std.heap.c_allocator;
	41	}
	42	break :gpa std.heap.raw_c_allocator;
	43	};
	44
	45	var arena_allocator = std.heap.ArenaAllocator.init(gpa);
31	46	defer arena_allocator.deinit();
32	47	const arena = arena_allocator.allocator();
33	48
34		var ctx = TestContext.init(std.testing.allocator, arena);
	49	var ctx = TestContext.init(gpa, arena);
35	50	defer ctx.deinit();
36	51
37	52	{
...	...	@@ -536,6 +551,7 @@ fn sortTestFilenames(filenames: [][]const u8) void {
536	551	}
537	552
538	553	pub const TestContext = struct {
	554	gpa: Allocator,
539	555	arena: Allocator,
540	556	cases: std.ArrayList(Case),
541	557
...	...	@@ -604,6 +620,8 @@ pub const TestContext = struct {
604	620
605	621	files: std.ArrayList(File),
606	622
	623	result: anyerror!void = {},
	624
607	625	pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void {
608	626	case.files.append(.{ .path = name, .src = src }) catch @panic("out of memory");
609	627	}
...	...	@@ -1185,6 +1203,7 @@ pub const TestContext = struct {
1185	1203
1186	1204	fn init(gpa: Allocator, arena: Allocator) TestContext {
1187	1205	return .{
	1206	.gpa = gpa,
1188	1207	.cases = std.ArrayList(Case).init(gpa),
1189	1208	.arena = arena,
1190	1209	};
...	...	@@ -1204,19 +1223,23 @@ pub const TestContext = struct {
1204	1223	}
1205	1224
1206	1225	fn run(self: *TestContext) !void {
1207		const host = try std.zig.system.NativeTargetInfo.detect(std.testing.allocator, .{});
	1226	const host = try std.zig.system.NativeTargetInfo.detect(self.gpa, .{});
1208	1227
1209	1228	var progress = std.Progress{};
1210	1229	const root_node = progress.start("compiler", self.cases.items.len);
1211	1230	defer root_node.end();
1212	1231
1213		var zig_lib_directory = try introspect.findZigLibDir(std.testing.allocator);
	1232	var zig_lib_directory = try introspect.findZigLibDir(self.gpa);
1214	1233	defer zig_lib_directory.handle.close();
1215		defer std.testing.allocator.free(zig_lib_directory.path.?);
	1234	defer self.gpa.free(zig_lib_directory.path.?);
	1235
	1236	var aux_thread_pool: ThreadPool = undefined;
	1237	try aux_thread_pool.init(self.gpa);
	1238	defer aux_thread_pool.deinit();
1216	1239
1217		var thread_pool: ThreadPool = undefined;
1218		try thread_pool.init(std.testing.allocator);
1219		defer thread_pool.deinit();
	1240	var case_thread_pool: ThreadPool = undefined;
	1241	try case_thread_pool.init(self.gpa);
	1242	defer case_thread_pool.deinit();
1220	1243
1221	1244	// Use the same global cache dir for all the tests, such that we for example don't have to
1222	1245	// rebuild musl libc for every case (when LLVM backend is enabled).
...	...	@@ -1225,60 +1248,90 @@ pub const TestContext = struct {
1225	1248
1226	1249	var cache_dir = try global_tmp.dir.makeOpenPath("zig-cache", .{});
1227	1250	defer cache_dir.close();
1228		const tmp_dir_path = try std.fs.path.join(std.testing.allocator, &[_][]const u8{ ".", "zig-cache", "tmp", &global_tmp.sub_path });
1229		defer std.testing.allocator.free(tmp_dir_path);
	1251	const tmp_dir_path = try std.fs.path.join(self.gpa, &[_][]const u8{ ".", "zig-cache", "tmp", &global_tmp.sub_path });
	1252	defer self.gpa.free(tmp_dir_path);
1230	1253
1231	1254	const global_cache_directory: Compilation.Directory = .{
1232	1255	.handle = cache_dir,
1233		.path = try std.fs.path.join(std.testing.allocator, &[_][]const u8{ tmp_dir_path, "zig-cache" }),
	1256	.path = try std.fs.path.join(self.gpa, &[_][]const u8{ tmp_dir_path, "zig-cache" }),
1234	1257	};
1235		defer std.testing.allocator.free(global_cache_directory.path.?);
1236
1237		var fail_count: usize = 0;
	1258	defer self.gpa.free(global_cache_directory.path.?);
	1259
	1260	{
	1261	var wait_group: WaitGroup = .{};
	1262	defer wait_group.wait();
	1263
	1264	for (self.cases.items) \|*case\| {
	1265	if (build_options.skip_non_native) {
	1266	if (case.target.getCpuArch() != builtin.cpu.arch)
	1267	continue;
	1268	if (case.target.getObjectFormat() != builtin.object_format)
	1269	continue;
	1270	}
1238	1271
1239		for (self.cases.items) \|case\| {
1240		if (build_options.skip_non_native) {
1241		if (case.target.getCpuArch() != builtin.cpu.arch)
	1272	// Skip tests that require LLVM backend when it is not available
	1273	if (!build_options.have_llvm and case.backend == .llvm)
1242	1274	continue;
1243		if (case.target.getObjectFormat() != builtin.object_format)
1244		continue;
1245		}
1246	1275
1247		// Skip tests that require LLVM backend when it is not available
1248		if (!build_options.have_llvm and case.backend == .llvm)
1249		continue;
	1276	if (build_options.test_filter) \|test_filter\| {
	1277	if (std.mem.indexOf(u8, case.name, test_filter) == null) continue;
	1278	}
1250	1279
1251		if (build_options.test_filter) \|test_filter\| {
1252		if (std.mem.indexOf(u8, case.name, test_filter) == null) continue;
	1280	wait_group.start();
	1281	try case_thread_pool.spawn(workerRunOneCase, .{
	1282	self.gpa,
	1283	root_node,
	1284	case,
	1285	zig_lib_directory,
	1286	&aux_thread_pool,
	1287	global_cache_directory,
	1288	host,
	1289	&wait_group,
	1290	});
1253	1291	}
1254		var prg_node = root_node.start(case.name, case.updates.items.len);
1255		prg_node.activate();
1256		defer prg_node.end();
1257
1258		// So that we can see which test case failed when the leak checker goes off,
1259		// or there's an internal error
1260		progress.initial_delay_ns = 0;
1261		progress.refresh_rate_ns = 0;
1262
1263		runOneCase(
1264		std.testing.allocator,
1265		&prg_node,
1266		case,
1267		zig_lib_directory,
1268		&thread_pool,
1269		global_cache_directory,
1270		host,
1271		) catch \|err\| {
	1292	}
	1293
	1294	var fail_count: usize = 0;
	1295	for (self.cases.items) \|*case\| {
	1296	case.result catch \|err\| {
1272	1297	fail_count += 1;
1273		print("test '{s}' failed: {s}\n\n", .{ case.name, @errorName(err) });
	1298	print("{s} failed: {s}\n", .{ case.name, @errorName(err) });
1274	1299	};
1275	1300	}
	1301
1276	1302	if (fail_count != 0) {
1277	1303	print("{d} tests failed\n", .{fail_count});
1278	1304	return error.TestFailed;
1279	1305	}
1280	1306	}
1281	1307
	1308	fn workerRunOneCase(
	1309	gpa: Allocator,
	1310	root_node: *std.Progress.Node,
	1311	case: *Case,
	1312	zig_lib_directory: Compilation.Directory,
	1313	thread_pool: *ThreadPool,
	1314	global_cache_directory: Compilation.Directory,
	1315	host: std.zig.system.NativeTargetInfo,
	1316	wait_group: *WaitGroup,
	1317	) void {
	1318	defer wait_group.finish();
	1319
	1320	var prg_node = root_node.start(case.name, case.updates.items.len);
	1321	prg_node.activate();
	1322	defer prg_node.end();
	1323
	1324	case.result = runOneCase(
	1325	gpa,
	1326	&prg_node,
	1327	case.*,
	1328	zig_lib_directory,
	1329	thread_pool,
	1330	global_cache_directory,
	1331	host,
	1332	);
	1333	}
	1334
1282	1335	fn runOneCase(
1283	1336	allocator: Allocator,
1284	1337	root_node: *std.Progress.Node,
...	...	@@ -1368,6 +1421,11 @@ pub const TestContext = struct {
1368	1421	try zig_args.append("-O");
1369	1422	try zig_args.append(@tagName(case.optimize_mode));
1370	1423
	1424	// Prevent sub-process progress bar from interfering with the
	1425	// one in this parent process.
	1426	try zig_args.append("--color");
	1427	try zig_args.append("off");
	1428
1371	1429	const result = try std.ChildProcess.exec(.{
1372	1430	.allocator = arena,
1373	1431	.argv = zig_args.items,
...	...	@@ -1529,6 +1587,8 @@ pub const TestContext = struct {
1529	1587	.use_llvm = use_llvm,
1530	1588	.use_stage1 = null, // We already handled stage1 tests
1531	1589	.self_exe_path = std.testing.zig_exe_path,
	1590	// TODO instead of turning off color, pass in a std.Progress.Node
	1591	.color = .off,
1532	1592	});
1533	1593	defer comp.destroy();
1534	1594
...	...	@@ -1820,18 +1880,31 @@ pub const TestContext = struct {
1820	1880
1821	1881	try comp.makeBinFileExecutable();
1822	1882
1823		break :x std.ChildProcess.exec(.{
1824		.allocator = allocator,
1825		.argv = argv.items,
1826		.cwd_dir = tmp.dir,
1827		.cwd = tmp_dir_path,
1828		}) catch \|err\| {
1829		print("\nupdate_index={d} The following command failed with {s}:\n", .{
1830		update_index, @errorName(err),
1831		});
1832		dumpArgs(argv.items);
1833		return error.ChildProcessExecution;
1834		};
	1883	while (true) {
	1884	break :x std.ChildProcess.exec(.{
	1885	.allocator = allocator,
	1886	.argv = argv.items,
	1887	.cwd_dir = tmp.dir,
	1888	.cwd = tmp_dir_path,
	1889	}) catch \|err\| switch (err) {
	1890	error.FileBusy => {
	1891	// There is a fundamental design flaw in Unix systems with how
	1892	// ETXTBSY interacts with fork+exec.
	1893	// https://github.com/golang/go/issues/22315
	1894	// https://bugs.openjdk.org/browse/JDK-8068370
	1895	// Unfortunately, this could be a real error, but we can't
	1896	// tell the difference here.
	1897	continue;
	1898	},
	1899	else => {
	1900	print("\n{s}.{d} The following command failed with {s}:\n", .{
	1901	case.name, update_index, @errorName(err),
	1902	});
	1903	dumpArgs(argv.items);
	1904	return error.ChildProcessExecution;
	1905	},
	1906	};
	1907	}
1835	1908	};
1836	1909	var test_node = update_node.start("test", 0);
1837	1910	test_node.activate();