Commit 22690efcc2378222503cb8aaad26a6f4a539f5aa | zig

author	Andrew Kelley <andrew@ziglang.org> 2022-06-13 18:59:52-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2022-06-14 17:15:13-04:00
log	22690efcc2378222503cb8aaad26a6f4a539f5aa
tree	2dc41347d1db482beccbae41f8ecd078dc3ee9fe
parent	47c4d4450281bfddebf89cbad4f9c1fdcc7b0b65

multi-thread `zig build test-cases`

Instead of always using std.testing.allocator, the test harness now follows
the same logic as self-hosted for choosing an allocator - that is - it
uses C allocator when linking libc, std.testing.allocator otherwise, and
respects `-Dforce-gpa` to override the decision. I did this because
I found GeneralPurposeAllocator to be prohibitively slow when doing
multi-threading, even in the context of a debug build.

There is now a second thread pool which is used to spawn each
test case. The stage2 tests are passed the first thread pool. If it were
only multi-threading the stage1 tests then we could use the same thread
pool for everything. However, the problem with this strategy with stage2
is that stage2 wants to spawn tasks and then call wait() on the main
thread. If we use the same thread pool for everything, we get a deadlock
because all the threads end up all hanging at wait() and nothing is
getting done. So we use our second thread pool to simulate a "process pool"
of sorts.

I spent most of the time working on this commit scratching my head trying
to figure out why I was getting ETXTBSY when spawning the test cases.
Turns out it's a fundamental Unix design flaw, already a known, unsolved
issue by Go and Java maintainers:

https://github.com/golang/go/issues/22315
https://bugs.openjdk.org/browse/JDK-8068370

With this change, the following command, executed on my laptop, went from
6m24s to 1m44s:

```
stage1/bin/zig build test-cases -fqemu -fwasmtime -Denable-llvm
```

closes #11818

2 files changed, 137 insertions(+), 63 deletions(-)

build.zig
src
- test.zig

build.zig+1

...	@@ -398,6 +398,7 @@ pub fn build(b: *Builder) !void {	...	@@ -398,6 +398,7 @@ pub fn build(b: *Builder) !void {
398	test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);	398	test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
399	test_cases_options.addOption(bool, "llvm_has_ve", llvm_has_ve);	399	test_cases_options.addOption(bool, "llvm_has_ve", llvm_has_ve);
400	test_cases_options.addOption(bool, "llvm_has_arc", llvm_has_arc);	400	test_cases_options.addOption(bool, "llvm_has_arc", llvm_has_arc);
		401	test_cases_options.addOption(bool, "force_gpa", force_gpa);
401	test_cases_options.addOption(bool, "enable_qemu", b.enable_qemu);	402	test_cases_options.addOption(bool, "enable_qemu", b.enable_qemu);
402	test_cases_options.addOption(bool, "enable_wine", b.enable_wine);	403	test_cases_options.addOption(bool, "enable_wine", b.enable_wine);
403	test_cases_options.addOption(bool, "enable_wasmtime", b.enable_wasmtime);	404	test_cases_options.addOption(bool, "enable_wasmtime", b.enable_wasmtime);

src/test.zig+136-63

...	@@ -1,11 +1,19 @@	...	@@ -1,11 +1,19 @@
1	const std = @import("std");	1	const std = @import("std");
2	const builtin = @import("builtin");	2	const builtin = @import("builtin");
		3	const Allocator = std.mem.Allocator;
		4	const CrossTarget = std.zig.CrossTarget;
		5	const print = std.debug.print;
		6	const assert = std.debug.assert;
		7
3	const link = @import("link.zig");	8	const link = @import("link.zig");
4	const Compilation = @import("Compilation.zig");	9	const Compilation = @import("Compilation.zig");
5	const Allocator = std.mem.Allocator;
6	const Package = @import("Package.zig");	10	const Package = @import("Package.zig");
7	const introspect = @import("introspect.zig");	11	const introspect = @import("introspect.zig");
8	const build_options = @import("build_options");	12	const build_options = @import("build_options");
		13	const ThreadPool = @import("ThreadPool.zig");
		14	const WaitGroup = @import("WaitGroup.zig");
		15	const zig_h = link.File.C.zig_h;
		16
9	const enable_qemu: bool = build_options.enable_qemu;	17	const enable_qemu: bool = build_options.enable_qemu;
10	const enable_wine: bool = build_options.enable_wine;	18	const enable_wine: bool = build_options.enable_wine;
11	const enable_wasmtime: bool = build_options.enable_wasmtime;	19	const enable_wasmtime: bool = build_options.enable_wasmtime;
...	@@ -13,12 +21,6 @@ const enable_darling: bool = build_options.enable_darling;	...	@@ -13,12 +21,6 @@ const enable_darling: bool = build_options.enable_darling;
13	const enable_rosetta: bool = build_options.enable_rosetta;	21	const enable_rosetta: bool = build_options.enable_rosetta;
14	const glibc_runtimes_dir: ?[]const u8 = build_options.glibc_runtimes_dir;	22	const glibc_runtimes_dir: ?[]const u8 = build_options.glibc_runtimes_dir;
15	const skip_stage1 = build_options.skip_stage1;	23	const skip_stage1 = build_options.skip_stage1;
16	const ThreadPool = @import("ThreadPool.zig");
17	const CrossTarget = std.zig.CrossTarget;
18	const print = std.debug.print;
19	const assert = std.debug.assert;
20
21	const zig_h = link.File.C.zig_h;
22		24
23	const hr = "=" ** 80;	25	const hr = "=" ** 80;
24		26
...	@@ -27,11 +29,24 @@ test {	...	@@ -27,11 +29,24 @@ test {
27	@import("stage1.zig").os_init();	29	@import("stage1.zig").os_init();
28	}	30	}
29		31
30	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);	32	const use_gpa = build_options.force_gpa or !builtin.link_libc;
		33	const gpa = gpa: {
		34	if (use_gpa) {
		35	break :gpa std.testing.allocator;
		36	}
		37	// We would prefer to use raw libc allocator here, but cannot
		38	// use it if it won't support the alignment we need.
		39	if (@alignOf(std.c.max_align_t) < @alignOf(i128)) {
		40	break :gpa std.heap.c_allocator;
		41	}
		42	break :gpa std.heap.raw_c_allocator;
		43	};
		44
		45	var arena_allocator = std.heap.ArenaAllocator.init(gpa);
31	defer arena_allocator.deinit();	46	defer arena_allocator.deinit();
32	const arena = arena_allocator.allocator();	47	const arena = arena_allocator.allocator();
33		48
34	var ctx = TestContext.init(std.testing.allocator, arena);	49	var ctx = TestContext.init(gpa, arena);
35	defer ctx.deinit();	50	defer ctx.deinit();
36		51
37	{	52	{
...	@@ -536,6 +551,7 @@ fn sortTestFilenames(filenames: [][]const u8) void {	...	@@ -536,6 +551,7 @@ fn sortTestFilenames(filenames: [][]const u8) void {
536	}	551	}
537		552
538	pub const TestContext = struct {	553	pub const TestContext = struct {
		554	gpa: Allocator,
539	arena: Allocator,	555	arena: Allocator,
540	cases: std.ArrayList(Case),	556	cases: std.ArrayList(Case),
541		557
...	@@ -604,6 +620,8 @@ pub const TestContext = struct {	...	@@ -604,6 +620,8 @@ pub const TestContext = struct {
604		620
605	files: std.ArrayList(File),	621	files: std.ArrayList(File),
606		622
		623	result: anyerror!void = {},
		624
607	pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void {	625	pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void {
608	case.files.append(.{ .path = name, .src = src }) catch @panic("out of memory");	626	case.files.append(.{ .path = name, .src = src }) catch @panic("out of memory");
609	}	627	}
...	@@ -1185,6 +1203,7 @@ pub const TestContext = struct {	...	@@ -1185,6 +1203,7 @@ pub const TestContext = struct {
1185		1203
1186	fn init(gpa: Allocator, arena: Allocator) TestContext {	1204	fn init(gpa: Allocator, arena: Allocator) TestContext {
1187	return .{	1205	return .{
		1206	.gpa = gpa,
1188	.cases = std.ArrayList(Case).init(gpa),	1207	.cases = std.ArrayList(Case).init(gpa),
1189	.arena = arena,	1208	.arena = arena,
1190	};	1209	};
...	@@ -1204,19 +1223,23 @@ pub const TestContext = struct {	...	@@ -1204,19 +1223,23 @@ pub const TestContext = struct {
1204	}	1223	}
1205		1224
1206	fn run(self: *TestContext) !void {	1225	fn run(self: *TestContext) !void {
1207	const host = try std.zig.system.NativeTargetInfo.detect(std.testing.allocator, .{});	1226	const host = try std.zig.system.NativeTargetInfo.detect(self.gpa, .{});
1208		1227
1209	var progress = std.Progress{};	1228	var progress = std.Progress{};
1210	const root_node = progress.start("compiler", self.cases.items.len);	1229	const root_node = progress.start("compiler", self.cases.items.len);
1211	defer root_node.end();	1230	defer root_node.end();
1212		1231
1213	var zig_lib_directory = try introspect.findZigLibDir(std.testing.allocator);	1232	var zig_lib_directory = try introspect.findZigLibDir(self.gpa);
1214	defer zig_lib_directory.handle.close();	1233	defer zig_lib_directory.handle.close();
1215	defer std.testing.allocator.free(zig_lib_directory.path.?);	1234	defer self.gpa.free(zig_lib_directory.path.?);
		1235
		1236	var aux_thread_pool: ThreadPool = undefined;
		1237	try aux_thread_pool.init(self.gpa);
		1238	defer aux_thread_pool.deinit();
1216		1239
1217	var thread_pool: ThreadPool = undefined;	1240	var case_thread_pool: ThreadPool = undefined;
1218	try thread_pool.init(std.testing.allocator);	1241	try case_thread_pool.init(self.gpa);
1219	defer thread_pool.deinit();	1242	defer case_thread_pool.deinit();
1220		1243
1221	// Use the same global cache dir for all the tests, such that we for example don't have to	1244	// Use the same global cache dir for all the tests, such that we for example don't have to
1222	// rebuild musl libc for every case (when LLVM backend is enabled).	1245	// rebuild musl libc for every case (when LLVM backend is enabled).
...	@@ -1225,60 +1248,90 @@ pub const TestContext = struct {	...	@@ -1225,60 +1248,90 @@ pub const TestContext = struct {
1225		1248
1226	var cache_dir = try global_tmp.dir.makeOpenPath("zig-cache", .{});	1249	var cache_dir = try global_tmp.dir.makeOpenPath("zig-cache", .{});
1227	defer cache_dir.close();	1250	defer cache_dir.close();
1228	const tmp_dir_path = try std.fs.path.join(std.testing.allocator, &[_][]const u8{ ".", "zig-cache", "tmp", &global_tmp.sub_path });	1251	const tmp_dir_path = try std.fs.path.join(self.gpa, &[_][]const u8{ ".", "zig-cache", "tmp", &global_tmp.sub_path });
1229	defer std.testing.allocator.free(tmp_dir_path);	1252	defer self.gpa.free(tmp_dir_path);
1230		1253
1231	const global_cache_directory: Compilation.Directory = .{	1254	const global_cache_directory: Compilation.Directory = .{
1232	.handle = cache_dir,	1255	.handle = cache_dir,
1233	.path = try std.fs.path.join(std.testing.allocator, &[_][]const u8{ tmp_dir_path, "zig-cache" }),	1256	.path = try std.fs.path.join(self.gpa, &[_][]const u8{ tmp_dir_path, "zig-cache" }),
1234	};	1257	};
1235	defer std.testing.allocator.free(global_cache_directory.path.?);	1258	defer self.gpa.free(global_cache_directory.path.?);
1236		1259
1237	var fail_count: usize = 0;	1260	{
		1261	var wait_group: WaitGroup = .{};
		1262	defer wait_group.wait();
		1263
		1264	for (self.cases.items) \|*case\| {
		1265	if (build_options.skip_non_native) {
		1266	if (case.target.getCpuArch() != builtin.cpu.arch)
		1267	continue;
		1268	if (case.target.getObjectFormat() != builtin.object_format)
		1269	continue;
		1270	}
1238		1271
1239	for (self.cases.items) \|case\| {	1272	// Skip tests that require LLVM backend when it is not available
1240	if (build_options.skip_non_native) {	1273	if (!build_options.have_llvm and case.backend == .llvm)
1241	if (case.target.getCpuArch() != builtin.cpu.arch)
1242	continue;	1274	continue;
1243	if (case.target.getObjectFormat() != builtin.object_format)
1244	continue;
1245	}
1246		1275
1247	// Skip tests that require LLVM backend when it is not available	1276	if (build_options.test_filter) \|test_filter\| {
1248	if (!build_options.have_llvm and case.backend == .llvm)	1277	if (std.mem.indexOf(u8, case.name, test_filter) == null) continue;
1249	continue;	1278	}
1250		1279
1251	if (build_options.test_filter) \|test_filter\| {	1280	wait_group.start();
1252	if (std.mem.indexOf(u8, case.name, test_filter) == null) continue;	1281	try case_thread_pool.spawn(workerRunOneCase, .{
		1282	self.gpa,
		1283	root_node,
		1284	case,
		1285	zig_lib_directory,
		1286	&aux_thread_pool,
		1287	global_cache_directory,
		1288	host,
		1289	&wait_group,
		1290	});
1253	}	1291	}
1254	var prg_node = root_node.start(case.name, case.updates.items.len);	1292	}
1255	prg_node.activate();	1293
1256	defer prg_node.end();	1294	var fail_count: usize = 0;
1257		1295	for (self.cases.items) \|*case\| {
1258	// So that we can see which test case failed when the leak checker goes off,	1296	case.result catch \|err\| {
1259	// or there's an internal error
1260	progress.initial_delay_ns = 0;
1261	progress.refresh_rate_ns = 0;
1262
1263	runOneCase(
1264	std.testing.allocator,
1265	&prg_node,
1266	case,
1267	zig_lib_directory,
1268	&thread_pool,
1269	global_cache_directory,
1270	host,
1271	) catch \|err\| {
1272	fail_count += 1;	1297	fail_count += 1;
1273	print("test '{s}' failed: {s}\n\n", .{ case.name, @errorName(err) });	1298	print("{s} failed: {s}\n", .{ case.name, @errorName(err) });
1274	};	1299	};
1275	}	1300	}
		1301
1276	if (fail_count != 0) {	1302	if (fail_count != 0) {
1277	print("{d} tests failed\n", .{fail_count});	1303	print("{d} tests failed\n", .{fail_count});
1278	return error.TestFailed;	1304	return error.TestFailed;
1279	}	1305	}
1280	}	1306	}
1281		1307
		1308	fn workerRunOneCase(
		1309	gpa: Allocator,
		1310	root_node: *std.Progress.Node,
		1311	case: *Case,
		1312	zig_lib_directory: Compilation.Directory,
		1313	thread_pool: *ThreadPool,
		1314	global_cache_directory: Compilation.Directory,
		1315	host: std.zig.system.NativeTargetInfo,
		1316	wait_group: *WaitGroup,
		1317	) void {
		1318	defer wait_group.finish();
		1319
		1320	var prg_node = root_node.start(case.name, case.updates.items.len);
		1321	prg_node.activate();
		1322	defer prg_node.end();
		1323
		1324	case.result = runOneCase(
		1325	gpa,
		1326	&prg_node,
		1327	case.*,
		1328	zig_lib_directory,
		1329	thread_pool,
		1330	global_cache_directory,
		1331	host,
		1332	);
		1333	}
		1334
1282	fn runOneCase(	1335	fn runOneCase(
1283	allocator: Allocator,	1336	allocator: Allocator,
1284	root_node: *std.Progress.Node,	1337	root_node: *std.Progress.Node,
...	@@ -1368,6 +1421,11 @@ pub const TestContext = struct {	...	@@ -1368,6 +1421,11 @@ pub const TestContext = struct {
1368	try zig_args.append("-O");	1421	try zig_args.append("-O");
1369	try zig_args.append(@tagName(case.optimize_mode));	1422	try zig_args.append(@tagName(case.optimize_mode));
1370		1423
		1424	// Prevent sub-process progress bar from interfering with the
		1425	// one in this parent process.
		1426	try zig_args.append("--color");
		1427	try zig_args.append("off");
		1428
1371	const result = try std.ChildProcess.exec(.{	1429	const result = try std.ChildProcess.exec(.{
1372	.allocator = arena,	1430	.allocator = arena,
1373	.argv = zig_args.items,	1431	.argv = zig_args.items,
...	@@ -1529,6 +1587,8 @@ pub const TestContext = struct {	...	@@ -1529,6 +1587,8 @@ pub const TestContext = struct {
1529	.use_llvm = use_llvm,	1587	.use_llvm = use_llvm,
1530	.use_stage1 = null, // We already handled stage1 tests	1588	.use_stage1 = null, // We already handled stage1 tests
1531	.self_exe_path = std.testing.zig_exe_path,	1589	.self_exe_path = std.testing.zig_exe_path,
		1590	// TODO instead of turning off color, pass in a std.Progress.Node
		1591	.color = .off,
1532	});	1592	});
1533	defer comp.destroy();	1593	defer comp.destroy();
1534		1594
...	@@ -1820,18 +1880,31 @@ pub const TestContext = struct {	...	@@ -1820,18 +1880,31 @@ pub const TestContext = struct {
1820		1880
1821	try comp.makeBinFileExecutable();	1881	try comp.makeBinFileExecutable();
1822		1882
1823	break :x std.ChildProcess.exec(.{	1883	while (true) {
1824	.allocator = allocator,	1884	break :x std.ChildProcess.exec(.{
1825	.argv = argv.items,	1885	.allocator = allocator,
1826	.cwd_dir = tmp.dir,	1886	.argv = argv.items,
1827	.cwd = tmp_dir_path,	1887	.cwd_dir = tmp.dir,
1828	}) catch \|err\| {	1888	.cwd = tmp_dir_path,
1829	print("\nupdate_index={d} The following command failed with {s}:\n", .{	1889	}) catch \|err\| switch (err) {
1830	update_index, @errorName(err),	1890	error.FileBusy => {
1831	});	1891	// There is a fundamental design flaw in Unix systems with how
1832	dumpArgs(argv.items);	1892	// ETXTBSY interacts with fork+exec.
1833	return error.ChildProcessExecution;	1893	// https://github.com/golang/go/issues/22315
1834	};	1894	// https://bugs.openjdk.org/browse/JDK-8068370
		1895	// Unfortunately, this could be a real error, but we can't
		1896	// tell the difference here.
		1897	continue;
		1898	},
		1899	else => {
		1900	print("\n{s}.{d} The following command failed with {s}:\n", .{
		1901	case.name, update_index, @errorName(err),
		1902	});
		1903	dumpArgs(argv.items);
		1904	return error.ChildProcessExecution;
		1905	},
		1906	};
		1907	}
1835	};	1908	};
1836	var test_node = update_node.start("test", 0);	1909	var test_node = update_node.start("test", 0);
1837	test_node.activate();	1910	test_node.activate();