Commit 0d1cd0d4822628c104890af4c31cdf38c6f96d35 | zig

src/Compilation.zig+45-17

...	...	@@ -26,6 +26,8 @@ const Module = @import("Module.zig");
26	26	const Cache = @import("Cache.zig");
27	27	const stage1 = @import("stage1.zig");
28	28	const translate_c = @import("translate_c.zig");
	29	const ThreadPool = @import("ThreadPool.zig");
	30	const WaitGroup = @import("WaitGroup.zig");
29	31
30	32	/// General-purpose allocator. Used for both temporary and long-term storage.
31	33	gpa: *Allocator,
...	...	@@ -79,6 +81,7 @@ zig_lib_directory: Directory,
79	81	local_cache_directory: Directory,
80	82	global_cache_directory: Directory,
81	83	libc_include_dir_list: []const []const u8,
	84	thread_pool: *ThreadPool,
82	85
83	86	/// Populated when we build the libc++ static library. A Job to build this is placed in the queue
84	87	/// and resolved before calling linker.flush().
...	...	@@ -335,6 +338,7 @@ pub const InitOptions = struct {
335	338	root_name: []const u8,
336	339	root_pkg: ?*Package,
337	340	output_mode: std.builtin.OutputMode,
	341	thread_pool: *ThreadPool,
338	342	dynamic_linker: ?[]const u8 = null,
339	343	/// `null` means to not emit a binary file.
340	344	emit_bin: ?EmitLoc,
...	...	@@ -985,6 +989,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
985	989	.self_exe_path = options.self_exe_path,
986	990	.libc_include_dir_list = libc_dirs.libc_include_dir_list,
987	991	.sanitize_c = sanitize_c,
	992	.thread_pool = options.thread_pool,
988	993	.clang_passthrough_mode = options.clang_passthrough_mode,
989	994	.clang_preprocessor_mode = options.clang_preprocessor_mode,
990	995	.verbose_cc = options.verbose_cc,
...	...	@@ -1380,24 +1385,14 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
1380	1385	var c_comp_progress_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
1381	1386	defer c_comp_progress_node.end();
1382	1387
	1388	var wg = WaitGroup{};
	1389	defer wg.wait();
	1390
1383	1391	while (self.c_object_work_queue.readItem()) \|c_object\| {
1384		self.updateCObject(c_object, &c_comp_progress_node) catch \|err\| switch (err) {
1385		error.AnalysisFail => continue,
1386		else => {
1387		{
1388		var lock = self.mutex.acquire();
1389		defer lock.release();
1390		try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1);
1391		self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create(
1392		self.gpa,
1393		0,
1394		"unable to build C object: {s}",
1395		.{@errorName(err)},
1396		));
1397		}
1398		c_object.status = .{ .failure = {} };
1399		},
1400		};
	1392	wg.start();
	1393	try self.thread_pool.spawn(workerUpdateCObject, .{
	1394	self, c_object, &c_comp_progress_node, &wg,
	1395	});
1401	1396	}
1402	1397
1403	1398	while (self.work_queue.readItem()) \|work_item\| switch (work_item) {
...	...	@@ -1721,6 +1716,37 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
1721	1716	};
1722	1717	}
1723	1718
	1719	fn workerUpdateCObject(
	1720	comp: *Compilation,
	1721	c_object: *CObject,
	1722	progress_node: *std.Progress.Node,
	1723	wg: *WaitGroup,
	1724	) void {
	1725	defer wg.stop();
	1726
	1727	comp.updateCObject(c_object, progress_node) catch \|err\| switch (err) {
	1728	error.AnalysisFail => return,
	1729	else => {
	1730	{
	1731	var lock = comp.mutex.acquire();
	1732	defer lock.release();
	1733	comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1) catch {
	1734	fatal("TODO handle this by setting c_object.status = oom failure", .{});
	1735	};
	1736	comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, ErrorMsg.create(
	1737	comp.gpa,
	1738	0,
	1739	"unable to build C object: {s}",
	1740	.{@errorName(err)},
	1741	) catch {
	1742	fatal("TODO handle this by setting c_object.status = oom failure", .{});
	1743	});
	1744	}
	1745	c_object.status = .{ .failure = {} };
	1746	},
	1747	};
	1748	}
	1749
1724	1750	fn updateCObject(comp: Compilation, c_object: CObject, c_comp_progress_node: *std.Progress.Node) !void {
1725	1751	if (!build_options.have_llvm) {
1726	1752	return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});
...	...	@@ -2800,6 +2826,7 @@ fn buildOutputFromZig(
2800	2826	.root_name = root_name,
2801	2827	.root_pkg = &root_pkg,
2802	2828	.output_mode = fixed_output_mode,
	2829	.thread_pool = comp.thread_pool,
2803	2830	.libc_installation = comp.bin_file.options.libc_installation,
2804	2831	.emit_bin = emit_bin,
2805	2832	.optimize_mode = optimize_mode,
...	...	@@ -3173,6 +3200,7 @@ pub fn build_crt_file(
3173	3200	.root_name = root_name,
3174	3201	.root_pkg = null,
3175	3202	.output_mode = output_mode,
	3203	.thread_pool = comp.thread_pool,
3176	3204	.libc_installation = comp.bin_file.options.libc_installation,
3177	3205	.emit_bin = emit_bin,
3178	3206	.optimize_mode = comp.bin_file.options.optimize_mode,

src/ThreadPool.zig created+116

...	...	@@ -0,0 +1,116 @@
	1	const std = @import("std");
	2	const ThreadPool = @This();
	3
	4	lock: std.Mutex = .{},
	5	is_running: bool = true,
	6	allocator: *std.mem.Allocator,
	7	running: usize = 0,
	8	threads: []*std.Thread,
	9	run_queue: RunQueue = .{},
	10	idle_queue: IdleQueue = .{},
	11
	12	const IdleQueue = std.SinglyLinkedList(std.AutoResetEvent);
	13	const RunQueue = std.SinglyLinkedList(Runnable);
	14	const Runnable = struct {
	15	runFn: fn (*Runnable) void,
	16	};
	17
	18	pub fn init(self: ThreadPool, allocator: std.mem.Allocator) !void {
	19	self.* = .{
	20	.allocator = allocator,
	21	.threads = &[_]*std.Thread{},
	22	};
	23
	24	errdefer self.deinit();
	25
	26	var num_threads = std.Thread.cpuCount() catch 1;
	27	if (num_threads > 0)
	28	self.threads = try allocator.alloc(*std.Thread, num_threads);
	29
	30	while (num_threads > 0) : (num_threads -= 1) {
	31	const thread = try std.Thread.spawn(self, runWorker);
	32	self.threads[self.running] = thread;
	33	self.running += 1;
	34	}
	35	}
	36
	37	pub fn deinit(self: *ThreadPool) void {
	38	self.shutdown();
	39
	40	std.debug.assert(!self.is_running);
	41	for (self.threads[0..self.running]) \|thread\|
	42	thread.wait();
	43
	44	defer self.threads = &[_]*std.Thread{};
	45	if (self.running > 0)
	46	self.allocator.free(self.threads);
	47	}
	48
	49	pub fn shutdown(self: *ThreadPool) void {
	50	const held = self.lock.acquire();
	51
	52	if (!self.is_running)
	53	return held.release();
	54
	55	var idle_queue = self.idle_queue;
	56	self.idle_queue = .{};
	57	self.is_running = false;
	58	held.release();
	59
	60	while (idle_queue.popFirst()) \|idle_node\|
	61	idle_node.data.set();
	62	}
	63
	64	pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
	65	const Args = @TypeOf(args);
	66	const Closure = struct {
	67	arguments: Args,
	68	pool: *ThreadPool,
	69	run_node: RunQueue.Node = .{ .data = .{ .runFn = runFn } },
	70
	71	fn runFn(runnable: *Runnable) void {
	72	const run_node = @fieldParentPtr(RunQueue.Node, "data", runnable);
	73	const closure = @fieldParentPtr(@This(), "run_node", run_node);
	74	const result = @call(.{}, func, closure.arguments);
	75	closure.pool.allocator.destroy(closure);
	76	}
	77	};
	78
	79	const closure = try self.allocator.create(Closure);
	80	errdefer self.allocator.destroy(closure);
	81	closure.* = .{
	82	.arguments = args,
	83	.pool = self,
	84	};
	85
	86	const held = self.lock.acquire();
	87	self.run_queue.prepend(&closure.run_node);
	88
	89	const idle_node = self.idle_queue.popFirst();
	90	held.release();
	91
	92	if (idle_node) \|node\|
	93	node.data.set();
	94	}
	95
	96	fn runWorker(self: *ThreadPool) void {
	97	while (true) {
	98	const held = self.lock.acquire();
	99
	100	if (self.run_queue.popFirst()) \|run_node\| {
	101	held.release();
	102	(run_node.data.runFn)(&run_node.data);
	103	continue;
	104	}
	105
	106	if (!self.is_running) {
	107	held.release();
	108	return;
	109	}
	110
	111	var idle_node = IdleQueue.Node{ .data = .{} };
	112	self.idle_queue.prepend(&idle_node);
	113	held.release();
	114	idle_node.data.wait();
	115	}
	116	}

src/WaitGroup.zig created+22

...	...	@@ -0,0 +1,22 @@
	1	const std = @import("std");
	2	const WaitGroup = @This();
	3
	4	counter: usize = 0,
	5	event: ?*std.AutoResetEvent = null,
	6
	7	pub fn start(self: *WaitGroup) void {
	8	_ = @atomicRmw(usize, &self.counter, .Add, 1, .SeqCst);
	9	}
	10
	11	pub fn stop(self: *WaitGroup) void {
	12	if (@atomicRmw(usize, &self.counter, .Sub, 1, .SeqCst) == 1)
	13	if (@atomicRmw(?*std.AutoResetEvent, &self.event, .Xchg, null, .SeqCst)) \|event\|
	14	event.set();
	15	}
	16
	17	pub fn wait(self: *WaitGroup) void {
	18	var event = std.AutoResetEvent{};
	19	@atomicStore(?*std.AutoResetEvent, &self.event, &event, .SeqCst);
	20	if (@atomicLoad(usize, &self.counter, .SeqCst) != 0)
	21	event.wait();
	22	}

src/glibc.zig+1

...	...	@@ -936,6 +936,7 @@ fn buildSharedLib(
936	936	.root_pkg = null,
937	937	.output_mode = .Lib,
938	938	.link_mode = .Dynamic,
	939	.thread_pool = comp.thread_pool,
939	940	.libc_installation = comp.bin_file.options.libc_installation,
940	941	.emit_bin = emit_bin,
941	942	.optimize_mode = comp.bin_file.options.optimize_mode,

src/libcxx.zig+2

...	...	@@ -162,6 +162,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
162	162	.root_name = root_name,
163	163	.root_pkg = null,
164	164	.output_mode = output_mode,
	165	.thread_pool = comp.thread_pool,
165	166	.libc_installation = comp.bin_file.options.libc_installation,
166	167	.emit_bin = emit_bin,
167	168	.optimize_mode = comp.bin_file.options.optimize_mode,
...	...	@@ -280,6 +281,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
280	281	.root_name = root_name,
281	282	.root_pkg = null,
282	283	.output_mode = output_mode,
	284	.thread_pool = comp.thread_pool,
283	285	.libc_installation = comp.bin_file.options.libc_installation,
284	286	.emit_bin = emit_bin,
285	287	.optimize_mode = comp.bin_file.options.optimize_mode,

src/libunwind.zig+1

...	...	@@ -95,6 +95,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
95	95	.root_name = root_name,
96	96	.root_pkg = null,
97	97	.output_mode = output_mode,
	98	.thread_pool = comp.thread_pool,
98	99	.libc_installation = comp.bin_file.options.libc_installation,
99	100	.emit_bin = emit_bin,
100	101	.optimize_mode = comp.bin_file.options.optimize_mode,

src/main.zig+10

...	...	@@ -19,6 +19,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
19	19	const translate_c = @import("translate_c.zig");
20	20	const Cache = @import("Cache.zig");
21	21	const target_util = @import("target.zig");
	22	const ThreadPool = @import("ThreadPool.zig");
22	23
23	24	pub fn fatal(comptime format: []const u8, args: anytype) noreturn {
24	25	std.log.emerg(format, args);
...	...	@@ -1632,6 +1633,10 @@ fn buildOutputType(
1632	1633	};
1633	1634	defer zig_lib_directory.handle.close();
1634	1635
	1636	var thread_pool: ThreadPool = undefined;
	1637	try thread_pool.init(gpa);
	1638	defer thread_pool.deinit();
	1639
1635	1640	var libc_installation: ?LibCInstallation = null;
1636	1641	defer if (libc_installation) \|*l\| l.deinit(gpa);
1637	1642
...	...	@@ -1747,6 +1752,7 @@ fn buildOutputType(
1747	1752	.single_threaded = single_threaded,
1748	1753	.function_sections = function_sections,
1749	1754	.self_exe_path = self_exe_path,
	1755	.thread_pool = &thread_pool,
1750	1756	.clang_passthrough_mode = arg_mode != .build,
1751	1757	.clang_preprocessor_mode = clang_preprocessor_mode,
1752	1758	.version = optional_version,
...	...	@@ -2412,6 +2418,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !v
2412	2418	.directory = null, // Use the local zig-cache.
2413	2419	.basename = exe_basename,
2414	2420	};
	2421	var thread_pool: ThreadPool = undefined;
	2422	try thread_pool.init(gpa);
	2423	defer thread_pool.deinit();
2415	2424	const comp = Compilation.create(gpa, .{
2416	2425	.zig_lib_directory = zig_lib_directory,
2417	2426	.local_cache_directory = local_cache_directory,
...	...	@@ -2427,6 +2436,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !v
2427	2436	.emit_h = null,
2428	2437	.optimize_mode = .Debug,
2429	2438	.self_exe_path = self_exe_path,
	2439	.thread_pool = &thread_pool,
2430	2440	}) catch \|err\| {
2431	2441	fatal("unable to create compilation: {}", .{@errorName(err)});
2432	2442	};

src/musl.zig+1

...	...	@@ -200,6 +200,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
200	200	.root_pkg = null,
201	201	.output_mode = .Lib,
202	202	.link_mode = .Dynamic,
	203	.thread_pool = comp.thread_pool,
203	204	.libc_installation = comp.bin_file.options.libc_installation,
204	205	.emit_bin = Compilation.EmitLoc{ .directory = null, .basename = "libc.so" },
205	206	.optimize_mode = comp.bin_file.options.optimize_mode,

src/test.zig+14-1

...	...	@@ -10,6 +10,7 @@ const enable_qemu: bool = build_options.enable_qemu;
10	10	const enable_wine: bool = build_options.enable_wine;
11	11	const enable_wasmtime: bool = build_options.enable_wasmtime;
12	12	const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir;
	13	const ThreadPool = @import("ThreadPool.zig");
13	14
14	15	const cheader = @embedFile("link/cbe.h");
15	16
...	...	@@ -467,6 +468,10 @@ pub const TestContext = struct {
467	468	defer zig_lib_directory.handle.close();
468	469	defer std.testing.allocator.free(zig_lib_directory.path.?);
469	470
	471	var thread_pool: ThreadPool = undefined;
	472	try thread_pool.init(std.testing.allocator);
	473	defer thread_pool.deinit();
	474
470	475	for (self.cases.items) \|case\| {
471	476	if (build_options.skip_non_native and case.target.getCpuArch() != std.Target.current.cpu.arch)
472	477	continue;
...	...	@@ -480,7 +485,13 @@ pub const TestContext = struct {
480	485	progress.initial_delay_ns = 0;
481	486	progress.refresh_rate_ns = 0;
482	487
483		try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_directory);
	488	try self.runOneCase(
	489	std.testing.allocator,
	490	&prg_node,
	491	case,
	492	zig_lib_directory,
	493	&thread_pool,
	494	);
484	495	}
485	496	}
486	497
...	...	@@ -490,6 +501,7 @@ pub const TestContext = struct {
490	501	root_node: *std.Progress.Node,
491	502	case: Case,
492	503	zig_lib_directory: Compilation.Directory,
	504	thread_pool: *ThreadPool,
493	505	) !void {
494	506	const target_info = try std.zig.system.NativeTargetInfo.detect(allocator, case.target);
495	507	const target = target_info.target;
...	...	@@ -539,6 +551,7 @@ pub const TestContext = struct {
539	551	.local_cache_directory = zig_cache_directory,
540	552	.global_cache_directory = zig_cache_directory,
541	553	.zig_lib_directory = zig_lib_directory,
	554	.thread_pool = thread_pool,
542	555	.root_name = "test_case",
543	556	.target = target,
544	557	// TODO: support tests for object file building, and library builds

author	Andrew Kelley <andrew@ziglang.org> 2020-12-15 18:30:29-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-12-20 15:08:59-07:00
log	0d1cd0d4822628c104890af4c31cdf38c6f96d35
tree	f61949f7abc888284fcd33c4696fcb71701f5e16
parent	01d33855c736b04160d9616f138442aa4e41a738

use kprotty's ThreadPool implementation (v5)