Commit 4918605176c2e48c178847ea281b790334207733 | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-12-20 21:19:05-05:00
committer	GitHub <noreply@github.com> 2020-12-20 21:19:05-05:00
log	4918605176c2e48c178847ea281b790334207733
tree	4adb4cbf5e6764df65747fb3bae2c1640060eb89
parent	4964bb3282bf13de03a79fad1fb9bca104dc1930
parent	1d94a6893689ad1fb8e06308ae51603a6c8708a8
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #7462 from ziglang/parallel-c-objects

Introduce a ThreadPool and parallel execution of some of the compilation work items

19 files changed, 731 insertions(+), 383 deletions(-)

CMakeLists.txt
ci
- drone
  - linux_script
lib
- std
  - Progress.zig
  - auto_reset_event.zig
  - progress.zig
  - special
    test_runner.zig
  - std.zig
src

CMakeLists.txt+11-1

...	...	@@ -75,6 +75,7 @@ set(ZIG_TARGET_TRIPLE "native" CACHE STRING "arch-os-abi to output binaries for"
75	75	set(ZIG_TARGET_MCPU "baseline" CACHE STRING "-mcpu parameter to output binaries for")
76	76	set(ZIG_EXECUTABLE "" CACHE STRING "(when cross compiling) path to already-built zig binary")
77	77	set(ZIG_PREFER_LLVM_CONFIG off CACHE BOOL "(when cross compiling) use llvm-config to find target llvm dependencies if needed")
	78	set(ZIG_SINGLE_THREADED off CACHE BOOL "limit the zig compiler to use only 1 thread")
78	79
79	80	find_package(llvm)
80	81	find_package(clang)
...	...	@@ -410,7 +411,7 @@ set(ZIG_STAGE2_SOURCES
410	411	"${CMAKE_SOURCE_DIR}/lib/std/os/windows/win32error.zig"
411	412	"${CMAKE_SOURCE_DIR}/lib/std/pdb.zig"
412	413	"${CMAKE_SOURCE_DIR}/lib/std/process.zig"
413		"${CMAKE_SOURCE_DIR}/lib/std/progress.zig"
	414	"${CMAKE_SOURCE_DIR}/lib/std/Progress.zig"
414	415	"${CMAKE_SOURCE_DIR}/lib/std/rand.zig"
415	416	"${CMAKE_SOURCE_DIR}/lib/std/reset_event.zig"
416	417	"${CMAKE_SOURCE_DIR}/lib/std/sort.zig"
...	...	@@ -510,10 +511,13 @@ set(ZIG_STAGE2_SOURCES
510	511	"${CMAKE_SOURCE_DIR}/src/Cache.zig"
511	512	"${CMAKE_SOURCE_DIR}/src/Compilation.zig"
512	513	"${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig"
	514	"${CMAKE_SOURCE_DIR}/src/Event.zig"
513	515	"${CMAKE_SOURCE_DIR}/src/Module.zig"
514	516	"${CMAKE_SOURCE_DIR}/src/Package.zig"
515	517	"${CMAKE_SOURCE_DIR}/src/RangeSet.zig"
	518	"${CMAKE_SOURCE_DIR}/src/ThreadPool.zig"
516	519	"${CMAKE_SOURCE_DIR}/src/TypedValue.zig"
	520	"${CMAKE_SOURCE_DIR}/src/WaitGroup.zig"
517	521	"${CMAKE_SOURCE_DIR}/src/astgen.zig"
518	522	"${CMAKE_SOURCE_DIR}/src/clang.zig"
519	523	"${CMAKE_SOURCE_DIR}/src/clang_options.zig"
...	...	@@ -713,6 +717,11 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
713	717	else()
714	718	set(ZIG1_RELEASE_ARG -OReleaseFast --strip)
715	719	endif()
	720	if(ZIG_SINGLE_THREADED)
	721	set(ZIG1_SINGLE_THREADED_ARG "--single-threaded")
	722	else()
	723	set(ZIG1_SINGLE_THREADED_ARG "")
	724	endif()
716	725
717	726	set(BUILD_ZIG1_ARGS
718	727	"src/stage1.zig"
...	...	@@ -722,6 +731,7 @@ set(BUILD_ZIG1_ARGS
722	731	--override-lib-dir "${CMAKE_SOURCE_DIR}/lib"
723	732	"-femit-bin=${ZIG1_OBJECT}"
724	733	"${ZIG1_RELEASE_ARG}"
	734	"${ZIG1_SINGLE_THREADED_ARG}"
725	735	-lc
726	736	--pkg-begin build_options "${ZIG_CONFIG_ZIG_OUT}"
727	737	--pkg-end

ci/drone/linux_script+2-1

...	...	@@ -17,7 +17,8 @@ git config core.abbrev 9
17	17
18	18	mkdir build
19	19	cd build
20		cmake .. -DCMAKE_BUILD_TYPE=Release "-DCMAKE_INSTALL_PREFIX=$DISTDIR" -DZIG_STATIC=ON -DCMAKE_PREFIX_PATH=/deps/local -GNinja
	20	# TODO figure out why Drone CI is deadlocking and stop passing -DZIG_SINGLE_THREADED=ON
	21	cmake .. -DCMAKE_BUILD_TYPE=Release "-DCMAKE_INSTALL_PREFIX=$DISTDIR" -DZIG_STATIC=ON -DCMAKE_PREFIX_PATH=/deps/local -GNinja -DZIG_SINGLE_THREADED=ON
21	22
22	23	samu install
23	24	./zig build test -Dskip-release -Dskip-non-native

lib/std/Progress.zig created+345

...	...	@@ -0,0 +1,345 @@
	1	// SPDX-License-Identifier: MIT
	2	// Copyright (c) 2015-2020 Zig Contributors
	3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
	4	// The MIT license requires this copyright notice to be included in all copies
	5	// and substantial portions of the software.
	6
	7	//! This API non-allocating, non-fallible, and thread-safe.
	8	//! The tradeoff is that users of this API must provide the storage
	9	//! for each `Progress.Node`.
	10	//!
	11	//! Initialize the struct directly, overriding these fields as desired:
	12	//! * `refresh_rate_ms`
	13	//! * `initial_delay_ms`
	14
	15	const std = @import("std");
	16	const windows = std.os.windows;
	17	const testing = std.testing;
	18	const assert = std.debug.assert;
	19	const Progress = @This();
	20
	21	/// `null` if the current node (and its children) should
	22	/// not print on update()
	23	terminal: ?std.fs.File = undefined,
	24
	25	/// Whether the terminal supports ANSI escape codes.
	26	supports_ansi_escape_codes: bool = false,
	27
	28	root: Node = undefined,
	29
	30	/// Keeps track of how much time has passed since the beginning.
	31	/// Used to compare with `initial_delay_ms` and `refresh_rate_ms`.
	32	timer: std.time.Timer = undefined,
	33
	34	/// When the previous refresh was written to the terminal.
	35	/// Used to compare with `refresh_rate_ms`.
	36	prev_refresh_timestamp: u64 = undefined,
	37
	38	/// This buffer represents the maximum number of bytes written to the terminal
	39	/// with each refresh.
	40	output_buffer: [100]u8 = undefined,
	41
	42	/// How many nanoseconds between writing updates to the terminal.
	43	refresh_rate_ns: u64 = 50 * std.time.ns_per_ms,
	44
	45	/// How many nanoseconds to keep the output hidden
	46	initial_delay_ns: u64 = 500 * std.time.ns_per_ms,
	47
	48	done: bool = true,
	49
	50	/// Protects the `refresh` function, as well as `node.recently_updated_child`.
	51	/// Without this, callsites would call `Node.end` and then free `Node` memory
	52	/// while it was still being accessed by the `refresh` function.
	53	update_lock: std.Mutex = .{},
	54
	55	/// Keeps track of how many columns in the terminal have been output, so that
	56	/// we can move the cursor back later.
	57	columns_written: usize = undefined,
	58
	59	/// Represents one unit of progress. Each node can have children nodes, or
	60	/// one can use integers with `update`.
	61	pub const Node = struct {
	62	context: *Progress,
	63	parent: ?*Node,
	64	name: []const u8,
	65	/// Must be handled atomically to be thread-safe.
	66	recently_updated_child: ?*Node = null,
	67	/// Must be handled atomically to be thread-safe. 0 means null.
	68	unprotected_estimated_total_items: usize,
	69	/// Must be handled atomically to be thread-safe.
	70	unprotected_completed_items: usize,
	71
	72	/// Create a new child progress node. Thread-safe.
	73	/// Call `Node.end` when done.
	74	/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
	75	/// API to set `self.parent.recently_updated_child` with the return value.
	76	/// Until that is fixed you probably want to call `activate` on the return value.
	77	/// Passing 0 for `estimated_total_items` means unknown.
	78	pub fn start(self: *Node, name: []const u8, estimated_total_items: usize) Node {
	79	return Node{
	80	.context = self.context,
	81	.parent = self,
	82	.name = name,
	83	.unprotected_estimated_total_items = estimated_total_items,
	84	.unprotected_completed_items = 0,
	85	};
	86	}
	87
	88	/// This is the same as calling `start` and then `end` on the returned `Node`. Thread-safe.
	89	pub fn completeOne(self: *Node) void {
	90	self.activate();
	91	_ = @atomicRmw(usize, &self.unprotected_completed_items, .Add, 1, .Monotonic);
	92	self.context.maybeRefresh();
	93	}
	94
	95	/// Finish a started `Node`. Thread-safe.
	96	pub fn end(self: *Node) void {
	97	self.context.maybeRefresh();
	98	if (self.parent) \|parent\| {
	99	{
	100	const held = self.context.update_lock.acquire();
	101	defer held.release();
	102	_ = @cmpxchgStrong(?*Node, &parent.recently_updated_child, self, null, .Monotonic, .Monotonic);
	103	}
	104	parent.completeOne();
	105	} else {
	106	self.context.done = true;
	107	self.context.refresh();
	108	}
	109	}
	110
	111	/// Tell the parent node that this node is actively being worked on. Thread-safe.
	112	pub fn activate(self: *Node) void {
	113	if (self.parent) \|parent\| {
	114	@atomicStore(?*Node, &parent.recently_updated_child, self, .Release);
	115	}
	116	}
	117
	118	/// Thread-safe. 0 means unknown.
	119	pub fn setEstimatedTotalItems(self: *Node, count: usize) void {
	120	@atomicStore(usize, &self.unprotected_estimated_total_items, count, .Monotonic);
	121	}
	122
	123	/// Thread-safe.
	124	pub fn setCompletedItems(self: *Node, completed_items: usize) void {
	125	@atomicStore(usize, &self.unprotected_completed_items, completed_items, .Monotonic);
	126	}
	127	};
	128
	129	/// Create a new progress node.
	130	/// Call `Node.end` when done.
	131	/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
	132	/// API to return Progress rather than accept it as a parameter.
	133	/// `estimated_total_items` value of 0 means unknown.
	134	pub fn start(self: Progress, name: []const u8, estimated_total_items: usize) !Node {
	135	const stderr = std.io.getStdErr();
	136	self.terminal = null;
	137	if (stderr.supportsAnsiEscapeCodes()) {
	138	self.terminal = stderr;
	139	self.supports_ansi_escape_codes = true;
	140	} else if (std.builtin.os.tag == .windows and stderr.isTty()) {
	141	self.terminal = stderr;
	142	}
	143	self.root = Node{
	144	.context = self,
	145	.parent = null,
	146	.name = name,
	147	.unprotected_estimated_total_items = estimated_total_items,
	148	.unprotected_completed_items = 0,
	149	};
	150	self.columns_written = 0;
	151	self.prev_refresh_timestamp = 0;
	152	self.timer = try std.time.Timer.start();
	153	self.done = false;
	154	return &self.root;
	155	}
	156
	157	/// Updates the terminal if enough time has passed since last update. Thread-safe.
	158	pub fn maybeRefresh(self: *Progress) void {
	159	const now = self.timer.read();
	160	if (now < self.initial_delay_ns) return;
	161	const held = self.update_lock.tryAcquire() orelse return;
	162	defer held.release();
	163	if (now - self.prev_refresh_timestamp < self.refresh_rate_ns) return;
	164	return self.refreshWithHeldLock();
	165	}
	166
	167	/// Updates the terminal and resets `self.next_refresh_timestamp`. Thread-safe.
	168	pub fn refresh(self: *Progress) void {
	169	const held = self.update_lock.tryAcquire() orelse return;
	170	defer held.release();
	171
	172	return self.refreshWithHeldLock();
	173	}
	174
	175	fn refreshWithHeldLock(self: *Progress) void {
	176	const file = self.terminal orelse return;
	177
	178	const prev_columns_written = self.columns_written;
	179	var end: usize = 0;
	180	if (self.columns_written > 0) {
	181	// restore the cursor position by moving the cursor
	182	// `columns_written` cells to the left, then clear the rest of the
	183	// line
	184	if (self.supports_ansi_escape_codes) {
	185	end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[{d}D", .{self.columns_written}) catch unreachable).len;
	186	end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[0K", .{}) catch unreachable).len;
	187	} else if (std.builtin.os.tag == .windows) winapi: {
	188	var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;
	189	if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.TRUE)
	190	unreachable;
	191
	192	var cursor_pos = windows.COORD{
	193	.X = info.dwCursorPosition.X - @intCast(windows.SHORT, self.columns_written),
	194	.Y = info.dwCursorPosition.Y,
	195	};
	196
	197	if (cursor_pos.X < 0)
	198	cursor_pos.X = 0;
	199
	200	const fill_chars = @intCast(windows.DWORD, info.dwSize.X - cursor_pos.X);
	201
	202	var written: windows.DWORD = undefined;
	203	if (windows.kernel32.FillConsoleOutputAttribute(
	204	file.handle,
	205	info.wAttributes,
	206	fill_chars,
	207	cursor_pos,
	208	&written,
	209	) != windows.TRUE) {
	210	// Stop trying to write to this file.
	211	self.terminal = null;
	212	break :winapi;
	213	}
	214	if (windows.kernel32.FillConsoleOutputCharacterA(
	215	file.handle,
	216	' ',
	217	fill_chars,
	218	cursor_pos,
	219	&written,
	220	) != windows.TRUE) unreachable;
	221
	222	if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE)
	223	unreachable;
	224	} else unreachable;
	225
	226	self.columns_written = 0;
	227	}
	228
	229	if (!self.done) {
	230	var need_ellipse = false;
	231	var maybe_node: ?*Node = &self.root;
	232	while (maybe_node) \|node\| {
	233	if (need_ellipse) {
	234	self.bufWrite(&end, "... ", .{});
	235	}
	236	need_ellipse = false;
	237	const eti = @atomicLoad(usize, &node.unprotected_estimated_total_items, .Monotonic);
	238	const completed_items = @atomicLoad(usize, &node.unprotected_completed_items, .Monotonic);
	239	if (node.name.len != 0 or eti > 0) {
	240	if (node.name.len != 0) {
	241	self.bufWrite(&end, "{s}", .{node.name});
	242	need_ellipse = true;
	243	}
	244	if (eti > 0) {
	245	if (need_ellipse) self.bufWrite(&end, " ", .{});
	246	self.bufWrite(&end, "[{d}/{d}] ", .{ completed_items + 1, eti });
	247	need_ellipse = false;
	248	} else if (completed_items != 0) {
	249	if (need_ellipse) self.bufWrite(&end, " ", .{});
	250	self.bufWrite(&end, "[{d}] ", .{completed_items + 1});
	251	need_ellipse = false;
	252	}
	253	}
	254	maybe_node = @atomicLoad(?*Node, &node.recently_updated_child, .Acquire);
	255	}
	256	if (need_ellipse) {
	257	self.bufWrite(&end, "... ", .{});
	258	}
	259	}
	260
	261	_ = file.write(self.output_buffer[0..end]) catch \|e\| {
	262	// Stop trying to write to this file once it errors.
	263	self.terminal = null;
	264	};
	265	self.prev_refresh_timestamp = self.timer.read();
	266	}
	267
	268	pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void {
	269	const file = self.terminal orelse return;
	270	self.refresh();
	271	file.outStream().print(format, args) catch {
	272	self.terminal = null;
	273	return;
	274	};
	275	self.columns_written = 0;
	276	}
	277
	278	fn bufWrite(self: Progress, end: usize, comptime format: []const u8, args: anytype) void {
	279	if (std.fmt.bufPrint(self.output_buffer[end.*..], format, args)) \|written\| {
	280	const amt = written.len;
	281	end.* += amt;
	282	self.columns_written += amt;
	283	} else \|err\| switch (err) {
	284	error.NoSpaceLeft => {
	285	self.columns_written += self.output_buffer.len - end.*;
	286	end.* = self.output_buffer.len;
	287	},
	288	}
	289	const bytes_needed_for_esc_codes_at_end = if (std.builtin.os.tag == .windows) 0 else 11;
	290	const max_end = self.output_buffer.len - bytes_needed_for_esc_codes_at_end;
	291	if (end.* > max_end) {
	292	const suffix = "... ";
	293	self.columns_written = self.columns_written - (end.* - max_end) + suffix.len;
	294	std.mem.copy(u8, self.output_buffer[max_end..], suffix);
	295	end.* = max_end + suffix.len;
	296	}
	297	}
	298
	299	test "basic functionality" {
	300	var disable = true;
	301	if (disable) {
	302	// This test is disabled because it uses time.sleep() and is therefore slow. It also
	303	// prints bogus progress data to stderr.
	304	return error.SkipZigTest;
	305	}
	306	var progress = Progress{};
	307	const root_node = try progress.start("", 100);
	308	defer root_node.end();
	309
	310	const sub_task_names = [_][]const u8{
	311	"reticulating splines",
	312	"adjusting shoes",
	313	"climbing towers",
	314	"pouring juice",
	315	};
	316	var next_sub_task: usize = 0;
	317
	318	var i: usize = 0;
	319	while (i < 100) : (i += 1) {
	320	var node = root_node.start(sub_task_names[next_sub_task], 5);
	321	node.activate();
	322	next_sub_task = (next_sub_task + 1) % sub_task_names.len;
	323
	324	node.completeOne();
	325	std.time.sleep(5 * std.time.ns_per_ms);
	326	node.completeOne();
	327	node.completeOne();
	328	std.time.sleep(5 * std.time.ns_per_ms);
	329	node.completeOne();
	330	node.completeOne();
	331	std.time.sleep(5 * std.time.ns_per_ms);
	332
	333	node.end();
	334
	335	std.time.sleep(5 * std.time.ns_per_ms);
	336	}
	337	{
	338	var node = root_node.start("this is a really long name designed to activate the truncation code. let's find out if it works", 0);
	339	node.activate();
	340	std.time.sleep(10 * std.time.ns_per_ms);
	341	progress.refresh();
	342	std.time.sleep(10 * std.time.ns_per_ms);
	343	node.end();
	344	}
	345	}

lib/std/auto_reset_event.zig+22-22

...	...	@@ -11,33 +11,33 @@ const assert = std.debug.assert;
11	11	/// Similar to std.ResetEvent but on `set()` it also (atomically) does `reset()`.
12	12	/// Unlike std.ResetEvent, `wait()` can only be called by one thread (MPSC-like).
13	13	pub const AutoResetEvent = struct {
14		// AutoResetEvent has 3 possible states:
15		// - UNSET: the AutoResetEvent is currently unset
16		// - SET: the AutoResetEvent was notified before a wait() was called
17		// - <std.ResetEvent pointer>: there is an active waiter waiting for a notification.
18		//
19		// When attempting to wait:
20		// if the event is unset, it registers a ResetEvent pointer to be notified when the event is set
21		// if the event is already set, then it consumes the notification and resets the event.
22		//
23		// When attempting to notify:
24		// if the event is unset, then we set the event
25		// if theres a waiting ResetEvent, then we unset the event and notify the ResetEvent
26		//
27		// This ensures that the event is automatically reset after a wait() has been issued
28		// and avoids the race condition when using std.ResetEvent in the following scenario:
29		// thread 1 \| thread 2
30		// std.ResetEvent.wait() \|
31		// \| std.ResetEvent.set()
32		// \| std.ResetEvent.set()
33		// std.ResetEvent.reset() \|
34		// std.ResetEvent.wait() \| (missed the second .set() notification above)
	14	/// AutoResetEvent has 3 possible states:
	15	/// - UNSET: the AutoResetEvent is currently unset
	16	/// - SET: the AutoResetEvent was notified before a wait() was called
	17	/// - <std.ResetEvent pointer>: there is an active waiter waiting for a notification.
	18	///
	19	/// When attempting to wait:
	20	/// if the event is unset, it registers a ResetEvent pointer to be notified when the event is set
	21	/// if the event is already set, then it consumes the notification and resets the event.
	22	///
	23	/// When attempting to notify:
	24	/// if the event is unset, then we set the event
	25	/// if theres a waiting ResetEvent, then we unset the event and notify the ResetEvent
	26	///
	27	/// This ensures that the event is automatically reset after a wait() has been issued
	28	/// and avoids the race condition when using std.ResetEvent in the following scenario:
	29	/// thread 1 \| thread 2
	30	/// std.ResetEvent.wait() \|
	31	/// \| std.ResetEvent.set()
	32	/// \| std.ResetEvent.set()
	33	/// std.ResetEvent.reset() \|
	34	/// std.ResetEvent.wait() \| (missed the second .set() notification above)
35	35	state: usize = UNSET,
36	36
37	37	const UNSET = 0;
38	38	const SET = 1;
39	39
40		// the minimum alignment for the `std.ResetEvent` created by wait()
	40	/// the minimum alignment for the `std.ResetEvent` created by wait()
41	41	const event_align = std.math.max(@alignOf(std.ResetEvent), 2);
42	42
43	43	pub fn wait(self: *AutoResetEvent) void {

lib/std/progress.zig deleted-310

...	...	@@ -1,310 +0,0 @@
1		// SPDX-License-Identifier: MIT
2		// Copyright (c) 2015-2020 Zig Contributors
3		// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4		// The MIT license requires this copyright notice to be included in all copies
5		// and substantial portions of the software.
6		const std = @import("std");
7		const windows = std.os.windows;
8		const testing = std.testing;
9		const assert = std.debug.assert;
10
11		/// This API is non-allocating and non-fallible. The tradeoff is that users of
12		/// this API must provide the storage for each `Progress.Node`.
13		/// Initialize the struct directly, overriding these fields as desired:
14		/// * `refresh_rate_ms`
15		/// * `initial_delay_ms`
16		pub const Progress = struct {
17		/// `null` if the current node (and its children) should
18		/// not print on update()
19		terminal: ?std.fs.File = undefined,
20
21		/// Whether the terminal supports ANSI escape codes.
22		supports_ansi_escape_codes: bool = false,
23
24		root: Node = undefined,
25
26		/// Keeps track of how much time has passed since the beginning.
27		/// Used to compare with `initial_delay_ms` and `refresh_rate_ms`.
28		timer: std.time.Timer = undefined,
29
30		/// When the previous refresh was written to the terminal.
31		/// Used to compare with `refresh_rate_ms`.
32		prev_refresh_timestamp: u64 = undefined,
33
34		/// This buffer represents the maximum number of bytes written to the terminal
35		/// with each refresh.
36		output_buffer: [100]u8 = undefined,
37
38		/// How many nanoseconds between writing updates to the terminal.
39		refresh_rate_ns: u64 = 50 * std.time.ns_per_ms,
40
41		/// How many nanoseconds to keep the output hidden
42		initial_delay_ns: u64 = 500 * std.time.ns_per_ms,
43
44		done: bool = true,
45
46		/// Keeps track of how many columns in the terminal have been output, so that
47		/// we can move the cursor back later.
48		columns_written: usize = undefined,
49
50		/// Represents one unit of progress. Each node can have children nodes, or
51		/// one can use integers with `update`.
52		pub const Node = struct {
53		context: *Progress,
54		parent: ?*Node,
55		completed_items: usize,
56		name: []const u8,
57		recently_updated_child: ?*Node = null,
58
59		/// This field may be updated freely.
60		estimated_total_items: ?usize,
61
62		/// Create a new child progress node.
63		/// Call `Node.end` when done.
64		/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
65		/// API to set `self.parent.recently_updated_child` with the return value.
66		/// Until that is fixed you probably want to call `activate` on the return value.
67		pub fn start(self: *Node, name: []const u8, estimated_total_items: ?usize) Node {
68		return Node{
69		.context = self.context,
70		.parent = self,
71		.completed_items = 0,
72		.name = name,
73		.estimated_total_items = estimated_total_items,
74		};
75		}
76
77		/// This is the same as calling `start` and then `end` on the returned `Node`.
78		pub fn completeOne(self: *Node) void {
79		if (self.parent) \|parent\| parent.recently_updated_child = self;
80		self.completed_items += 1;
81		self.context.maybeRefresh();
82		}
83
84		pub fn end(self: *Node) void {
85		self.context.maybeRefresh();
86		if (self.parent) \|parent\| {
87		if (parent.recently_updated_child) \|parent_child\| {
88		if (parent_child == self) {
89		parent.recently_updated_child = null;
90		}
91		}
92		parent.completeOne();
93		} else {
94		self.context.done = true;
95		self.context.refresh();
96		}
97		}
98
99		/// Tell the parent node that this node is actively being worked on.
100		pub fn activate(self: *Node) void {
101		if (self.parent) \|parent\| parent.recently_updated_child = self;
102		}
103		};
104
105		/// Create a new progress node.
106		/// Call `Node.end` when done.
107		/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
108		/// API to return Progress rather than accept it as a parameter.
109		pub fn start(self: Progress, name: []const u8, estimated_total_items: ?usize) !Node {
110		const stderr = std.io.getStdErr();
111		self.terminal = null;
112		if (stderr.supportsAnsiEscapeCodes()) {
113		self.terminal = stderr;
114		self.supports_ansi_escape_codes = true;
115		} else if (std.builtin.os.tag == .windows and stderr.isTty()) {
116		self.terminal = stderr;
117		}
118		self.root = Node{
119		.context = self,
120		.parent = null,
121		.completed_items = 0,
122		.name = name,
123		.estimated_total_items = estimated_total_items,
124		};
125		self.columns_written = 0;
126		self.prev_refresh_timestamp = 0;
127		self.timer = try std.time.Timer.start();
128		self.done = false;
129		return &self.root;
130		}
131
132		/// Updates the terminal if enough time has passed since last update.
133		pub fn maybeRefresh(self: *Progress) void {
134		const now = self.timer.read();
135		if (now < self.initial_delay_ns) return;
136		if (now - self.prev_refresh_timestamp < self.refresh_rate_ns) return;
137		self.refresh();
138		}
139
140		/// Updates the terminal and resets `self.next_refresh_timestamp`.
141		pub fn refresh(self: *Progress) void {
142		const file = self.terminal orelse return;
143
144		const prev_columns_written = self.columns_written;
145		var end: usize = 0;
146		if (self.columns_written > 0) {
147		// restore the cursor position by moving the cursor
148		// `columns_written` cells to the left, then clear the rest of the
149		// line
150		if (self.supports_ansi_escape_codes) {
151		end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[{}D", .{self.columns_written}) catch unreachable).len;
152		end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[0K", .{}) catch unreachable).len;
153		} else if (std.builtin.os.tag == .windows) winapi: {
154		var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;
155		if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.TRUE)
156		unreachable;
157
158		var cursor_pos = windows.COORD{
159		.X = info.dwCursorPosition.X - @intCast(windows.SHORT, self.columns_written),
160		.Y = info.dwCursorPosition.Y,
161		};
162
163		if (cursor_pos.X < 0)
164		cursor_pos.X = 0;
165
166		const fill_chars = @intCast(windows.DWORD, info.dwSize.X - cursor_pos.X);
167
168		var written: windows.DWORD = undefined;
169		if (windows.kernel32.FillConsoleOutputAttribute(
170		file.handle,
171		info.wAttributes,
172		fill_chars,
173		cursor_pos,
174		&written,
175		) != windows.TRUE) {
176		// Stop trying to write to this file.
177		self.terminal = null;
178		break :winapi;
179		}
180		if (windows.kernel32.FillConsoleOutputCharacterA(
181		file.handle,
182		' ',
183		fill_chars,
184		cursor_pos,
185		&written,
186		) != windows.TRUE) unreachable;
187
188		if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE)
189		unreachable;
190		} else unreachable;
191
192		self.columns_written = 0;
193		}
194
195		if (!self.done) {
196		var need_ellipse = false;
197		var maybe_node: ?*Node = &self.root;
198		while (maybe_node) \|node\| {
199		if (need_ellipse) {
200		self.bufWrite(&end, "... ", .{});
201		}
202		need_ellipse = false;
203		if (node.name.len != 0 or node.estimated_total_items != null) {
204		if (node.name.len != 0) {
205		self.bufWrite(&end, "{}", .{node.name});
206		need_ellipse = true;
207		}
208		if (node.estimated_total_items) \|total\| {
209		if (need_ellipse) self.bufWrite(&end, " ", .{});
210		self.bufWrite(&end, "[{}/{}] ", .{ node.completed_items + 1, total });
211		need_ellipse = false;
212		} else if (node.completed_items != 0) {
213		if (need_ellipse) self.bufWrite(&end, " ", .{});
214		self.bufWrite(&end, "[{}] ", .{node.completed_items + 1});
215		need_ellipse = false;
216		}
217		}
218		maybe_node = node.recently_updated_child;
219		}
220		if (need_ellipse) {
221		self.bufWrite(&end, "... ", .{});
222		}
223		}
224
225		_ = file.write(self.output_buffer[0..end]) catch \|e\| {
226		// Stop trying to write to this file once it errors.
227		self.terminal = null;
228		};
229		self.prev_refresh_timestamp = self.timer.read();
230		}
231
232		pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void {
233		const file = self.terminal orelse return;
234		self.refresh();
235		file.outStream().print(format, args) catch {
236		self.terminal = null;
237		return;
238		};
239		self.columns_written = 0;
240		}
241
242		fn bufWrite(self: Progress, end: usize, comptime format: []const u8, args: anytype) void {
243		if (std.fmt.bufPrint(self.output_buffer[end.*..], format, args)) \|written\| {
244		const amt = written.len;
245		end.* += amt;
246		self.columns_written += amt;
247		} else \|err\| switch (err) {
248		error.NoSpaceLeft => {
249		self.columns_written += self.output_buffer.len - end.*;
250		end.* = self.output_buffer.len;
251		},
252		}
253		const bytes_needed_for_esc_codes_at_end = if (std.builtin.os.tag == .windows) 0 else 11;
254		const max_end = self.output_buffer.len - bytes_needed_for_esc_codes_at_end;
255		if (end.* > max_end) {
256		const suffix = "... ";
257		self.columns_written = self.columns_written - (end.* - max_end) + suffix.len;
258		std.mem.copy(u8, self.output_buffer[max_end..], suffix);
259		end.* = max_end + suffix.len;
260		}
261		}
262		};
263
264		test "basic functionality" {
265		var disable = true;
266		if (disable) {
267		// This test is disabled because it uses time.sleep() and is therefore slow. It also
268		// prints bogus progress data to stderr.
269		return error.SkipZigTest;
270		}
271		var progress = Progress{};
272		const root_node = try progress.start("", 100);
273		defer root_node.end();
274
275		const sub_task_names = [_][]const u8{
276		"reticulating splines",
277		"adjusting shoes",
278		"climbing towers",
279		"pouring juice",
280		};
281		var next_sub_task: usize = 0;
282
283		var i: usize = 0;
284		while (i < 100) : (i += 1) {
285		var node = root_node.start(sub_task_names[next_sub_task], 5);
286		node.activate();
287		next_sub_task = (next_sub_task + 1) % sub_task_names.len;
288
289		node.completeOne();
290		std.time.sleep(5 * std.time.ns_per_ms);
291		node.completeOne();
292		node.completeOne();
293		std.time.sleep(5 * std.time.ns_per_ms);
294		node.completeOne();
295		node.completeOne();
296		std.time.sleep(5 * std.time.ns_per_ms);
297
298		node.end();
299
300		std.time.sleep(5 * std.time.ns_per_ms);
301		}
302		{
303		var node = root_node.start("this is a really long name designed to activate the truncation code. let's find out if it works", null);
304		node.activate();
305		std.time.sleep(10 * std.time.ns_per_ms);
306		progress.refresh();
307		std.time.sleep(10 * std.time.ns_per_ms);
308		node.end();
309		}
310		}

lib/std/special/test_runner.zig+1-1

...	...	@@ -36,7 +36,7 @@ pub fn main() anyerror!void {
36	36	}
37	37	std.testing.log_level = .warn;
38	38
39		var test_node = root_node.start(test_fn.name, null);
	39	var test_node = root_node.start(test_fn.name, 0);
40	40	test_node.activate();
41	41	progress.refresh();
42	42	if (progress.terminal == null) {

lib/std/std.zig+1-1

...	...	@@ -29,7 +29,7 @@ pub const PackedIntArrayEndian = @import("packed_int_array.zig").PackedIntArrayE
29	29	pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;
30	30	pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;
31	31	pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
32		pub const Progress = @import("progress.zig").Progress;
	32	pub const Progress = @import("Progress.zig");
33	33	pub const ResetEvent = @import("reset_event.zig").ResetEvent;
34	34	pub const SemanticVersion = @import("SemanticVersion.zig");
35	35	pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;

src/Compilation.zig+85-30

...	...	@@ -26,6 +26,8 @@ const Module = @import("Module.zig");
26	26	const Cache = @import("Cache.zig");
27	27	const stage1 = @import("stage1.zig");
28	28	const translate_c = @import("translate_c.zig");
	29	const ThreadPool = @import("ThreadPool.zig");
	30	const WaitGroup = @import("WaitGroup.zig");
29	31
30	32	/// General-purpose allocator. Used for both temporary and long-term storage.
31	33	gpa: *Allocator,
...	...	@@ -41,7 +43,12 @@ link_error_flags: link.File.ErrorFlags = .{},
41	43
42	44	work_queue: std.fifo.LinearFifo(Job, .Dynamic),
43	45
	46	/// These jobs are to invoke the Clang compiler to create an object file, which
	47	/// gets linked with the Compilation.
	48	c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic),
	49
44	50	/// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
	51	/// This data is accessed by multiple threads and is protected by `mutex`.
45	52	failed_c_objects: std.AutoArrayHashMapUnmanaged(CObject, ErrorMsg) = .{},
46	53
47	54	keep_source_files_loaded: bool,
...	...	@@ -74,6 +81,7 @@ zig_lib_directory: Directory,
74	81	local_cache_directory: Directory,
75	82	global_cache_directory: Directory,
76	83	libc_include_dir_list: []const []const u8,
	84	thread_pool: *ThreadPool,
77	85
78	86	/// Populated when we build the libc++ static library. A Job to build this is placed in the queue
79	87	/// and resolved before calling linker.flush().
...	...	@@ -111,6 +119,9 @@ owned_link_dir: ?std.fs.Dir,
111	119	/// Don't use this for anything other than stage1 compatibility.
112	120	color: @import("main.zig").Color = .auto,
113	121
	122	/// This mutex guards all `Compilation` mutable state.
	123	mutex: std.Mutex = .{},
	124
114	125	test_filter: ?[]const u8,
115	126	test_name_prefix: ?[]const u8,
116	127	test_evented_io: bool,
...	...	@@ -150,9 +161,6 @@ const Job = union(enum) {
150	161	/// The source file containing the Decl has been updated, and so the
151	162	/// Decl may need its line number information updated in the debug info.
152	163	update_line_number: *Module.Decl,
153		/// Invoke the Clang compiler to create an object file, which gets linked
154		/// with the Compilation.
155		c_object: *CObject,
156	164
157	165	/// one of the glibc static objects
158	166	glibc_crt_file: glibc.CRTFile,
...	...	@@ -330,6 +338,7 @@ pub const InitOptions = struct {
330	338	root_name: []const u8,
331	339	root_pkg: ?*Package,
332	340	output_mode: std.builtin.OutputMode,
	341	thread_pool: *ThreadPool,
333	342	dynamic_linker: ?[]const u8 = null,
334	343	/// `null` means to not emit a binary file.
335	344	emit_bin: ?EmitLoc,
...	...	@@ -971,6 +980,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
971	980	.emit_analysis = options.emit_analysis,
972	981	.emit_docs = options.emit_docs,
973	982	.work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa),
	983	.c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),
974	984	.keep_source_files_loaded = options.keep_source_files_loaded,
975	985	.use_clang = use_clang,
976	986	.clang_argv = options.clang_argv,
...	...	@@ -979,6 +989,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
979	989	.self_exe_path = options.self_exe_path,
980	990	.libc_include_dir_list = libc_dirs.libc_include_dir_list,
981	991	.sanitize_c = sanitize_c,
	992	.thread_pool = options.thread_pool,
982	993	.clang_passthrough_mode = options.clang_passthrough_mode,
983	994	.clang_preprocessor_mode = options.clang_preprocessor_mode,
984	995	.verbose_cc = options.verbose_cc,
...	...	@@ -1190,11 +1201,13 @@ pub fn update(self: *Compilation) !void {
1190	1201	const tracy = trace(@src());
1191	1202	defer tracy.end();
1192	1203
	1204	self.c_object_cache_digest_set.clearRetainingCapacity();
	1205
1193	1206	// For compiling C objects, we rely on the cache hash system to avoid duplicating work.
1194	1207	// Add a Job for each C object.
1195		try self.work_queue.ensureUnusedCapacity(self.c_object_table.items().len);
	1208	try self.c_object_work_queue.ensureUnusedCapacity(self.c_object_table.items().len);
1196	1209	for (self.c_object_table.items()) \|entry\| {
1197		self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key });
	1210	self.c_object_work_queue.writeItemAssumeCapacity(entry.key);
1198	1211	}
1199	1212
1200	1213	const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_llvm;
...	...	@@ -1365,13 +1378,23 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
1365	1378
1366	1379	pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemory }!void {
1367	1380	var progress: std.Progress = .{};
1368		var main_progress_node = try progress.start("", null);
	1381	var main_progress_node = try progress.start("", 0);
1369	1382	defer main_progress_node.end();
1370	1383	if (self.color == .off) progress.terminal = null;
1371	1384
1372	1385	var c_comp_progress_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
1373	1386	defer c_comp_progress_node.end();
1374	1387
	1388	var wg = WaitGroup{};
	1389	defer wg.wait();
	1390
	1391	while (self.c_object_work_queue.readItem()) \|c_object\| {
	1392	wg.start();
	1393	try self.thread_pool.spawn(workerUpdateCObject, .{
	1394	self, c_object, &c_comp_progress_node, &wg,
	1395	});
	1396	}
	1397
1375	1398	while (self.work_queue.readItem()) \|work_item\| switch (work_item) {
1376	1399	.codegen_decl => \|decl\| switch (decl.analysis) {
1377	1400	.unreferenced => unreachable,
...	...	@@ -1447,21 +1470,6 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
1447	1470	decl.analysis = .codegen_failure_retryable;
1448	1471	};
1449	1472	},
1450		.c_object => \|c_object\| {
1451		self.updateCObject(c_object, &c_comp_progress_node) catch \|err\| switch (err) {
1452		error.AnalysisFail => continue,
1453		else => {
1454		try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1);
1455		self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create(
1456		self.gpa,
1457		0,
1458		"unable to build C object: {s}",
1459		.{@errorName(err)},
1460		));
1461		c_object.status = .{ .failure = {} };
1462		},
1463		};
1464		},
1465	1473	.glibc_crt_file => \|crt_file\| {
1466	1474	glibc.buildCRTFile(self, crt_file) catch \|err\| {
1467	1475	// TODO Expose this as a normal compile error rather than crashing here.
...	...	@@ -1553,7 +1561,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
1553	1561	};
1554	1562	}
1555	1563
1556		pub fn obtainCObjectCacheManifest(comp: *Compilation) Cache.Manifest {
	1564	pub fn obtainCObjectCacheManifest(comp: *const Compilation) Cache.Manifest {
1557	1565	var man = comp.cache_parent.obtain();
1558	1566
1559	1567	// Only things that need to be added on top of the base hash, and only things
...	...	@@ -1708,6 +1716,37 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
1708	1716	};
1709	1717	}
1710	1718
	1719	fn workerUpdateCObject(
	1720	comp: *Compilation,
	1721	c_object: *CObject,
	1722	progress_node: *std.Progress.Node,
	1723	wg: *WaitGroup,
	1724	) void {
	1725	defer wg.stop();
	1726
	1727	comp.updateCObject(c_object, progress_node) catch \|err\| switch (err) {
	1728	error.AnalysisFail => return,
	1729	else => {
	1730	{
	1731	const lock = comp.mutex.acquire();
	1732	defer lock.release();
	1733	comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1) catch {
	1734	fatal("TODO handle this by setting c_object.status = oom failure", .{});
	1735	};
	1736	comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, ErrorMsg.create(
	1737	comp.gpa,
	1738	0,
	1739	"unable to build C object: {s}",
	1740	.{@errorName(err)},
	1741	) catch {
	1742	fatal("TODO handle this by setting c_object.status = oom failure", .{});
	1743	});
	1744	}
	1745	c_object.status = .{ .failure = {} };
	1746	},
	1747	};
	1748	}
	1749
1711	1750	fn updateCObject(comp: Compilation, c_object: CObject, c_comp_progress_node: *std.Progress.Node) !void {
1712	1751	if (!build_options.have_llvm) {
1713	1752	return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});
...	...	@@ -1720,6 +1759,8 @@ fn updateCObject(comp: Compilation, c_object: CObject, c_comp_progress_node: *
1720	1759
1721	1760	if (c_object.clearStatus(comp.gpa)) {
1722	1761	// There was previous failure.
	1762	const lock = comp.mutex.acquire();
	1763	defer lock.release();
1723	1764	comp.failed_c_objects.removeAssertDiscard(c_object);
1724	1765	}
1725	1766
...	...	@@ -1747,8 +1788,16 @@ fn updateCObject(comp: Compilation, c_object: CObject, c_comp_progress_node: *
1747	1788	}
1748	1789
1749	1790	{
1750		const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, man.hash.peekBin());
1751		if (gop.found_existing) {
	1791	const is_collision = blk: {
	1792	const bin_digest = man.hash.peekBin();
	1793
	1794	const lock = comp.mutex.acquire();
	1795	defer lock.release();
	1796
	1797	const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest);
	1798	break :blk gop.found_existing;
	1799	};
	1800	if (is_collision) {
1752	1801	return comp.failCObj(
1753	1802	c_object,
1754	1803	"the same source file was already added to the same compilation with the same flags",
...	...	@@ -1764,7 +1813,7 @@ fn updateCObject(comp: Compilation, c_object: CObject, c_comp_progress_node: *
1764	1813	const c_source_basename = std.fs.path.basename(c_object.src.src_path);
1765	1814
1766	1815	c_comp_progress_node.activate();
1767		var child_progress_node = c_comp_progress_node.start(c_source_basename, null);
	1816	var child_progress_node = c_comp_progress_node.start(c_source_basename, 0);
1768	1817	child_progress_node.activate();
1769	1818	defer child_progress_node.end();
1770	1819
...	...	@@ -1929,7 +1978,7 @@ pub fn addTranslateCCArgs(
1929	1978
1930	1979	/// Add common C compiler args between translate-c and C object compilation.
1931	1980	pub fn addCCArgs(
1932		comp: *Compilation,
	1981	comp: *const Compilation,
1933	1982	arena: *Allocator,
1934	1983	argv: *std.ArrayList([]const u8),
1935	1984	ext: FileExt,
...	...	@@ -2164,10 +2213,14 @@ fn failCObj(comp: Compilation, c_object: CObject, comptime format: []const u8,
2164	2213
2165	2214	fn failCObjWithOwnedErrorMsg(comp: Compilation, c_object: CObject, err_msg: *ErrorMsg) InnerError {
2166	2215	{
2167		errdefer err_msg.destroy(comp.gpa);
2168		try comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1);
	2216	const lock = comp.mutex.acquire();
	2217	defer lock.release();
	2218	{
	2219	errdefer err_msg.destroy(comp.gpa);
	2220	try comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1);
	2221	}
	2222	comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg);
2169	2223	}
2170		comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg);
2171	2224	c_object.status = .failure;
2172	2225	return error.AnalysisFail;
2173	2226	}
...	...	@@ -2324,7 +2377,7 @@ test "classifyFileExt" {
2324	2377	std.testing.expectEqual(FileExt.zir, classifyFileExt("foo.zir"));
2325	2378	}
2326	2379
2327		fn haveFramePointer(comp: *Compilation) bool {
	2380	fn haveFramePointer(comp: *const Compilation) bool {
2328	2381	// If you complicate this logic make sure you update the parent cache hash.
2329	2382	// Right now it's not in the cache hash because the value depends on optimize_mode
2330	2383	// and strip which are both already part of the hash.
...	...	@@ -2775,6 +2828,7 @@ fn buildOutputFromZig(
2775	2828	.root_name = root_name,
2776	2829	.root_pkg = &root_pkg,
2777	2830	.output_mode = fixed_output_mode,
	2831	.thread_pool = comp.thread_pool,
2778	2832	.libc_installation = comp.bin_file.options.libc_installation,
2779	2833	.emit_bin = emit_bin,
2780	2834	.optimize_mode = optimize_mode,
...	...	@@ -3148,6 +3202,7 @@ pub fn build_crt_file(
3148	3202	.root_name = root_name,
3149	3203	.root_pkg = null,
3150	3204	.output_mode = output_mode,
	3205	.thread_pool = comp.thread_pool,
3151	3206	.libc_installation = comp.bin_file.options.libc_installation,
3152	3207	.emit_bin = emit_bin,
3153	3208	.optimize_mode = comp.bin_file.options.optimize_mode,

src/Event.zig created+43

...	...	@@ -0,0 +1,43 @@
	1	// SPDX-License-Identifier: MIT
	2	// Copyright (c) 2015-2020 Zig Contributors
	3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
	4	// The MIT license requires this copyright notice to be included in all copies
	5	// and substantial portions of the software.
	6	const std = @import("std");
	7	const Event = @This();
	8
	9	lock: std.Mutex = .{},
	10	event: std.ResetEvent = undefined,
	11	state: enum { empty, waiting, notified } = .empty,
	12
	13	pub fn wait(self: *Event) void {
	14	const held = self.lock.acquire();
	15
	16	switch (self.state) {
	17	.empty => {
	18	self.state = .waiting;
	19	self.event = @TypeOf(self.event).init();
	20	held.release();
	21	self.event.wait();
	22	self.event.deinit();
	23	},
	24	.waiting => unreachable,
	25	.notified => held.release(),
	26	}
	27	}
	28
	29	pub fn set(self: *Event) void {
	30	const held = self.lock.acquire();
	31
	32	switch (self.state) {
	33	.empty => {
	34	self.state = .notified;
	35	held.release();
	36	},
	37	.waiting => {
	38	held.release();
	39	self.event.set();
	40	},
	41	.notified => unreachable,
	42	}
	43	}

src/ThreadPool.zig created+126

...	...	@@ -0,0 +1,126 @@
	1	// SPDX-License-Identifier: MIT
	2	// Copyright (c) 2015-2020 Zig Contributors
	3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
	4	// The MIT license requires this copyright notice to be included in all copies
	5	// and substantial portions of the software.
	6	const std = @import("std");
	7	const ThreadPool = @This();
	8
	9	lock: std.Mutex = .{},
	10	is_running: bool = true,
	11	allocator: *std.mem.Allocator,
	12	running: usize = 0,
	13	threads: []*std.Thread,
	14	run_queue: RunQueue = .{},
	15	idle_queue: IdleQueue = .{},
	16
	17	const IdleQueue = std.SinglyLinkedList(std.AutoResetEvent);
	18	const RunQueue = std.SinglyLinkedList(Runnable);
	19	const Runnable = struct {
	20	runFn: fn (*Runnable) void,
	21	};
	22
	23	pub fn init(self: ThreadPool, allocator: std.mem.Allocator) !void {
	24	self.* = .{
	25	.allocator = allocator,
	26	.threads = &[_]*std.Thread{},
	27	};
	28	if (std.builtin.single_threaded)
	29	return;
	30
	31	errdefer self.deinit();
	32
	33	var num_threads = std.Thread.cpuCount() catch 1;
	34	if (num_threads > 0)
	35	self.threads = try allocator.alloc(*std.Thread, num_threads);
	36
	37	while (num_threads > 0) : (num_threads -= 1) {
	38	const thread = try std.Thread.spawn(self, runWorker);
	39	self.threads[self.running] = thread;
	40	self.running += 1;
	41	}
	42	}
	43
	44	pub fn deinit(self: *ThreadPool) void {
	45	self.shutdown();
	46
	47	std.debug.assert(!self.is_running);
	48	for (self.threads[0..self.running]) \|thread\|
	49	thread.wait();
	50
	51	defer self.threads = &[_]*std.Thread{};
	52	if (self.running > 0)
	53	self.allocator.free(self.threads);
	54	}
	55
	56	pub fn shutdown(self: *ThreadPool) void {
	57	const held = self.lock.acquire();
	58
	59	if (!self.is_running)
	60	return held.release();
	61
	62	var idle_queue = self.idle_queue;
	63	self.idle_queue = .{};
	64	self.is_running = false;
	65	held.release();
	66
	67	while (idle_queue.popFirst()) \|idle_node\|
	68	idle_node.data.set();
	69	}
	70
	71	pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
	72	if (std.builtin.single_threaded) {
	73	@call(.{}, func, args);
	74	return;
	75	}
	76	const Args = @TypeOf(args);
	77	const Closure = struct {
	78	arguments: Args,
	79	pool: *ThreadPool,
	80	run_node: RunQueue.Node = .{ .data = .{ .runFn = runFn } },
	81
	82	fn runFn(runnable: *Runnable) void {
	83	const run_node = @fieldParentPtr(RunQueue.Node, "data", runnable);
	84	const closure = @fieldParentPtr(@This(), "run_node", run_node);
	85	const result = @call(.{}, func, closure.arguments);
	86	closure.pool.allocator.destroy(closure);
	87	}
	88	};
	89
	90	const closure = try self.allocator.create(Closure);
	91	closure.* = .{
	92	.arguments = args,
	93	.pool = self,
	94	};
	95
	96	const held = self.lock.acquire();
	97	self.run_queue.prepend(&closure.run_node);
	98
	99	const idle_node = self.idle_queue.popFirst();
	100	held.release();
	101
	102	if (idle_node) \|node\|
	103	node.data.set();
	104	}
	105
	106	fn runWorker(self: *ThreadPool) void {
	107	while (true) {
	108	const held = self.lock.acquire();
	109
	110	if (self.run_queue.popFirst()) \|run_node\| {
	111	held.release();
	112	(run_node.data.runFn)(&run_node.data);
	113	continue;
	114	}
	115
	116	if (!self.is_running) {
	117	held.release();
	118	return;
	119	}
	120
	121	var idle_node = IdleQueue.Node{ .data = .{} };
	122	self.idle_queue.prepend(&idle_node);
	123	held.release();
	124	idle_node.data.wait();
	125	}
	126	}

src/WaitGroup.zig created+46

...	...	@@ -0,0 +1,46 @@
	1	// SPDX-License-Identifier: MIT
	2	// Copyright (c) 2015-2020 Zig Contributors
	3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
	4	// The MIT license requires this copyright notice to be included in all copies
	5	// and substantial portions of the software.
	6	const std = @import("std");
	7	const WaitGroup = @This();
	8	const Event = @import("Event.zig");
	9
	10	lock: std.Mutex = .{},
	11	counter: usize = 0,
	12	event: ?*Event = null,
	13
	14	pub fn start(self: *WaitGroup) void {
	15	const held = self.lock.acquire();
	16	defer held.release();
	17
	18	self.counter += 1;
	19	}
	20
	21	pub fn stop(self: *WaitGroup) void {
	22	var event: ?*Event = null;
	23	defer if (event) \|waiter\|
	24	waiter.set();
	25
	26	const held = self.lock.acquire();
	27	defer held.release();
	28
	29	self.counter -= 1;
	30	if (self.counter == 0)
	31	std.mem.swap(?*Event, &self.event, &event);
	32	}
	33
	34	pub fn wait(self: *WaitGroup) void {
	35	var event = Event{};
	36	var has_event = false;
	37	defer if (has_event)
	38	event.wait();
	39
	40	const held = self.lock.acquire();
	41	defer held.release();
	42
	43	has_event = self.counter != 0;
	44	if (has_event)
	45	self.event = &event;
	46	}

src/glibc.zig+1

...	...	@@ -936,6 +936,7 @@ fn buildSharedLib(
936	936	.root_pkg = null,
937	937	.output_mode = .Lib,
938	938	.link_mode = .Dynamic,
	939	.thread_pool = comp.thread_pool,
939	940	.libc_installation = comp.bin_file.options.libc_installation,
940	941	.emit_bin = emit_bin,
941	942	.optimize_mode = comp.bin_file.options.optimize_mode,

src/libcxx.zig+2

...	...	@@ -162,6 +162,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
162	162	.root_name = root_name,
163	163	.root_pkg = null,
164	164	.output_mode = output_mode,
	165	.thread_pool = comp.thread_pool,
165	166	.libc_installation = comp.bin_file.options.libc_installation,
166	167	.emit_bin = emit_bin,
167	168	.optimize_mode = comp.bin_file.options.optimize_mode,
...	...	@@ -280,6 +281,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
280	281	.root_name = root_name,
281	282	.root_pkg = null,
282	283	.output_mode = output_mode,
	284	.thread_pool = comp.thread_pool,
283	285	.libc_installation = comp.bin_file.options.libc_installation,
284	286	.emit_bin = emit_bin,
285	287	.optimize_mode = comp.bin_file.options.optimize_mode,

src/libunwind.zig+1

...	...	@@ -95,6 +95,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
95	95	.root_name = root_name,
96	96	.root_pkg = null,
97	97	.output_mode = output_mode,
	98	.thread_pool = comp.thread_pool,
98	99	.libc_installation = comp.bin_file.options.libc_installation,
99	100	.emit_bin = emit_bin,
100	101	.optimize_mode = comp.bin_file.options.optimize_mode,

src/main.zig+10

...	...	@@ -19,6 +19,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
19	19	const translate_c = @import("translate_c.zig");
20	20	const Cache = @import("Cache.zig");
21	21	const target_util = @import("target.zig");
	22	const ThreadPool = @import("ThreadPool.zig");
22	23
23	24	pub fn fatal(comptime format: []const u8, args: anytype) noreturn {
24	25	std.log.emerg(format, args);
...	...	@@ -1632,6 +1633,10 @@ fn buildOutputType(
1632	1633	};
1633	1634	defer zig_lib_directory.handle.close();
1634	1635
	1636	var thread_pool: ThreadPool = undefined;
	1637	try thread_pool.init(gpa);
	1638	defer thread_pool.deinit();
	1639
1635	1640	var libc_installation: ?LibCInstallation = null;
1636	1641	defer if (libc_installation) \|*l\| l.deinit(gpa);
1637	1642
...	...	@@ -1747,6 +1752,7 @@ fn buildOutputType(
1747	1752	.single_threaded = single_threaded,
1748	1753	.function_sections = function_sections,
1749	1754	.self_exe_path = self_exe_path,
	1755	.thread_pool = &thread_pool,
1750	1756	.clang_passthrough_mode = arg_mode != .build,
1751	1757	.clang_preprocessor_mode = clang_preprocessor_mode,
1752	1758	.version = optional_version,
...	...	@@ -2412,6 +2418,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !v
2412	2418	.directory = null, // Use the local zig-cache.
2413	2419	.basename = exe_basename,
2414	2420	};
	2421	var thread_pool: ThreadPool = undefined;
	2422	try thread_pool.init(gpa);
	2423	defer thread_pool.deinit();
2415	2424	const comp = Compilation.create(gpa, .{
2416	2425	.zig_lib_directory = zig_lib_directory,
2417	2426	.local_cache_directory = local_cache_directory,
...	...	@@ -2427,6 +2436,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !v
2427	2436	.emit_h = null,
2428	2437	.optimize_mode = .Debug,
2429	2438	.self_exe_path = self_exe_path,
	2439	.thread_pool = &thread_pool,
2430	2440	}) catch \|err\| {
2431	2441	fatal("unable to create compilation: {}", .{@errorName(err)});
2432	2442	};

src/musl.zig+1

...	...	@@ -200,6 +200,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
200	200	.root_pkg = null,
201	201	.output_mode = .Lib,
202	202	.link_mode = .Dynamic,
	203	.thread_pool = comp.thread_pool,
203	204	.libc_installation = comp.bin_file.options.libc_installation,
204	205	.emit_bin = Compilation.EmitLoc{ .directory = null, .basename = "libc.so" },
205	206	.optimize_mode = comp.bin_file.options.optimize_mode,

src/stage1.zig+4-4

...	...	@@ -293,7 +293,7 @@ export fn stage2_progress_start_root(
293	293	) *std.Progress.Node {
294	294	return progress.start(
295	295	name_ptr[0..name_len],
296		if (estimated_total_items == 0) null else estimated_total_items,
	296	estimated_total_items,
297	297	) catch @panic("timer unsupported");
298	298	}
299	299
...	...	@@ -312,7 +312,7 @@ export fn stage2_progress_start(
312	312	const child_node = std.heap.c_allocator.create(std.Progress.Node) catch @panic("out of memory");
313	313	child_node.* = node.start(
314	314	name_ptr[0..name_len],
315		if (estimated_total_items == 0) null else estimated_total_items,
	315	estimated_total_items,
316	316	);
317	317	child_node.activate();
318	318	return child_node;
...	...	@@ -333,8 +333,8 @@ export fn stage2_progress_complete_one(node: *std.Progress.Node) void {
333	333
334	334	// ABI warning
335	335	export fn stage2_progress_update_node(node: *std.Progress.Node, done_count: usize, total_count: usize) void {
336		node.completed_items = done_count;
337		node.estimated_total_items = total_count;
	336	node.setCompletedItems(done_count);
	337	node.setEstimatedTotalItems(total_count);
338	338	node.activate();
339	339	node.context.maybeRefresh();
340	340	}

src/stage1/zig0.cpp+4

...	...	@@ -266,6 +266,7 @@ int main(int argc, char **argv) {
266	266	TargetSubsystem subsystem = TargetSubsystemAuto;
267	267	const char *override_lib_dir = nullptr;
268	268	const char *mcpu = nullptr;
	269	bool single_threaded = false;
269	270
270	271	for (int i = 1; i < argc; i += 1) {
271	272	char *arg = argv[i];
...	...	@@ -281,6 +282,8 @@ int main(int argc, char **argv) {
281	282	optimize_mode = BuildModeSafeRelease;
282	283	} else if (strcmp(arg, "-OReleaseSmall") == 0) {
283	284	optimize_mode = BuildModeSmallRelease;
	285	} else if (strcmp(arg, "--single-threaded") == 0) {
	286	single_threaded = true;
284	287	} else if (strcmp(arg, "--help") == 0) {
285	288	return print_full_usage(arg0, stdout, EXIT_SUCCESS);
286	289	} else if (strcmp(arg, "--strip") == 0) {
...	...	@@ -469,6 +472,7 @@ int main(int argc, char **argv) {
469	472	stage1->link_libcpp = link_libcpp;
470	473	stage1->subsystem = subsystem;
471	474	stage1->pic = true;
	475	stage1->is_single_threaded = single_threaded;
472	476
473	477	zig_stage1_build_object(stage1);
474	478

src/test.zig+26-13

...	...	@@ -10,6 +10,7 @@ const enable_qemu: bool = build_options.enable_qemu;
10	10	const enable_wine: bool = build_options.enable_wine;
11	11	const enable_wasmtime: bool = build_options.enable_wasmtime;
12	12	const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir;
	13	const ThreadPool = @import("ThreadPool.zig");
13	14
14	15	const cheader = @embedFile("link/cbe.h");
15	16
...	...	@@ -467,6 +468,10 @@ pub const TestContext = struct {
467	468	defer zig_lib_directory.handle.close();
468	469	defer std.testing.allocator.free(zig_lib_directory.path.?);
469	470
	471	var thread_pool: ThreadPool = undefined;
	472	try thread_pool.init(std.testing.allocator);
	473	defer thread_pool.deinit();
	474
470	475	for (self.cases.items) \|case\| {
471	476	if (build_options.skip_non_native and case.target.getCpuArch() != std.Target.current.cpu.arch)
472	477	continue;
...	...	@@ -480,7 +485,13 @@ pub const TestContext = struct {
480	485	progress.initial_delay_ns = 0;
481	486	progress.refresh_rate_ns = 0;
482	487
483		try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_directory);
	488	try self.runOneCase(
	489	std.testing.allocator,
	490	&prg_node,
	491	case,
	492	zig_lib_directory,
	493	&thread_pool,
	494	);
484	495	}
485	496	}
486	497
...	...	@@ -490,6 +501,7 @@ pub const TestContext = struct {
490	501	root_node: *std.Progress.Node,
491	502	case: Case,
492	503	zig_lib_directory: Compilation.Directory,
	504	thread_pool: *ThreadPool,
493	505	) !void {
494	506	const target_info = try std.zig.system.NativeTargetInfo.detect(allocator, case.target);
495	507	const target = target_info.target;
...	...	@@ -539,6 +551,7 @@ pub const TestContext = struct {
539	551	.local_cache_directory = zig_cache_directory,
540	552	.global_cache_directory = zig_cache_directory,
541	553	.zig_lib_directory = zig_lib_directory,
	554	.thread_pool = thread_pool,
542	555	.root_name = "test_case",
543	556	.target = target,
544	557	// TODO: support tests for object file building, and library builds
...	...	@@ -565,12 +578,12 @@ pub const TestContext = struct {
565	578	update_node.activate();
566	579	defer update_node.end();
567	580
568		var sync_node = update_node.start("write", null);
	581	var sync_node = update_node.start("write", 0);
569	582	sync_node.activate();
570	583	try tmp.dir.writeFile(tmp_src_path, update.src);
571	584	sync_node.end();
572	585
573		var module_node = update_node.start("parse/analysis/codegen", null);
	586	var module_node = update_node.start("parse/analysis/codegen", 0);
574	587	module_node.activate();
575	588	try comp.makeBinFileWritable();
576	589	try comp.update();
...	...	@@ -622,21 +635,21 @@ pub const TestContext = struct {
622	635	}
623	636	}
624	637	} else {
625		update_node.estimated_total_items = 5;
626		var emit_node = update_node.start("emit", null);
	638	update_node.setEstimatedTotalItems(5);
	639	var emit_node = update_node.start("emit", 0);
627	640	emit_node.activate();
628	641	var new_zir_module = try zir.emit(allocator, comp.bin_file.options.module.?);
629	642	defer new_zir_module.deinit(allocator);
630	643	emit_node.end();
631	644
632		var write_node = update_node.start("write", null);
	645	var write_node = update_node.start("write", 0);
633	646	write_node.activate();
634	647	var out_zir = std.ArrayList(u8).init(allocator);
635	648	defer out_zir.deinit();
636	649	try new_zir_module.writeToStream(allocator, out_zir.outStream());
637	650	write_node.end();
638	651
639		var test_node = update_node.start("assert", null);
	652	var test_node = update_node.start("assert", 0);
640	653	test_node.activate();
641	654	defer test_node.end();
642	655
...	...	@@ -653,7 +666,7 @@ pub const TestContext = struct {
653	666	}
654	667	},
655	668	.Error => \|e\| {
656		var test_node = update_node.start("assert", null);
	669	var test_node = update_node.start("assert", 0);
657	670	test_node.activate();
658	671	defer test_node.end();
659	672	var handled_errors = try arena.alloc(bool, e.len);
...	...	@@ -710,9 +723,9 @@ pub const TestContext = struct {
710	723	.Execution => \|expected_stdout\| {
711	724	std.debug.assert(!case.cbe);
712	725
713		update_node.estimated_total_items = 4;
	726	update_node.setEstimatedTotalItems(4);
714	727	var exec_result = x: {
715		var exec_node = update_node.start("execute", null);
	728	var exec_node = update_node.start("execute", 0);
716	729	exec_node.activate();
717	730	defer exec_node.end();
718	731
...	...	@@ -775,7 +788,7 @@ pub const TestContext = struct {
775	788	.cwd_dir = tmp.dir,
776	789	});
777	790	};
778		var test_node = update_node.start("test", null);
	791	var test_node = update_node.start("test", 0);
779	792	test_node.activate();
780	793	defer test_node.end();
781	794	defer allocator.free(exec_result.stdout);
...	...	@@ -854,7 +867,7 @@ pub const TestContext = struct {
854	867	};
855	868
856	869	{
857		var load_node = update_node.start("load", null);
	870	var load_node = update_node.start("load", 0);
858	871	load_node.activate();
859	872	defer load_node.end();
860	873
...	...	@@ -892,7 +905,7 @@ pub const TestContext = struct {
892	905	}
893	906	}
894	907
895		var exec_node = update_node.start("execute", null);
	908	var exec_node = update_node.start("execute", 0);
896	909	exec_node.activate();
897	910	defer exec_node.end();
898	911