Commit 4918605176c2e48c178847ea281b790334207733 | zig

CMakeLists.txt+11-1

 @@ -75,6 +75,7 @@ set(ZIG_TARGET_TRIPLE "native" CACHE STRING "arch-os-abi to output binaries for"
 set(ZIG_TARGET_MCPU "baseline" CACHE STRING "-mcpu parameter to output binaries for")
 set(ZIG_EXECUTABLE "" CACHE STRING "(when cross compiling) path to already-built zig binary")
 set(ZIG_PREFER_LLVM_CONFIG off CACHE BOOL "(when cross compiling) use llvm-config to find target llvm dependencies if needed")
+set(ZIG_SINGLE_THREADED off CACHE BOOL "limit the zig compiler to use only 1 thread")
 find_package(llvm)
 find_package(clang)
 @@ -410,7 +411,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/os/windows/win32error.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/pdb.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/process.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/progress.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Progress.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/rand.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/reset_event.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/sort.zig"
 @@ -510,10 +511,13 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/Cache.zig"
     "${CMAKE_SOURCE_DIR}/src/Compilation.zig"
     "${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig"
+    "${CMAKE_SOURCE_DIR}/src/Event.zig"
     "${CMAKE_SOURCE_DIR}/src/Module.zig"
     "${CMAKE_SOURCE_DIR}/src/Package.zig"
     "${CMAKE_SOURCE_DIR}/src/RangeSet.zig"
+    "${CMAKE_SOURCE_DIR}/src/ThreadPool.zig"
     "${CMAKE_SOURCE_DIR}/src/TypedValue.zig"
+    "${CMAKE_SOURCE_DIR}/src/WaitGroup.zig"
     "${CMAKE_SOURCE_DIR}/src/astgen.zig"
     "${CMAKE_SOURCE_DIR}/src/clang.zig"
     "${CMAKE_SOURCE_DIR}/src/clang_options.zig"
 @@ -713,6 +717,11 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
 else()
     set(ZIG1_RELEASE_ARG -OReleaseFast --strip)
 endif()
+if(ZIG_SINGLE_THREADED)
+  set(ZIG1_SINGLE_THREADED_ARG "--single-threaded")
+else()
+  set(ZIG1_SINGLE_THREADED_ARG "")
+endif()
 set(BUILD_ZIG1_ARGS
     "src/stage1.zig"
 @@ -722,6 +731,7 @@ set(BUILD_ZIG1_ARGS
     --override-lib-dir "${CMAKE_SOURCE_DIR}/lib"
     "-femit-bin=${ZIG1_OBJECT}"
     "${ZIG1_RELEASE_ARG}"
+    "${ZIG1_SINGLE_THREADED_ARG}"
     -lc
     --pkg-begin build_options "${ZIG_CONFIG_ZIG_OUT}"
     --pkg-end

ci/drone/linux_script+2-1

...	@@ -17,7 +17,8 @@ git config core.abbrev 9	...	@@ -17,7 +17,8 @@ git config core.abbrev 9
17		17
18	mkdir build	18	mkdir build
19	cd build	19	cd build
20	cmake .. -DCMAKE_BUILD_TYPE=Release "-DCMAKE_INSTALL_PREFIX=$DISTDIR" -DZIG_STATIC=ON -DCMAKE_PREFIX_PATH=/deps/local -GNinja	20	# TODO figure out why Drone CI is deadlocking and stop passing -DZIG_SINGLE_THREADED=ON
		21	cmake .. -DCMAKE_BUILD_TYPE=Release "-DCMAKE_INSTALL_PREFIX=$DISTDIR" -DZIG_STATIC=ON -DCMAKE_PREFIX_PATH=/deps/local -GNinja -DZIG_SINGLE_THREADED=ON
21		22
22	samu install	23	samu install
23	./zig build test -Dskip-release -Dskip-non-native	24	./zig build test -Dskip-release -Dskip-non-native

lib/std/Progress.zig created+345

...	@@ -0,0 +1,345 @@
1	// SPDX-License-Identifier: MIT
2	// Copyright (c) 2015-2020 Zig Contributors
3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4	// The MIT license requires this copyright notice to be included in all copies
5	// and substantial portions of the software.
6
7	//! This API non-allocating, non-fallible, and thread-safe.
8	//! The tradeoff is that users of this API must provide the storage
9	//! for each `Progress.Node`.
10	//!
11	//! Initialize the struct directly, overriding these fields as desired:
12	//! * `refresh_rate_ms`
13	//! * `initial_delay_ms`
14
15	const std = @import("std");
16	const windows = std.os.windows;
17	const testing = std.testing;
18	const assert = std.debug.assert;
19	const Progress = @This();
20
21	/// `null` if the current node (and its children) should
22	/// not print on update()
23	terminal: ?std.fs.File = undefined,
24
25	/// Whether the terminal supports ANSI escape codes.
26	supports_ansi_escape_codes: bool = false,
27
28	root: Node = undefined,
29
30	/// Keeps track of how much time has passed since the beginning.
31	/// Used to compare with `initial_delay_ms` and `refresh_rate_ms`.
32	timer: std.time.Timer = undefined,
33
34	/// When the previous refresh was written to the terminal.
35	/// Used to compare with `refresh_rate_ms`.
36	prev_refresh_timestamp: u64 = undefined,
37
38	/// This buffer represents the maximum number of bytes written to the terminal
39	/// with each refresh.
40	output_buffer: [100]u8 = undefined,
41
42	/// How many nanoseconds between writing updates to the terminal.
43	refresh_rate_ns: u64 = 50 * std.time.ns_per_ms,
44
45	/// How many nanoseconds to keep the output hidden
46	initial_delay_ns: u64 = 500 * std.time.ns_per_ms,
47
48	done: bool = true,
49
50	/// Protects the `refresh` function, as well as `node.recently_updated_child`.
51	/// Without this, callsites would call `Node.end` and then free `Node` memory
52	/// while it was still being accessed by the `refresh` function.
53	update_lock: std.Mutex = .{},
54
55	/// Keeps track of how many columns in the terminal have been output, so that
56	/// we can move the cursor back later.
57	columns_written: usize = undefined,
58
59	/// Represents one unit of progress. Each node can have children nodes, or
60	/// one can use integers with `update`.
61	pub const Node = struct {
62	context: *Progress,
63	parent: ?*Node,
64	name: []const u8,
65	/// Must be handled atomically to be thread-safe.
66	recently_updated_child: ?*Node = null,
67	/// Must be handled atomically to be thread-safe. 0 means null.
68	unprotected_estimated_total_items: usize,
69	/// Must be handled atomically to be thread-safe.
70	unprotected_completed_items: usize,
71
72	/// Create a new child progress node. Thread-safe.
73	/// Call `Node.end` when done.
74	/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
75	/// API to set `self.parent.recently_updated_child` with the return value.
76	/// Until that is fixed you probably want to call `activate` on the return value.
77	/// Passing 0 for `estimated_total_items` means unknown.
78	pub fn start(self: *Node, name: []const u8, estimated_total_items: usize) Node {
79	return Node{
80	.context = self.context,
81	.parent = self,
82	.name = name,
83	.unprotected_estimated_total_items = estimated_total_items,
84	.unprotected_completed_items = 0,
85	};
86	}
87
88	/// This is the same as calling `start` and then `end` on the returned `Node`. Thread-safe.
89	pub fn completeOne(self: *Node) void {
90	self.activate();
91	_ = @atomicRmw(usize, &self.unprotected_completed_items, .Add, 1, .Monotonic);
92	self.context.maybeRefresh();
93	}
94
95	/// Finish a started `Node`. Thread-safe.
96	pub fn end(self: *Node) void {
97	self.context.maybeRefresh();
98	if (self.parent) \|parent\| {
99	{
100	const held = self.context.update_lock.acquire();
101	defer held.release();
102	_ = @cmpxchgStrong(?*Node, &parent.recently_updated_child, self, null, .Monotonic, .Monotonic);
103	}
104	parent.completeOne();
105	} else {
106	self.context.done = true;
107	self.context.refresh();
108	}
109	}
110
111	/// Tell the parent node that this node is actively being worked on. Thread-safe.
112	pub fn activate(self: *Node) void {
113	if (self.parent) \|parent\| {
114	@atomicStore(?*Node, &parent.recently_updated_child, self, .Release);
115	}
116	}
117
118	/// Thread-safe. 0 means unknown.
119	pub fn setEstimatedTotalItems(self: *Node, count: usize) void {
120	@atomicStore(usize, &self.unprotected_estimated_total_items, count, .Monotonic);
121	}
122
123	/// Thread-safe.
124	pub fn setCompletedItems(self: *Node, completed_items: usize) void {
125	@atomicStore(usize, &self.unprotected_completed_items, completed_items, .Monotonic);
126	}
127	};
128
129	/// Create a new progress node.
130	/// Call `Node.end` when done.
131	/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
132	/// API to return Progress rather than accept it as a parameter.
133	/// `estimated_total_items` value of 0 means unknown.
134	pub fn start(self: Progress, name: []const u8, estimated_total_items: usize) !Node {
135	const stderr = std.io.getStdErr();
136	self.terminal = null;
137	if (stderr.supportsAnsiEscapeCodes()) {
138	self.terminal = stderr;
139	self.supports_ansi_escape_codes = true;
140	} else if (std.builtin.os.tag == .windows and stderr.isTty()) {
141	self.terminal = stderr;
142	}
143	self.root = Node{
144	.context = self,
145	.parent = null,
146	.name = name,
147	.unprotected_estimated_total_items = estimated_total_items,
148	.unprotected_completed_items = 0,
149	};
150	self.columns_written = 0;
151	self.prev_refresh_timestamp = 0;
152	self.timer = try std.time.Timer.start();
153	self.done = false;
154	return &self.root;
155	}
156
157	/// Updates the terminal if enough time has passed since last update. Thread-safe.
158	pub fn maybeRefresh(self: *Progress) void {
159	const now = self.timer.read();
160	if (now < self.initial_delay_ns) return;
161	const held = self.update_lock.tryAcquire() orelse return;
162	defer held.release();
163	if (now - self.prev_refresh_timestamp < self.refresh_rate_ns) return;
164	return self.refreshWithHeldLock();
165	}
166
167	/// Updates the terminal and resets `self.next_refresh_timestamp`. Thread-safe.
168	pub fn refresh(self: *Progress) void {
169	const held = self.update_lock.tryAcquire() orelse return;
170	defer held.release();
171
172	return self.refreshWithHeldLock();
173	}
174
175	fn refreshWithHeldLock(self: *Progress) void {
176	const file = self.terminal orelse return;
177
178	const prev_columns_written = self.columns_written;
179	var end: usize = 0;
180	if (self.columns_written > 0) {
181	// restore the cursor position by moving the cursor
182	// `columns_written` cells to the left, then clear the rest of the
183	// line
184	if (self.supports_ansi_escape_codes) {
185	end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[{d}D", .{self.columns_written}) catch unreachable).len;
186	end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[0K", .{}) catch unreachable).len;
187	} else if (std.builtin.os.tag == .windows) winapi: {
188	var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;
189	if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.TRUE)
190	unreachable;
191
192	var cursor_pos = windows.COORD{
193	.X = info.dwCursorPosition.X - @intCast(windows.SHORT, self.columns_written),
194	.Y = info.dwCursorPosition.Y,
195	};
196
197	if (cursor_pos.X < 0)
198	cursor_pos.X = 0;
199
200	const fill_chars = @intCast(windows.DWORD, info.dwSize.X - cursor_pos.X);
201
202	var written: windows.DWORD = undefined;
203	if (windows.kernel32.FillConsoleOutputAttribute(
204	file.handle,
205	info.wAttributes,
206	fill_chars,
207	cursor_pos,
208	&written,
209	) != windows.TRUE) {
210	// Stop trying to write to this file.
211	self.terminal = null;
212	break :winapi;
213	}
214	if (windows.kernel32.FillConsoleOutputCharacterA(
215	file.handle,
216	' ',
217	fill_chars,
218	cursor_pos,
219	&written,
220	) != windows.TRUE) unreachable;
221
222	if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE)
223	unreachable;
224	} else unreachable;
225
226	self.columns_written = 0;
227	}
228
229	if (!self.done) {
230	var need_ellipse = false;
231	var maybe_node: ?*Node = &self.root;
232	while (maybe_node) \|node\| {
233	if (need_ellipse) {
234	self.bufWrite(&end, "... ", .{});
235	}
236	need_ellipse = false;
237	const eti = @atomicLoad(usize, &node.unprotected_estimated_total_items, .Monotonic);
238	const completed_items = @atomicLoad(usize, &node.unprotected_completed_items, .Monotonic);
239	if (node.name.len != 0 or eti > 0) {
240	if (node.name.len != 0) {
241	self.bufWrite(&end, "{s}", .{node.name});
242	need_ellipse = true;
243	}
244	if (eti > 0) {
245	if (need_ellipse) self.bufWrite(&end, " ", .{});
246	self.bufWrite(&end, "[{d}/{d}] ", .{ completed_items + 1, eti });
247	need_ellipse = false;
248	} else if (completed_items != 0) {
249	if (need_ellipse) self.bufWrite(&end, " ", .{});
250	self.bufWrite(&end, "[{d}] ", .{completed_items + 1});
251	need_ellipse = false;
252	}
253	}
254	maybe_node = @atomicLoad(?*Node, &node.recently_updated_child, .Acquire);
255	}
256	if (need_ellipse) {
257	self.bufWrite(&end, "... ", .{});
258	}
259	}
260
261	_ = file.write(self.output_buffer[0..end]) catch \|e\| {
262	// Stop trying to write to this file once it errors.
263	self.terminal = null;
264	};
265	self.prev_refresh_timestamp = self.timer.read();
266	}
267
268	pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void {
269	const file = self.terminal orelse return;
270	self.refresh();
271	file.outStream().print(format, args) catch {
272	self.terminal = null;
273	return;
274	};
275	self.columns_written = 0;
276	}
277
278	fn bufWrite(self: Progress, end: usize, comptime format: []const u8, args: anytype) void {
279	if (std.fmt.bufPrint(self.output_buffer[end.*..], format, args)) \|written\| {
280	const amt = written.len;
281	end.* += amt;
282	self.columns_written += amt;
283	} else \|err\| switch (err) {
284	error.NoSpaceLeft => {
285	self.columns_written += self.output_buffer.len - end.*;
286	end.* = self.output_buffer.len;
287	},
288	}
289	const bytes_needed_for_esc_codes_at_end = if (std.builtin.os.tag == .windows) 0 else 11;
290	const max_end = self.output_buffer.len - bytes_needed_for_esc_codes_at_end;
291	if (end.* > max_end) {
292	const suffix = "... ";
293	self.columns_written = self.columns_written - (end.* - max_end) + suffix.len;
294	std.mem.copy(u8, self.output_buffer[max_end..], suffix);
295	end.* = max_end + suffix.len;
296	}
297	}
298
299	test "basic functionality" {
300	var disable = true;
301	if (disable) {
302	// This test is disabled because it uses time.sleep() and is therefore slow. It also
303	// prints bogus progress data to stderr.
304	return error.SkipZigTest;
305	}
306	var progress = Progress{};
307	const root_node = try progress.start("", 100);
308	defer root_node.end();
309
310	const sub_task_names = [_][]const u8{
311	"reticulating splines",
312	"adjusting shoes",
313	"climbing towers",
314	"pouring juice",
315	};
316	var next_sub_task: usize = 0;
317
318	var i: usize = 0;
319	while (i < 100) : (i += 1) {
320	var node = root_node.start(sub_task_names[next_sub_task], 5);
321	node.activate();
322	next_sub_task = (next_sub_task + 1) % sub_task_names.len;
323
324	node.completeOne();
325	std.time.sleep(5 * std.time.ns_per_ms);
326	node.completeOne();
327	node.completeOne();
328	std.time.sleep(5 * std.time.ns_per_ms);
329	node.completeOne();
330	node.completeOne();
331	std.time.sleep(5 * std.time.ns_per_ms);
332
333	node.end();
334
335	std.time.sleep(5 * std.time.ns_per_ms);
336	}
337	{
338	var node = root_node.start("this is a really long name designed to activate the truncation code. let's find out if it works", 0);
339	node.activate();
340	std.time.sleep(10 * std.time.ns_per_ms);
341	progress.refresh();
342	std.time.sleep(10 * std.time.ns_per_ms);
343	node.end();
344	}
345	}

lib/std/auto_reset_event.zig+22-22

lib/std/progress.zig deleted-310

...	@@ -1,310 +0,0 @@
1	// SPDX-License-Identifier: MIT
2	// Copyright (c) 2015-2020 Zig Contributors
3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4	// The MIT license requires this copyright notice to be included in all copies
5	// and substantial portions of the software.
6	const std = @import("std");
7	const windows = std.os.windows;
8	const testing = std.testing;
9	const assert = std.debug.assert;
10
11	/// This API is non-allocating and non-fallible. The tradeoff is that users of
12	/// this API must provide the storage for each `Progress.Node`.
13	/// Initialize the struct directly, overriding these fields as desired:
14	/// * `refresh_rate_ms`
15	/// * `initial_delay_ms`
16	pub const Progress = struct {
17	/// `null` if the current node (and its children) should
18	/// not print on update()
19	terminal: ?std.fs.File = undefined,
20
21	/// Whether the terminal supports ANSI escape codes.
22	supports_ansi_escape_codes: bool = false,
23
24	root: Node = undefined,
25
26	/// Keeps track of how much time has passed since the beginning.
27	/// Used to compare with `initial_delay_ms` and `refresh_rate_ms`.
28	timer: std.time.Timer = undefined,
29
30	/// When the previous refresh was written to the terminal.
31	/// Used to compare with `refresh_rate_ms`.
32	prev_refresh_timestamp: u64 = undefined,
33
34	/// This buffer represents the maximum number of bytes written to the terminal
35	/// with each refresh.
36	output_buffer: [100]u8 = undefined,
37
38	/// How many nanoseconds between writing updates to the terminal.
39	refresh_rate_ns: u64 = 50 * std.time.ns_per_ms,
40
41	/// How many nanoseconds to keep the output hidden
42	initial_delay_ns: u64 = 500 * std.time.ns_per_ms,
43
44	done: bool = true,
45
46	/// Keeps track of how many columns in the terminal have been output, so that
47	/// we can move the cursor back later.
48	columns_written: usize = undefined,
49
50	/// Represents one unit of progress. Each node can have children nodes, or
51	/// one can use integers with `update`.
52	pub const Node = struct {
53	context: *Progress,
54	parent: ?*Node,
55	completed_items: usize,
56	name: []const u8,
57	recently_updated_child: ?*Node = null,
58
59	/// This field may be updated freely.
60	estimated_total_items: ?usize,
61
62	/// Create a new child progress node.
63	/// Call `Node.end` when done.
64	/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
65	/// API to set `self.parent.recently_updated_child` with the return value.
66	/// Until that is fixed you probably want to call `activate` on the return value.
67	pub fn start(self: *Node, name: []const u8, estimated_total_items: ?usize) Node {
68	return Node{
69	.context = self.context,
70	.parent = self,
71	.completed_items = 0,
72	.name = name,
73	.estimated_total_items = estimated_total_items,
74	};
75	}
76
77	/// This is the same as calling `start` and then `end` on the returned `Node`.
78	pub fn completeOne(self: *Node) void {
79	if (self.parent) \|parent\| parent.recently_updated_child = self;
80	self.completed_items += 1;
81	self.context.maybeRefresh();
82	}
83
84	pub fn end(self: *Node) void {
85	self.context.maybeRefresh();
86	if (self.parent) \|parent\| {
87	if (parent.recently_updated_child) \|parent_child\| {
88	if (parent_child == self) {
89	parent.recently_updated_child = null;
90	}
91	}
92	parent.completeOne();
93	} else {
94	self.context.done = true;
95	self.context.refresh();
96	}
97	}
98
99	/// Tell the parent node that this node is actively being worked on.
100	pub fn activate(self: *Node) void {
101	if (self.parent) \|parent\| parent.recently_updated_child = self;
102	}
103	};
104
105	/// Create a new progress node.
106	/// Call `Node.end` when done.
107	/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
108	/// API to return Progress rather than accept it as a parameter.
109	pub fn start(self: Progress, name: []const u8, estimated_total_items: ?usize) !Node {
110	const stderr = std.io.getStdErr();
111	self.terminal = null;
112	if (stderr.supportsAnsiEscapeCodes()) {
113	self.terminal = stderr;
114	self.supports_ansi_escape_codes = true;
115	} else if (std.builtin.os.tag == .windows and stderr.isTty()) {
116	self.terminal = stderr;
117	}
118	self.root = Node{
119	.context = self,
120	.parent = null,
121	.completed_items = 0,
122	.name = name,
123	.estimated_total_items = estimated_total_items,
124	};
125	self.columns_written = 0;
126	self.prev_refresh_timestamp = 0;
127	self.timer = try std.time.Timer.start();
128	self.done = false;
129	return &self.root;
130	}
131
132	/// Updates the terminal if enough time has passed since last update.
133	pub fn maybeRefresh(self: *Progress) void {
134	const now = self.timer.read();
135	if (now < self.initial_delay_ns) return;
136	if (now - self.prev_refresh_timestamp < self.refresh_rate_ns) return;
137	self.refresh();
138	}
139
140	/// Updates the terminal and resets `self.next_refresh_timestamp`.
141	pub fn refresh(self: *Progress) void {
142	const file = self.terminal orelse return;
143
144	const prev_columns_written = self.columns_written;
145	var end: usize = 0;
146	if (self.columns_written > 0) {
147	// restore the cursor position by moving the cursor
148	// `columns_written` cells to the left, then clear the rest of the
149	// line
150	if (self.supports_ansi_escape_codes) {
151	end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[{}D", .{self.columns_written}) catch unreachable).len;
152	end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[0K", .{}) catch unreachable).len;
153	} else if (std.builtin.os.tag == .windows) winapi: {
154	var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;
155	if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.TRUE)
156	unreachable;
157
158	var cursor_pos = windows.COORD{
159	.X = info.dwCursorPosition.X - @intCast(windows.SHORT, self.columns_written),
160	.Y = info.dwCursorPosition.Y,
161	};
162
163	if (cursor_pos.X < 0)
164	cursor_pos.X = 0;
165
166	const fill_chars = @intCast(windows.DWORD, info.dwSize.X - cursor_pos.X);
167
168	var written: windows.DWORD = undefined;
169	if (windows.kernel32.FillConsoleOutputAttribute(
170	file.handle,
171	info.wAttributes,
172	fill_chars,
173	cursor_pos,
174	&written,
175	) != windows.TRUE) {
176	// Stop trying to write to this file.
177	self.terminal = null;
178	break :winapi;
179	}
180	if (windows.kernel32.FillConsoleOutputCharacterA(
181	file.handle,
182	' ',
183	fill_chars,
184	cursor_pos,
185	&written,
186	) != windows.TRUE) unreachable;
187
188	if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE)
189	unreachable;
190	} else unreachable;
191
192	self.columns_written = 0;
193	}
194
195	if (!self.done) {
196	var need_ellipse = false;
197	var maybe_node: ?*Node = &self.root;
198	while (maybe_node) \|node\| {
199	if (need_ellipse) {
200	self.bufWrite(&end, "... ", .{});
201	}
202	need_ellipse = false;
203	if (node.name.len != 0 or node.estimated_total_items != null) {
204	if (node.name.len != 0) {
205	self.bufWrite(&end, "{}", .{node.name});
206	need_ellipse = true;
207	}
208	if (node.estimated_total_items) \|total\| {
209	if (need_ellipse) self.bufWrite(&end, " ", .{});
210	self.bufWrite(&end, "[{}/{}] ", .{ node.completed_items + 1, total });
211	need_ellipse = false;
212	} else if (node.completed_items != 0) {
213	if (need_ellipse) self.bufWrite(&end, " ", .{});
214	self.bufWrite(&end, "[{}] ", .{node.completed_items + 1});
215	need_ellipse = false;
216	}
217	}
218	maybe_node = node.recently_updated_child;
219	}
220	if (need_ellipse) {
221	self.bufWrite(&end, "... ", .{});
222	}
223	}
224
225	_ = file.write(self.output_buffer[0..end]) catch \|e\| {
226	// Stop trying to write to this file once it errors.
227	self.terminal = null;
228	};
229	self.prev_refresh_timestamp = self.timer.read();
230	}
231
232	pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void {
233	const file = self.terminal orelse return;
234	self.refresh();
235	file.outStream().print(format, args) catch {
236	self.terminal = null;
237	return;
238	};
239	self.columns_written = 0;
240	}
241
242	fn bufWrite(self: Progress, end: usize, comptime format: []const u8, args: anytype) void {
243	if (std.fmt.bufPrint(self.output_buffer[end.*..], format, args)) \|written\| {
244	const amt = written.len;
245	end.* += amt;
246	self.columns_written += amt;
247	} else \|err\| switch (err) {
248	error.NoSpaceLeft => {
249	self.columns_written += self.output_buffer.len - end.*;
250	end.* = self.output_buffer.len;
251	},
252	}
253	const bytes_needed_for_esc_codes_at_end = if (std.builtin.os.tag == .windows) 0 else 11;
254	const max_end = self.output_buffer.len - bytes_needed_for_esc_codes_at_end;
255	if (end.* > max_end) {
256	const suffix = "... ";
257	self.columns_written = self.columns_written - (end.* - max_end) + suffix.len;
258	std.mem.copy(u8, self.output_buffer[max_end..], suffix);
259	end.* = max_end + suffix.len;
260	}
261	}
262	};
263
264	test "basic functionality" {
265	var disable = true;
266	if (disable) {
267	// This test is disabled because it uses time.sleep() and is therefore slow. It also
268	// prints bogus progress data to stderr.
269	return error.SkipZigTest;
270	}
271	var progress = Progress{};
272	const root_node = try progress.start("", 100);
273	defer root_node.end();
274
275	const sub_task_names = [_][]const u8{
276	"reticulating splines",
277	"adjusting shoes",
278	"climbing towers",
279	"pouring juice",
280	};
281	var next_sub_task: usize = 0;
282
283	var i: usize = 0;
284	while (i < 100) : (i += 1) {
285	var node = root_node.start(sub_task_names[next_sub_task], 5);
286	node.activate();
287	next_sub_task = (next_sub_task + 1) % sub_task_names.len;
288
289	node.completeOne();
290	std.time.sleep(5 * std.time.ns_per_ms);
291	node.completeOne();
292	node.completeOne();
293	std.time.sleep(5 * std.time.ns_per_ms);
294	node.completeOne();
295	node.completeOne();
296	std.time.sleep(5 * std.time.ns_per_ms);
297
298	node.end();
299
300	std.time.sleep(5 * std.time.ns_per_ms);
301	}
302	{
303	var node = root_node.start("this is a really long name designed to activate the truncation code. let's find out if it works", null);
304	node.activate();
305	std.time.sleep(10 * std.time.ns_per_ms);
306	progress.refresh();
307	std.time.sleep(10 * std.time.ns_per_ms);
308	node.end();
309	}
310	}

lib/std/special/test_runner.zig+1-1

...	@@ -36,7 +36,7 @@ pub fn main() anyerror!void {	...	@@ -36,7 +36,7 @@ pub fn main() anyerror!void {
36	}	36	}
37	std.testing.log_level = .warn;	37	std.testing.log_level = .warn;
38		38
39	var test_node = root_node.start(test_fn.name, null);	39	var test_node = root_node.start(test_fn.name, 0);
40	test_node.activate();	40	test_node.activate();
41	progress.refresh();	41	progress.refresh();
42	if (progress.terminal == null) {	42	if (progress.terminal == null) {

lib/std/std.zig+1-1

...	@@ -29,7 +29,7 @@ pub const PackedIntArrayEndian = @import("packed_int_array.zig").PackedIntArrayE	...	@@ -29,7 +29,7 @@ pub const PackedIntArrayEndian = @import("packed_int_array.zig").PackedIntArrayE
29	pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;	29	pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;
30	pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;	30	pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;
31	pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;	31	pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
32	pub const Progress = @import("progress.zig").Progress;	32	pub const Progress = @import("Progress.zig");
33	pub const ResetEvent = @import("reset_event.zig").ResetEvent;	33	pub const ResetEvent = @import("reset_event.zig").ResetEvent;
34	pub const SemanticVersion = @import("SemanticVersion.zig");	34	pub const SemanticVersion = @import("SemanticVersion.zig");
35	pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;	35	pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;

src/Compilation.zig+85-30

 @@ -26,6 +26,8 @@ const Module = @import("Module.zig");
 const Cache = @import("Cache.zig");
 const stage1 = @import("stage1.zig");
 const translate_c = @import("translate_c.zig");
+const ThreadPool = @import("ThreadPool.zig");
+const WaitGroup = @import("WaitGroup.zig");
 /// General-purpose allocator. Used for both temporary and long-term storage.
 gpa: *Allocator,
 @@ -41,7 +43,12 @@ link_error_flags: link.File.ErrorFlags = .{},
 work_queue: std.fifo.LinearFifo(Job, .Dynamic),
+/// These jobs are to invoke the Clang compiler to create an object file, which
+/// gets linked with the Compilation.
+c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic),
 /// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
+/// This data is accessed by multiple threads and is protected by `mutex`.
 failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *ErrorMsg) = .{},
 keep_source_files_loaded: bool,
 @@ -74,6 +81,7 @@ zig_lib_directory: Directory,
 local_cache_directory: Directory,
 global_cache_directory: Directory,
 libc_include_dir_list: []const []const u8,
+thread_pool: *ThreadPool,
 /// Populated when we build the libc++ static library. A Job to build this is placed in the queue
 /// and resolved before calling linker.flush().
 @@ -111,6 +119,9 @@ owned_link_dir: ?std.fs.Dir,
 /// Don't use this for anything other than stage1 compatibility.
 color: @import("main.zig").Color = .auto,
+/// This mutex guards all `Compilation` mutable state.
+mutex: std.Mutex = .{},
 test_filter: ?[]const u8,
 test_name_prefix: ?[]const u8,
 test_evented_io: bool,
 @@ -150,9 +161,6 @@ const Job = union(enum) {
     /// The source file containing the Decl has been updated, and so the
     /// Decl may need its line number information updated in the debug info.
     update_line_number: *Module.Decl,
-    /// Invoke the Clang compiler to create an object file, which gets linked
-    /// with the Compilation.
-    c_object: *CObject,
     /// one of the glibc static objects
     glibc_crt_file: glibc.CRTFile,
 @@ -330,6 +338,7 @@ pub const InitOptions = struct {
     root_name: []const u8,
     root_pkg: ?*Package,
     output_mode: std.builtin.OutputMode,
+    thread_pool: *ThreadPool,
     dynamic_linker: ?[]const u8 = null,
     /// `null` means to not emit a binary file.
     emit_bin: ?EmitLoc,
 @@ -971,6 +980,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .emit_analysis = options.emit_analysis,
             .emit_docs = options.emit_docs,
             .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa),
+            .c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),
             .keep_source_files_loaded = options.keep_source_files_loaded,
             .use_clang = use_clang,
             .clang_argv = options.clang_argv,
 @@ -979,6 +989,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .self_exe_path = options.self_exe_path,
             .libc_include_dir_list = libc_dirs.libc_include_dir_list,
             .sanitize_c = sanitize_c,
+            .thread_pool = options.thread_pool,
             .clang_passthrough_mode = options.clang_passthrough_mode,
             .clang_preprocessor_mode = options.clang_preprocessor_mode,
             .verbose_cc = options.verbose_cc,
 @@ -1190,11 +1201,13 @@ pub fn update(self: *Compilation) !void {
     const tracy = trace(@src());
     defer tracy.end();
+    self.c_object_cache_digest_set.clearRetainingCapacity();
     // For compiling C objects, we rely on the cache hash system to avoid duplicating work.
     // Add a Job for each C object.
-    try self.work_queue.ensureUnusedCapacity(self.c_object_table.items().len);
+    try self.c_object_work_queue.ensureUnusedCapacity(self.c_object_table.items().len);
     for (self.c_object_table.items()) |entry| {
-        self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key });
+        self.c_object_work_queue.writeItemAssumeCapacity(entry.key);
     }
     const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_llvm;
 @@ -1365,13 +1378,23 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
 pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemory }!void {
     var progress: std.Progress = .{};
-    var main_progress_node = try progress.start("", null);
+    var main_progress_node = try progress.start("", 0);
     defer main_progress_node.end();
     if (self.color == .off) progress.terminal = null;
     var c_comp_progress_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
     defer c_comp_progress_node.end();
+    var wg = WaitGroup{};
+    defer wg.wait();
+    while (self.c_object_work_queue.readItem()) |c_object| {
+        wg.start();
+        try self.thread_pool.spawn(workerUpdateCObject, .{
+            self, c_object, &c_comp_progress_node, &wg,
+        });
+    }
     while (self.work_queue.readItem()) |work_item| switch (work_item) {
         .codegen_decl => |decl| switch (decl.analysis) {
             .unreferenced => unreachable,
 @@ -1447,21 +1470,6 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 decl.analysis = .codegen_failure_retryable;
             };
         },
-        .c_object => |c_object| {
-            self.updateCObject(c_object, &c_comp_progress_node) catch |err| switch (err) {
-                error.AnalysisFail => continue,
-                else => {
-                    try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1);
-                    self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create(
-                        self.gpa,
-                        0,
-                        "unable to build C object: {s}",
-                        .{@errorName(err)},
-                    ));
-                    c_object.status = .{ .failure = {} };
-                },
-            };
-        },
         .glibc_crt_file => |crt_file| {
             glibc.buildCRTFile(self, crt_file) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
 @@ -1553,7 +1561,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
     };
 }
-pub fn obtainCObjectCacheManifest(comp: *Compilation) Cache.Manifest {
+pub fn obtainCObjectCacheManifest(comp: *const Compilation) Cache.Manifest {
     var man = comp.cache_parent.obtain();
     // Only things that need to be added on top of the base hash, and only things
 @@ -1708,6 +1716,37 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
     };
 }
+fn workerUpdateCObject(
+    comp: *Compilation,
+    c_object: *CObject,
+    progress_node: *std.Progress.Node,
+    wg: *WaitGroup,
+) void {
+    defer wg.stop();
+    comp.updateCObject(c_object, progress_node) catch |err| switch (err) {
+        error.AnalysisFail => return,
+        else => {
+            {
+                const lock = comp.mutex.acquire();
+                defer lock.release();
+                comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1) catch {
+                    fatal("TODO handle this by setting c_object.status = oom failure", .{});
+                };
+                comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, ErrorMsg.create(
+                    comp.gpa,
+                    0,
+                    "unable to build C object: {s}",
+                    .{@errorName(err)},
+                ) catch {
+                    fatal("TODO handle this by setting c_object.status = oom failure", .{});
+                });
+            }
+            c_object.status = .{ .failure = {} };
+        },
+    };
+}
 fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *std.Progress.Node) !void {
     if (!build_options.have_llvm) {
         return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});
 @@ -1720,6 +1759,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
     if (c_object.clearStatus(comp.gpa)) {
         // There was previous failure.
+        const lock = comp.mutex.acquire();
+        defer lock.release();
         comp.failed_c_objects.removeAssertDiscard(c_object);
     }
 @@ -1747,8 +1788,16 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
     }
     {
-        const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, man.hash.peekBin());
+        const is_collision = blk: {
-        if (gop.found_existing) {
+            const bin_digest = man.hash.peekBin();
+            const lock = comp.mutex.acquire();
+            defer lock.release();
+            const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest);
+            break :blk gop.found_existing;
+        };
+        if (is_collision) {
             return comp.failCObj(
                 c_object,
                 "the same source file was already added to the same compilation with the same flags",
 @@ -1764,7 +1813,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
     const c_source_basename = std.fs.path.basename(c_object.src.src_path);
     c_comp_progress_node.activate();
-    var child_progress_node = c_comp_progress_node.start(c_source_basename, null);
+    var child_progress_node = c_comp_progress_node.start(c_source_basename, 0);
     child_progress_node.activate();
     defer child_progress_node.end();
 @@ -1929,7 +1978,7 @@ pub fn addTranslateCCArgs(
 /// Add common C compiler args between translate-c and C object compilation.
 pub fn addCCArgs(
-    comp: *Compilation,
+    comp: *const Compilation,
     arena: *Allocator,
     argv: *std.ArrayList([]const u8),
     ext: FileExt,
 @@ -2164,10 +2213,14 @@ fn failCObj(comp: *Compilation, c_object: *CObject, comptime format: []const u8,
 fn failCObjWithOwnedErrorMsg(comp: *Compilation, c_object: *CObject, err_msg: *ErrorMsg) InnerError {
     {
-        errdefer err_msg.destroy(comp.gpa);
+        const lock = comp.mutex.acquire();
-        try comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1);
+        defer lock.release();
+        {
+            errdefer err_msg.destroy(comp.gpa);
+            try comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1);
+        }
+        comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg);
     }
-    comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg);
     c_object.status = .failure;
     return error.AnalysisFail;
 }
 @@ -2324,7 +2377,7 @@ test "classifyFileExt" {
     std.testing.expectEqual(FileExt.zir, classifyFileExt("foo.zir"));
 }
-fn haveFramePointer(comp: *Compilation) bool {
+fn haveFramePointer(comp: *const Compilation) bool {
     // If you complicate this logic make sure you update the parent cache hash.
     // Right now it's not in the cache hash because the value depends on optimize_mode
     // and strip which are both already part of the hash.
 @@ -2775,6 +2828,7 @@ fn buildOutputFromZig(
         .root_name = root_name,
         .root_pkg = &root_pkg,
         .output_mode = fixed_output_mode,
+        .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = optimize_mode,
 @@ -3148,6 +3202,7 @@ pub fn build_crt_file(
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
+        .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,

src/Event.zig created+43

...	@@ -0,0 +1,43 @@
1	// SPDX-License-Identifier: MIT
2	// Copyright (c) 2015-2020 Zig Contributors
3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4	// The MIT license requires this copyright notice to be included in all copies
5	// and substantial portions of the software.
6	const std = @import("std");
7	const Event = @This();
8
9	lock: std.Mutex = .{},
10	event: std.ResetEvent = undefined,
11	state: enum { empty, waiting, notified } = .empty,
12
13	pub fn wait(self: *Event) void {
14	const held = self.lock.acquire();
15
16	switch (self.state) {
17	.empty => {
18	self.state = .waiting;
19	self.event = @TypeOf(self.event).init();
20	held.release();
21	self.event.wait();
22	self.event.deinit();
23	},
24	.waiting => unreachable,
25	.notified => held.release(),
26	}
27	}
28
29	pub fn set(self: *Event) void {
30	const held = self.lock.acquire();
31
32	switch (self.state) {
33	.empty => {
34	self.state = .notified;
35	held.release();
36	},
37	.waiting => {
38	held.release();
39	self.event.set();
40	},
41	.notified => unreachable,
42	}
43	}

src/ThreadPool.zig created+126

...	@@ -0,0 +1,126 @@
1	// SPDX-License-Identifier: MIT
2	// Copyright (c) 2015-2020 Zig Contributors
3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4	// The MIT license requires this copyright notice to be included in all copies
5	// and substantial portions of the software.
6	const std = @import("std");
7	const ThreadPool = @This();
8
9	lock: std.Mutex = .{},
10	is_running: bool = true,
11	allocator: *std.mem.Allocator,
12	running: usize = 0,
13	threads: []*std.Thread,
14	run_queue: RunQueue = .{},
15	idle_queue: IdleQueue = .{},
16
17	const IdleQueue = std.SinglyLinkedList(std.AutoResetEvent);
18	const RunQueue = std.SinglyLinkedList(Runnable);
19	const Runnable = struct {
20	runFn: fn (*Runnable) void,
21	};
22
23	pub fn init(self: ThreadPool, allocator: std.mem.Allocator) !void {
24	self.* = .{
25	.allocator = allocator,
26	.threads = &[_]*std.Thread{},
27	};
28	if (std.builtin.single_threaded)
29	return;
30
31	errdefer self.deinit();
32
33	var num_threads = std.Thread.cpuCount() catch 1;
34	if (num_threads > 0)
35	self.threads = try allocator.alloc(*std.Thread, num_threads);
36
37	while (num_threads > 0) : (num_threads -= 1) {
38	const thread = try std.Thread.spawn(self, runWorker);
39	self.threads[self.running] = thread;
40	self.running += 1;
41	}
42	}
43
44	pub fn deinit(self: *ThreadPool) void {
45	self.shutdown();
46
47	std.debug.assert(!self.is_running);
48	for (self.threads[0..self.running]) \|thread\|
49	thread.wait();
50
51	defer self.threads = &[_]*std.Thread{};
52	if (self.running > 0)
53	self.allocator.free(self.threads);
54	}
55
56	pub fn shutdown(self: *ThreadPool) void {
57	const held = self.lock.acquire();
58
59	if (!self.is_running)
60	return held.release();
61
62	var idle_queue = self.idle_queue;
63	self.idle_queue = .{};
64	self.is_running = false;
65	held.release();
66
67	while (idle_queue.popFirst()) \|idle_node\|
68	idle_node.data.set();
69	}
70
71	pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
72	if (std.builtin.single_threaded) {
73	@call(.{}, func, args);
74	return;
75	}
76	const Args = @TypeOf(args);
77	const Closure = struct {
78	arguments: Args,
79	pool: *ThreadPool,
80	run_node: RunQueue.Node = .{ .data = .{ .runFn = runFn } },
81
82	fn runFn(runnable: *Runnable) void {
83	const run_node = @fieldParentPtr(RunQueue.Node, "data", runnable);
84	const closure = @fieldParentPtr(@This(), "run_node", run_node);
85	const result = @call(.{}, func, closure.arguments);
86	closure.pool.allocator.destroy(closure);
87	}
88	};
89
90	const closure = try self.allocator.create(Closure);
91	closure.* = .{
92	.arguments = args,
93	.pool = self,
94	};
95
96	const held = self.lock.acquire();
97	self.run_queue.prepend(&closure.run_node);
98
99	const idle_node = self.idle_queue.popFirst();
100	held.release();
101
102	if (idle_node) \|node\|
103	node.data.set();
104	}
105
106	fn runWorker(self: *ThreadPool) void {
107	while (true) {
108	const held = self.lock.acquire();
109
110	if (self.run_queue.popFirst()) \|run_node\| {
111	held.release();
112	(run_node.data.runFn)(&run_node.data);
113	continue;
114	}
115
116	if (!self.is_running) {
117	held.release();
118	return;
119	}
120
121	var idle_node = IdleQueue.Node{ .data = .{} };
122	self.idle_queue.prepend(&idle_node);
123	held.release();
124	idle_node.data.wait();
125	}
126	}

src/WaitGroup.zig created+46

...	@@ -0,0 +1,46 @@
1	// SPDX-License-Identifier: MIT
2	// Copyright (c) 2015-2020 Zig Contributors
3	// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4	// The MIT license requires this copyright notice to be included in all copies
5	// and substantial portions of the software.
6	const std = @import("std");
7	const WaitGroup = @This();
8	const Event = @import("Event.zig");
9
10	lock: std.Mutex = .{},
11	counter: usize = 0,
12	event: ?*Event = null,
13
14	pub fn start(self: *WaitGroup) void {
15	const held = self.lock.acquire();
16	defer held.release();
17
18	self.counter += 1;
19	}
20
21	pub fn stop(self: *WaitGroup) void {
22	var event: ?*Event = null;
23	defer if (event) \|waiter\|
24	waiter.set();
25
26	const held = self.lock.acquire();
27	defer held.release();
28
29	self.counter -= 1;
30	if (self.counter == 0)
31	std.mem.swap(?*Event, &self.event, &event);
32	}
33
34	pub fn wait(self: *WaitGroup) void {
35	var event = Event{};
36	var has_event = false;
37	defer if (has_event)
38	event.wait();
39
40	const held = self.lock.acquire();
41	defer held.release();
42
43	has_event = self.counter != 0;
44	if (has_event)
45	self.event = &event;
46	}

src/glibc.zig+1

...	@@ -936,6 +936,7 @@ fn buildSharedLib(	...	@@ -936,6 +936,7 @@ fn buildSharedLib(
936	.root_pkg = null,	936	.root_pkg = null,
937	.output_mode = .Lib,	937	.output_mode = .Lib,
938	.link_mode = .Dynamic,	938	.link_mode = .Dynamic,
		939	.thread_pool = comp.thread_pool,
939	.libc_installation = comp.bin_file.options.libc_installation,	940	.libc_installation = comp.bin_file.options.libc_installation,
940	.emit_bin = emit_bin,	941	.emit_bin = emit_bin,
941	.optimize_mode = comp.bin_file.options.optimize_mode,	942	.optimize_mode = comp.bin_file.options.optimize_mode,

src/libcxx.zig+2

...	@@ -162,6 +162,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {	...	@@ -162,6 +162,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
162	.root_name = root_name,	162	.root_name = root_name,
163	.root_pkg = null,	163	.root_pkg = null,
164	.output_mode = output_mode,	164	.output_mode = output_mode,
		165	.thread_pool = comp.thread_pool,
165	.libc_installation = comp.bin_file.options.libc_installation,	166	.libc_installation = comp.bin_file.options.libc_installation,
166	.emit_bin = emit_bin,	167	.emit_bin = emit_bin,
167	.optimize_mode = comp.bin_file.options.optimize_mode,	168	.optimize_mode = comp.bin_file.options.optimize_mode,
...	@@ -280,6 +281,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {	...	@@ -280,6 +281,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
280	.root_name = root_name,	281	.root_name = root_name,
281	.root_pkg = null,	282	.root_pkg = null,
282	.output_mode = output_mode,	283	.output_mode = output_mode,
		284	.thread_pool = comp.thread_pool,
283	.libc_installation = comp.bin_file.options.libc_installation,	285	.libc_installation = comp.bin_file.options.libc_installation,
284	.emit_bin = emit_bin,	286	.emit_bin = emit_bin,
285	.optimize_mode = comp.bin_file.options.optimize_mode,	287	.optimize_mode = comp.bin_file.options.optimize_mode,

src/libunwind.zig+1

...	@@ -95,6 +95,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {	...	@@ -95,6 +95,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
95	.root_name = root_name,	95	.root_name = root_name,
96	.root_pkg = null,	96	.root_pkg = null,
97	.output_mode = output_mode,	97	.output_mode = output_mode,
		98	.thread_pool = comp.thread_pool,
98	.libc_installation = comp.bin_file.options.libc_installation,	99	.libc_installation = comp.bin_file.options.libc_installation,
99	.emit_bin = emit_bin,	100	.emit_bin = emit_bin,
100	.optimize_mode = comp.bin_file.options.optimize_mode,	101	.optimize_mode = comp.bin_file.options.optimize_mode,

src/main.zig+10

src/musl.zig+1

...	@@ -200,6 +200,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {	...	@@ -200,6 +200,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
200	.root_pkg = null,	200	.root_pkg = null,
201	.output_mode = .Lib,	201	.output_mode = .Lib,
202	.link_mode = .Dynamic,	202	.link_mode = .Dynamic,
		203	.thread_pool = comp.thread_pool,
203	.libc_installation = comp.bin_file.options.libc_installation,	204	.libc_installation = comp.bin_file.options.libc_installation,
204	.emit_bin = Compilation.EmitLoc{ .directory = null, .basename = "libc.so" },	205	.emit_bin = Compilation.EmitLoc{ .directory = null, .basename = "libc.so" },
205	.optimize_mode = comp.bin_file.options.optimize_mode,	206	.optimize_mode = comp.bin_file.options.optimize_mode,

src/stage1.zig+4-4

...	@@ -293,7 +293,7 @@ export fn stage2_progress_start_root(	...	@@ -293,7 +293,7 @@ export fn stage2_progress_start_root(
293	) *std.Progress.Node {	293	) *std.Progress.Node {
294	return progress.start(	294	return progress.start(
295	name_ptr[0..name_len],	295	name_ptr[0..name_len],
296	if (estimated_total_items == 0) null else estimated_total_items,	296	estimated_total_items,
297	) catch @panic("timer unsupported");	297	) catch @panic("timer unsupported");
298	}	298	}
299		299
...	@@ -312,7 +312,7 @@ export fn stage2_progress_start(	...	@@ -312,7 +312,7 @@ export fn stage2_progress_start(
312	const child_node = std.heap.c_allocator.create(std.Progress.Node) catch @panic("out of memory");	312	const child_node = std.heap.c_allocator.create(std.Progress.Node) catch @panic("out of memory");
313	child_node.* = node.start(	313	child_node.* = node.start(
314	name_ptr[0..name_len],	314	name_ptr[0..name_len],
315	if (estimated_total_items == 0) null else estimated_total_items,	315	estimated_total_items,
316	);	316	);
317	child_node.activate();	317	child_node.activate();
318	return child_node;	318	return child_node;
...	@@ -333,8 +333,8 @@ export fn stage2_progress_complete_one(node: *std.Progress.Node) void {	...	@@ -333,8 +333,8 @@ export fn stage2_progress_complete_one(node: *std.Progress.Node) void {
333		333
334	// ABI warning	334	// ABI warning
335	export fn stage2_progress_update_node(node: *std.Progress.Node, done_count: usize, total_count: usize) void {	335	export fn stage2_progress_update_node(node: *std.Progress.Node, done_count: usize, total_count: usize) void {
336	node.completed_items = done_count;	336	node.setCompletedItems(done_count);
337	node.estimated_total_items = total_count;	337	node.setEstimatedTotalItems(total_count);
338	node.activate();	338	node.activate();
339	node.context.maybeRefresh();	339	node.context.maybeRefresh();
340	}	340	}

src/stage1/zig0.cpp+4

src/test.zig+26-13

 @@ -10,6 +10,7 @@ const enable_qemu: bool = build_options.enable_qemu;
 const enable_wine: bool = build_options.enable_wine;
 const enable_wasmtime: bool = build_options.enable_wasmtime;
 const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir;
+const ThreadPool = @import("ThreadPool.zig");
 const cheader = @embedFile("link/cbe.h");
 @@ -467,6 +468,10 @@ pub const TestContext = struct {
         defer zig_lib_directory.handle.close();
         defer std.testing.allocator.free(zig_lib_directory.path.?);
+        var thread_pool: ThreadPool = undefined;
+        try thread_pool.init(std.testing.allocator);
+        defer thread_pool.deinit();
         for (self.cases.items) |case| {
             if (build_options.skip_non_native and case.target.getCpuArch() != std.Target.current.cpu.arch)
                 continue;
 @@ -480,7 +485,13 @@ pub const TestContext = struct {
             progress.initial_delay_ns = 0;
             progress.refresh_rate_ns = 0;
-            try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_directory);
+            try self.runOneCase(
+                std.testing.allocator,
+                &prg_node,
+                case,
+                zig_lib_directory,
+                &thread_pool,
+            );
         }
     }
 @@ -490,6 +501,7 @@ pub const TestContext = struct {
         root_node: *std.Progress.Node,
         case: Case,
         zig_lib_directory: Compilation.Directory,
+        thread_pool: *ThreadPool,
     ) !void {
         const target_info = try std.zig.system.NativeTargetInfo.detect(allocator, case.target);
         const target = target_info.target;
 @@ -539,6 +551,7 @@ pub const TestContext = struct {
             .local_cache_directory = zig_cache_directory,
             .global_cache_directory = zig_cache_directory,
             .zig_lib_directory = zig_lib_directory,
+            .thread_pool = thread_pool,
             .root_name = "test_case",
             .target = target,
             // TODO: support tests for object file building, and library builds
 @@ -565,12 +578,12 @@ pub const TestContext = struct {
             update_node.activate();
             defer update_node.end();
-            var sync_node = update_node.start("write", null);
+            var sync_node = update_node.start("write", 0);
             sync_node.activate();
             try tmp.dir.writeFile(tmp_src_path, update.src);
             sync_node.end();
-            var module_node = update_node.start("parse/analysis/codegen", null);
+            var module_node = update_node.start("parse/analysis/codegen", 0);
             module_node.activate();
             try comp.makeBinFileWritable();
             try comp.update();
 @@ -622,21 +635,21 @@ pub const TestContext = struct {
                             }
                         }
                     } else {
-                        update_node.estimated_total_items = 5;
+                        update_node.setEstimatedTotalItems(5);
-                        var emit_node = update_node.start("emit", null);
+                        var emit_node = update_node.start("emit", 0);
                         emit_node.activate();
                         var new_zir_module = try zir.emit(allocator, comp.bin_file.options.module.?);
                         defer new_zir_module.deinit(allocator);
                         emit_node.end();
-                        var write_node = update_node.start("write", null);
+                        var write_node = update_node.start("write", 0);
                         write_node.activate();
                         var out_zir = std.ArrayList(u8).init(allocator);
                         defer out_zir.deinit();
                         try new_zir_module.writeToStream(allocator, out_zir.outStream());
                         write_node.end();
-                        var test_node = update_node.start("assert", null);
+                        var test_node = update_node.start("assert", 0);
                         test_node.activate();
                         defer test_node.end();
 @@ -653,7 +666,7 @@ pub const TestContext = struct {
                     }
                 },
                 .Error => |e| {
-                    var test_node = update_node.start("assert", null);
+                    var test_node = update_node.start("assert", 0);
                     test_node.activate();
                     defer test_node.end();
                     var handled_errors = try arena.alloc(bool, e.len);
 @@ -710,9 +723,9 @@ pub const TestContext = struct {
                 .Execution => |expected_stdout| {
                     std.debug.assert(!case.cbe);
-                    update_node.estimated_total_items = 4;
+                    update_node.setEstimatedTotalItems(4);
                     var exec_result = x: {
-                        var exec_node = update_node.start("execute", null);
+                        var exec_node = update_node.start("execute", 0);
                         exec_node.activate();
                         defer exec_node.end();
 @@ -775,7 +788,7 @@ pub const TestContext = struct {
                             .cwd_dir = tmp.dir,
                         });
                     };
-                    var test_node = update_node.start("test", null);
+                    var test_node = update_node.start("test", 0);
                     test_node.activate();
                     defer test_node.end();
                     defer allocator.free(exec_result.stdout);
 @@ -854,7 +867,7 @@ pub const TestContext = struct {
         };
         {
-            var load_node = update_node.start("load", null);
+            var load_node = update_node.start("load", 0);
             load_node.activate();
             defer load_node.end();
 @@ -892,7 +905,7 @@ pub const TestContext = struct {
             }
         }
-        var exec_node = update_node.start("execute", null);
+        var exec_node = update_node.start("execute", 0);
         exec_node.activate();
         defer exec_node.end();

author	Andrew Kelley <andrew@ziglang.org> 2020-12-20 21:19:05-05:00
committer	GitHub <noreply@github.com> 2020-12-20 21:19:05-05:00
log	4918605176c2e48c178847ea281b790334207733
tree	4adb4cbf5e6764df65747fb3bae2c1640060eb89
parent	4964bb3282bf13de03a79fad1fb9bca104dc1930
parent	1d94a6893689ad1fb8e06308ae51603a6c8708a8
signature	Signed by PGP key 4AEE18F83AFDEB23

...	@@ -11,33 +11,33 @@ const assert = std.debug.assert;	...	@@ -11,33 +11,33 @@ const assert = std.debug.assert;
11	/// Similar to std.ResetEvent but on `set()` it also (atomically) does `reset()`.	11	/// Similar to std.ResetEvent but on `set()` it also (atomically) does `reset()`.
12	/// Unlike std.ResetEvent, `wait()` can only be called by one thread (MPSC-like).	12	/// Unlike std.ResetEvent, `wait()` can only be called by one thread (MPSC-like).
13	pub const AutoResetEvent = struct {	13	pub const AutoResetEvent = struct {
14	// AutoResetEvent has 3 possible states:	14	/// AutoResetEvent has 3 possible states:
15	// - UNSET: the AutoResetEvent is currently unset	15	/// - UNSET: the AutoResetEvent is currently unset
16	// - SET: the AutoResetEvent was notified before a wait() was called	16	/// - SET: the AutoResetEvent was notified before a wait() was called
17	// - <std.ResetEvent pointer>: there is an active waiter waiting for a notification.	17	/// - <std.ResetEvent pointer>: there is an active waiter waiting for a notification.
18	//	18	///
19	// When attempting to wait:	19	/// When attempting to wait:
20	// if the event is unset, it registers a ResetEvent pointer to be notified when the event is set	20	/// if the event is unset, it registers a ResetEvent pointer to be notified when the event is set
21	// if the event is already set, then it consumes the notification and resets the event.	21	/// if the event is already set, then it consumes the notification and resets the event.
22	//	22	///
23	// When attempting to notify:	23	/// When attempting to notify:
24	// if the event is unset, then we set the event	24	/// if the event is unset, then we set the event
25	// if theres a waiting ResetEvent, then we unset the event and notify the ResetEvent	25	/// if theres a waiting ResetEvent, then we unset the event and notify the ResetEvent
26	//	26	///
27	// This ensures that the event is automatically reset after a wait() has been issued	27	/// This ensures that the event is automatically reset after a wait() has been issued
28	// and avoids the race condition when using std.ResetEvent in the following scenario:	28	/// and avoids the race condition when using std.ResetEvent in the following scenario:
29	// thread 1 \| thread 2	29	/// thread 1 \| thread 2
30	// std.ResetEvent.wait() \|	30	/// std.ResetEvent.wait() \|
31	// \| std.ResetEvent.set()	31	/// \| std.ResetEvent.set()
32	// \| std.ResetEvent.set()	32	/// \| std.ResetEvent.set()
33	// std.ResetEvent.reset() \|	33	/// std.ResetEvent.reset() \|
34	// std.ResetEvent.wait() \| (missed the second .set() notification above)	34	/// std.ResetEvent.wait() \| (missed the second .set() notification above)
35	state: usize = UNSET,	35	state: usize = UNSET,
36		36
37	const UNSET = 0;	37	const UNSET = 0;
38	const SET = 1;	38	const SET = 1;
39		39
40	// the minimum alignment for the `std.ResetEvent` created by wait()	40	/// the minimum alignment for the `std.ResetEvent` created by wait()
41	const event_align = std.math.max(@alignOf(std.ResetEvent), 2);	41	const event_align = std.math.max(@alignOf(std.ResetEvent), 2);
42		42
43	pub fn wait(self: *AutoResetEvent) void {	43	pub fn wait(self: *AutoResetEvent) void {

...	@@ -19,6 +19,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;	...	@@ -19,6 +19,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
19	const translate_c = @import("translate_c.zig");	19	const translate_c = @import("translate_c.zig");
20	const Cache = @import("Cache.zig");	20	const Cache = @import("Cache.zig");
21	const target_util = @import("target.zig");	21	const target_util = @import("target.zig");
		22	const ThreadPool = @import("ThreadPool.zig");
22		23
23	pub fn fatal(comptime format: []const u8, args: anytype) noreturn {	24	pub fn fatal(comptime format: []const u8, args: anytype) noreturn {
24	std.log.emerg(format, args);	25	std.log.emerg(format, args);
...	@@ -1632,6 +1633,10 @@ fn buildOutputType(	...	@@ -1632,6 +1633,10 @@ fn buildOutputType(
1632	};	1633	};
1633	defer zig_lib_directory.handle.close();	1634	defer zig_lib_directory.handle.close();
1634		1635
		1636	var thread_pool: ThreadPool = undefined;
		1637	try thread_pool.init(gpa);
		1638	defer thread_pool.deinit();
		1639
1635	var libc_installation: ?LibCInstallation = null;	1640	var libc_installation: ?LibCInstallation = null;
1636	defer if (libc_installation) \|*l\| l.deinit(gpa);	1641	defer if (libc_installation) \|*l\| l.deinit(gpa);
1637		1642
...	@@ -1747,6 +1752,7 @@ fn buildOutputType(	...	@@ -1747,6 +1752,7 @@ fn buildOutputType(
1747	.single_threaded = single_threaded,	1752	.single_threaded = single_threaded,
1748	.function_sections = function_sections,	1753	.function_sections = function_sections,
1749	.self_exe_path = self_exe_path,	1754	.self_exe_path = self_exe_path,
		1755	.thread_pool = &thread_pool,
1750	.clang_passthrough_mode = arg_mode != .build,	1756	.clang_passthrough_mode = arg_mode != .build,
1751	.clang_preprocessor_mode = clang_preprocessor_mode,	1757	.clang_preprocessor_mode = clang_preprocessor_mode,
1752	.version = optional_version,	1758	.version = optional_version,
...	@@ -2412,6 +2418,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !v	...	@@ -2412,6 +2418,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !v
2412	.directory = null, // Use the local zig-cache.	2418	.directory = null, // Use the local zig-cache.
2413	.basename = exe_basename,	2419	.basename = exe_basename,
2414	};	2420	};
		2421	var thread_pool: ThreadPool = undefined;
		2422	try thread_pool.init(gpa);
		2423	defer thread_pool.deinit();
2415	const comp = Compilation.create(gpa, .{	2424	const comp = Compilation.create(gpa, .{
2416	.zig_lib_directory = zig_lib_directory,	2425	.zig_lib_directory = zig_lib_directory,
2417	.local_cache_directory = local_cache_directory,	2426	.local_cache_directory = local_cache_directory,
...	@@ -2427,6 +2436,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !v	...	@@ -2427,6 +2436,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !v
2427	.emit_h = null,	2436	.emit_h = null,
2428	.optimize_mode = .Debug,	2437	.optimize_mode = .Debug,
2429	.self_exe_path = self_exe_path,	2438	.self_exe_path = self_exe_path,
		2439	.thread_pool = &thread_pool,
2430	}) catch \|err\| {	2440	}) catch \|err\| {
2431	fatal("unable to create compilation: {}", .{@errorName(err)});	2441	fatal("unable to create compilation: {}", .{@errorName(err)});
2432	};	2442	};

...	@@ -266,6 +266,7 @@ int main(int argc, char **argv) {	...	@@ -266,6 +266,7 @@ int main(int argc, char **argv) {
266	TargetSubsystem subsystem = TargetSubsystemAuto;	266	TargetSubsystem subsystem = TargetSubsystemAuto;
267	const char *override_lib_dir = nullptr;	267	const char *override_lib_dir = nullptr;
268	const char *mcpu = nullptr;	268	const char *mcpu = nullptr;
		269	bool single_threaded = false;
269		270
270	for (int i = 1; i < argc; i += 1) {	271	for (int i = 1; i < argc; i += 1) {
271	char *arg = argv[i];	272	char *arg = argv[i];
...	@@ -281,6 +282,8 @@ int main(int argc, char **argv) {	...	@@ -281,6 +282,8 @@ int main(int argc, char **argv) {
281	optimize_mode = BuildModeSafeRelease;	282	optimize_mode = BuildModeSafeRelease;
282	} else if (strcmp(arg, "-OReleaseSmall") == 0) {	283	} else if (strcmp(arg, "-OReleaseSmall") == 0) {
283	optimize_mode = BuildModeSmallRelease;	284	optimize_mode = BuildModeSmallRelease;
		285	} else if (strcmp(arg, "--single-threaded") == 0) {
		286	single_threaded = true;
284	} else if (strcmp(arg, "--help") == 0) {	287	} else if (strcmp(arg, "--help") == 0) {
285	return print_full_usage(arg0, stdout, EXIT_SUCCESS);	288	return print_full_usage(arg0, stdout, EXIT_SUCCESS);
286	} else if (strcmp(arg, "--strip") == 0) {	289	} else if (strcmp(arg, "--strip") == 0) {
...	@@ -469,6 +472,7 @@ int main(int argc, char **argv) {	...	@@ -469,6 +472,7 @@ int main(int argc, char **argv) {
469	stage1->link_libcpp = link_libcpp;	472	stage1->link_libcpp = link_libcpp;
470	stage1->subsystem = subsystem;	473	stage1->subsystem = subsystem;
471	stage1->pic = true;	474	stage1->pic = true;
		475	stage1->is_single_threaded = single_threaded;
472		476
473	zig_stage1_build_object(stage1);	477	zig_stage1_build_object(stage1);
474		478

Merge pull request #7462 from ziglang/parallel-c-objects