authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-06-20 19:44:07-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-06-30 13:54:02-07:00
loge51fd6728fcc4dd93e3bd1ddb2e4ca96a8082b12
tree07b916bb71e2f6f733575bb3396a52c2a58a6af0
parentc395df25aba0f1bdc1dd0cb6b9c7f14a90e72dae

new thread pool jobserver integration

std.Thread.Pool: back to spawning all threads in initialization because it's overall simpler. This scheme requires init to be passed a pointer to the struct. std.process.Child: implement integration with thread pool jobserver. The environment variable is called `JOBSERVERV2`. The API works based on assigning a thread pool to the child process. build runner: store the thread pool in std.Build.Graph so that it can be passed to child processes during the make phase. Fix not allocating +1 pollfds in previous commit.

11 files changed, 171 insertions(+), 85 deletions(-)

lib/compiler/build_runner.zig+9-5
...@@ -74,6 +74,7 @@ pub fn main() !void {...@@ -74,6 +74,7 @@ pub fn main() !void {
74 .query = .{},74 .query = .{},
75 .result = try std.zig.system.resolveTargetQuery(.{}),75 .result = try std.zig.system.resolveTargetQuery(.{}),
76 },76 },
77 .thread_pool = undefined,
77 };78 };
7879
79 graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });80 graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
...@@ -92,6 +93,7 @@ pub fn main() !void {...@@ -92,6 +93,7 @@ pub fn main() !void {
92 var targets = ArrayList([]const u8).init(arena);93 var targets = ArrayList([]const u8).init(arena);
93 var debug_log_scopes = ArrayList([]const u8).init(arena);94 var debug_log_scopes = ArrayList([]const u8).init(arena);
94 var thread_pool_options: std.zig.ThreadPoolOptions = .{95 var thread_pool_options: std.zig.ThreadPoolOptions = .{
96 .allocator = arena,
95 .cache_directory = local_cache_directory,97 .cache_directory = local_cache_directory,
96 };98 };
9799
...@@ -448,7 +450,8 @@ fn runStepNames(...@@ -448,7 +450,8 @@ fn runStepNames(
448 }450 }
449 }451 }
450452
451 var thread_pool = try std.zig.initThreadPool(gpa, thread_pool_options);453 const thread_pool = &b.graph.thread_pool;
454 try std.zig.initThreadPool(thread_pool, thread_pool_options);
452 defer thread_pool.deinit();455 defer thread_pool.deinit();
453456
454 {457 {
...@@ -469,7 +472,7 @@ fn runStepNames(...@@ -469,7 +472,7 @@ fn runStepNames(
469 if (step.state == .skipped_oom) continue;472 if (step.state == .skipped_oom) continue;
470473
471 thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{474 thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{
472 &wait_group, &thread_pool, b, step, step_prog, run,475 &wait_group, b, step, step_prog, run,
473 });476 });
474 }477 }
475 }478 }
...@@ -890,12 +893,13 @@ fn constructGraphAndCheckForDependencyLoop(...@@ -890,12 +893,13 @@ fn constructGraphAndCheckForDependencyLoop(
890893
891fn workerMakeOneStep(894fn workerMakeOneStep(
892 wg: *std.Thread.WaitGroup,895 wg: *std.Thread.WaitGroup,
893 thread_pool: *std.Thread.Pool,
894 b: *std.Build,896 b: *std.Build,
895 s: *Step,897 s: *Step,
896 prog_node: std.Progress.Node,898 prog_node: std.Progress.Node,
897 run: *Run,899 run: *Run,
898) void {900) void {
901 const thread_pool = &b.graph.thread_pool;
902
899 // First, check the conditions for running this step. If they are not met,903 // First, check the conditions for running this step. If they are not met,
900 // then we return without doing the step, relying on another worker to904 // then we return without doing the step, relying on another worker to
901 // queue this step up again when dependencies are met.905 // queue this step up again when dependencies are met.
...@@ -975,7 +979,7 @@ fn workerMakeOneStep(...@@ -975,7 +979,7 @@ fn workerMakeOneStep(
975 // Successful completion of a step, so we queue up its dependants as well.979 // Successful completion of a step, so we queue up its dependants as well.
976 for (s.dependants.items) |dep| {980 for (s.dependants.items) |dep| {
977 thread_pool.spawnWg(wg, workerMakeOneStep, .{981 thread_pool.spawnWg(wg, workerMakeOneStep, .{
978 wg, thread_pool, b, dep, prog_node, run,982 wg, b, dep, prog_node, run,
979 });983 });
980 }984 }
981 }985 }
...@@ -1000,7 +1004,7 @@ fn workerMakeOneStep(...@@ -1000,7 +1004,7 @@ fn workerMakeOneStep(
1000 remaining -= dep.max_rss;1004 remaining -= dep.max_rss;
10011005
1002 thread_pool.spawnWg(wg, workerMakeOneStep, .{1006 thread_pool.spawnWg(wg, workerMakeOneStep, .{
1003 wg, thread_pool, b, dep, prog_node, run,1007 wg, b, dep, prog_node, run,
1004 });1008 });
1005 } else {1009 } else {
1006 run.memory_blocked_steps.items[i] = dep;1010 run.memory_blocked_steps.items[i] = dep;
lib/std/Build.zig+2
...@@ -120,6 +120,8 @@ pub const Graph = struct {...@@ -120,6 +120,8 @@ pub const Graph = struct {
120 needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},120 needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},
121 /// Information about the native target. Computed before build() is invoked.121 /// Information about the native target. Computed before build() is invoked.
122 host: ResolvedTarget,122 host: ResolvedTarget,
123 /// Uninitialized until the make phase.
124 thread_pool: std.Thread.Pool,
123};125};
124126
125const AvailableDeps = []const struct { []const u8, []const u8 };127const AvailableDeps = []const struct { []const u8, []const u8 };
lib/std/Build/Step.zig+5-2
...@@ -283,15 +283,17 @@ pub fn captureChildProcess(...@@ -283,15 +283,17 @@ pub fn captureChildProcess(
283 progress_node: std.Progress.Node,283 progress_node: std.Progress.Node,
284 argv: []const []const u8,284 argv: []const []const u8,
285) !std.process.Child.RunResult {285) !std.process.Child.RunResult {
286 const arena = s.owner.allocator;286 const b = s.owner;
287 const arena = b.allocator;
287288
288 try handleChildProcUnsupported(s, null, argv);289 try handleChildProcUnsupported(s, null, argv);
289 try handleVerbose(s.owner, null, argv);290 try handleVerbose(b, null, argv);
290291
291 const result = std.process.Child.run(.{292 const result = std.process.Child.run(.{
292 .allocator = arena,293 .allocator = arena,
293 .argv = argv,294 .argv = argv,
294 .progress_node = progress_node,295 .progress_node = progress_node,
296 .thread_pool = &b.graph.thread_pool,
295 }) catch |err| return s.fail("unable to spawn {s}: {s}", .{ argv[0], @errorName(err) });297 }) catch |err| return s.fail("unable to spawn {s}: {s}", .{ argv[0], @errorName(err) });
296298
297 if (result.stderr.len > 0) {299 if (result.stderr.len > 0) {
...@@ -334,6 +336,7 @@ pub fn evalZigProcess(...@@ -334,6 +336,7 @@ pub fn evalZigProcess(
334 child.stderr_behavior = .Pipe;336 child.stderr_behavior = .Pipe;
335 child.request_resource_usage_statistics = true;337 child.request_resource_usage_statistics = true;
336 child.progress_node = prog_node;338 child.progress_node = prog_node;
339 child.thread_pool = &b.graph.thread_pool;
337340
338 child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{341 child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{
339 argv[0], @errorName(err),342 argv[0], @errorName(err),
lib/std/Build/Step/Run.zig+1
...@@ -1246,6 +1246,7 @@ fn spawnChildAndCollect(...@@ -1246,6 +1246,7 @@ fn spawnChildAndCollect(
1246 if (run.stdio != .zig_test and !run.disable_zig_progress and !inherit) {1246 if (run.stdio != .zig_test and !run.disable_zig_progress and !inherit) {
1247 child.progress_node = prog_node;1247 child.progress_node = prog_node;
1248 }1248 }
1249 child.thread_pool = &b.graph.thread_pool;
12491250
1250 const term, const result, const elapsed_ns = t: {1251 const term, const result, const elapsed_ns = t: {
1251 if (inherit) std.debug.lockStdErr();1252 if (inherit) std.debug.lockStdErr();
lib/std/Thread/Pool.zig+26-60
...@@ -7,11 +7,9 @@ const assert = std.debug.assert;...@@ -7,11 +7,9 @@ const assert = std.debug.assert;
7mutex: std.Thread.Mutex,7mutex: std.Thread.Mutex,
8cond: std.Thread.Condition,8cond: std.Thread.Condition,
9run_queue: RunQueue,9run_queue: RunQueue,
10run_queue_len: usize,
11end_flag: bool,10end_flag: bool,
12allocator: std.mem.Allocator,11allocator: std.mem.Allocator,
13threads_buffer: []std.Thread,12threads: []std.Thread,
14threads_len: usize,
15job_server_options: Options.JobServer,13job_server_options: Options.JobServer,
16job_server: ?*JobServer,14job_server: ?*JobServer,
1715
...@@ -23,6 +21,9 @@ const Runnable = struct {...@@ -23,6 +21,9 @@ const Runnable = struct {
23const RunProto = *const fn (*Runnable) void;21const RunProto = *const fn (*Runnable) void;
2422
25pub const Options = struct {23pub const Options = struct {
24 /// Not required to be thread-safe; protected by the pool's mutex.
25 allocator: std.mem.Allocator,
26
26 /// Max number of threads to be actively working at the same time.27 /// Max number of threads to be actively working at the same time.
27 ///28 ///
28 /// `null` means to use the logical core count, leaving the main thread to29 /// `null` means to use the logical core count, leaving the main thread to
...@@ -49,22 +50,16 @@ pub const Options = struct {...@@ -49,22 +50,16 @@ pub const Options = struct {
49 };50 };
50};51};
5152
52/// After initializing the thread pool and spawning work, the main thread must53pub fn init(pool: *Pool, options: Options) !void {
53/// call `waitAndWork`.54 const allocator = options.allocator;
54pub fn init(55
55 /// Not required to be thread-safe; protected by the pool's mutex.56 pool.* = .{
56 allocator: std.mem.Allocator,
57 options: Options,
58) !Pool {
59 var pool: Pool = .{
60 .mutex = .{},57 .mutex = .{},
61 .cond = .{},58 .cond = .{},
62 .run_queue = .{},59 .run_queue = .{},
63 .run_queue_len = 0,
64 .end_flag = false,60 .end_flag = false,
65 .allocator = allocator,61 .allocator = allocator,
66 .threads_buffer = &.{},62 .threads = &.{},
67 .threads_len = 0,
68 .job_server_options = options.job_server,63 .job_server_options = options.job_server,
69 .job_server = null,64 .job_server = null,
70 };65 };
...@@ -75,8 +70,15 @@ pub fn init(...@@ -75,8 +70,15 @@ pub fn init(
75 const thread_count = options.n_jobs orelse @max(1, std.Thread.getCpuCount() catch 1);70 const thread_count = options.n_jobs orelse @max(1, std.Thread.getCpuCount() catch 1);
76 assert(thread_count > 0);71 assert(thread_count > 0);
7772
78 pool.threads_buffer = try allocator.alloc(std.Thread, thread_count);73 // Kill and join any threads we spawned and free memory on error.
79 errdefer allocator.free(pool.threads_buffer);74 pool.threads = try allocator.alloc(std.Thread, thread_count);
75 var spawned: usize = 0;
76 errdefer pool.join(spawned);
77
78 for (pool.threads) |*thread| {
79 thread.* = try std.Thread.spawn(.{}, worker, .{pool});
80 spawned += 1;
81 }
8082
81 switch (options.job_server) {83 switch (options.job_server) {
82 .abstain, .connect => {},84 .abstain, .connect => {},
...@@ -84,7 +86,7 @@ pub fn init(...@@ -84,7 +86,7 @@ pub fn init(
84 var server = try addr.listen(.{});86 var server = try addr.listen(.{});
85 errdefer server.deinit();87 errdefer server.deinit();
8688
87 const pollfds = try allocator.alloc(std.posix.pollfd, thread_count);89 const pollfds = try allocator.alloc(std.posix.pollfd, thread_count + 1);
88 errdefer allocator.free(pollfds);90 errdefer allocator.free(pollfds);
8991
90 const job_server = try allocator.create(JobServer);92 const job_server = try allocator.create(JobServer);
...@@ -99,11 +101,14 @@ pub fn init(...@@ -99,11 +101,14 @@ pub fn init(
99 pool.job_server = job_server;101 pool.job_server = job_server;
100 },102 },
101 }103 }
102
103 return pool;
104}104}
105105
106pub fn deinit(pool: *Pool) void {106pub fn deinit(pool: *Pool) void {
107 pool.join(pool.threads.len);
108 pool.* = undefined;
109}
110
111fn join(pool: *Pool, spawned: usize) void {
107 if (builtin.single_threaded)112 if (builtin.single_threaded)
108 return;113 return;
109114
...@@ -127,15 +132,10 @@ pub fn deinit(pool: *Pool) void {...@@ -127,15 +132,10 @@ pub fn deinit(pool: *Pool) void {
127 job_server.thread.join();132 job_server.thread.join();
128 }133 }
129134
130 // Since we set end_flag with the mutex locked, no more threads could have135 for (pool.threads[0..spawned]) |thread|
131 // been created.
132 const threads = pool.threads_buffer[0..pool.threads_len];
133
134 for (threads) |thread|
135 thread.join();136 thread.join();
136137
137 pool.allocator.free(pool.threads_buffer);138 pool.allocator.free(pool.threads);
138 pool.* = undefined;
139}139}
140140
141pub const JobServer = struct {141pub const JobServer = struct {
...@@ -256,22 +256,6 @@ pub fn spawnWg(pool: *Pool, wait_group: *WaitGroup, comptime func: anytype, args...@@ -256,22 +256,6 @@ pub fn spawnWg(pool: *Pool, wait_group: *WaitGroup, comptime func: anytype, args
256 };256 };
257257
258 pool.run_queue.prepend(&closure.run_node);258 pool.run_queue.prepend(&closure.run_node);
259 pool.run_queue_len += 1;
260
261 // If there was already any queued work, spawn a new thread if we are
262 // under the max.
263 if (pool.run_queue_len > 1 and pool.threads_len < pool.threads_buffer.len) {
264 if (std.Thread.spawn(.{}, worker, .{pool})) |new_thread| {
265 pool.threads_buffer[pool.threads_len] = new_thread;
266 pool.threads_len += 1;
267 } else |_| if (pool.threads_len == 0) {
268 pool.mutex.unlock();
269 @call(.auto, func, args);
270 wait_group.finish();
271 return;
272 }
273 }
274
275 pool.mutex.unlock();259 pool.mutex.unlock();
276 }260 }
277261
...@@ -319,21 +303,6 @@ pub fn spawn(pool: *Pool, comptime func: anytype, args: anytype) void {...@@ -319,21 +303,6 @@ pub fn spawn(pool: *Pool, comptime func: anytype, args: anytype) void {
319 };303 };
320304
321 pool.run_queue.prepend(&closure.run_node);305 pool.run_queue.prepend(&closure.run_node);
322 pool.run_queue_len += 1;
323
324 // If there was already any queued work, spawn a new thread if we are
325 // under the max.
326 if (pool.run_queue_len > 1 and pool.threads_len < pool.threads_buffer.len) {
327 if (std.Thread.spawn(.{}, worker, .{pool})) |new_thread| {
328 pool.threads_buffer[pool.threads_len] = new_thread;
329 pool.threads_len += 1;
330 } else |_| if (pool.threads_len == 0) {
331 pool.mutex.unlock();
332 @call(.auto, func, args);
333 return;
334 }
335 }
336
337 pool.mutex.unlock();306 pool.mutex.unlock();
338 }307 }
339308
...@@ -351,8 +320,6 @@ fn worker(pool: *Pool) void {...@@ -351,8 +320,6 @@ fn worker(pool: *Pool) void {
351320
352 while (true) {321 while (true) {
353 while (pool.run_queue.popFirst()) |run_node| {322 while (pool.run_queue.popFirst()) |run_node| {
354 pool.run_queue_len -= 1;
355
356 // Temporarily unlock the mutex in order to execute the run_node.323 // Temporarily unlock the mutex in order to execute the run_node.
357 pool.mutex.unlock();324 pool.mutex.unlock();
358 defer pool.mutex.lock();325 defer pool.mutex.lock();
...@@ -391,7 +358,6 @@ pub fn waitAndWork(pool: *Pool, wait_group: *WaitGroup) void {...@@ -391,7 +358,6 @@ pub fn waitAndWork(pool: *Pool, wait_group: *WaitGroup) void {
391 defer pool.mutex.unlock();358 defer pool.mutex.unlock();
392 break :blk pool.run_queue.popFirst();359 break :blk pool.run_queue.popFirst();
393 }) |run_node| {360 }) |run_node| {
394 pool.run_queue_len -= 1;
395 run_node.data.runFn(&run_node.data);361 run_node.data.runFn(&run_node.data);
396 continue;362 continue;
397 }363 }
lib/std/process.zig+71-8
...@@ -1816,6 +1816,14 @@ pub const CreateEnvironOptions = struct {...@@ -1816,6 +1816,14 @@ pub const CreateEnvironOptions = struct {
1816 /// If non-null, negative means to remove the environment variable, and >= 01816 /// If non-null, negative means to remove the environment variable, and >= 0
1817 /// means to provide it with the given integer.1817 /// means to provide it with the given integer.
1818 zig_progress_fd: ?i32 = null,1818 zig_progress_fd: ?i32 = null,
1819
1820 job_server_path: String = .unchanged,
1821
1822 pub const String = union(enum) {
1823 unchanged,
1824 deleted,
1825 updated: []const u8,
1826 };
1819};1827};
18201828
1821/// Creates a null-deliminated environment variable block in the format1829/// Creates a null-deliminated environment variable block in the format
...@@ -1825,8 +1833,8 @@ pub fn createEnvironFromMap(...@@ -1825,8 +1833,8 @@ pub fn createEnvironFromMap(
1825 map: *const EnvMap,1833 map: *const EnvMap,
1826 options: CreateEnvironOptions,1834 options: CreateEnvironOptions,
1827) Allocator.Error![:null]?[*:0]u8 {1835) Allocator.Error![:null]?[*:0]u8 {
1828 const ZigProgressAction = enum { nothing, edit, delete, add };1836 const EnvVarAction = enum { nothing, edit, delete, add };
1829 const zig_progress_action: ZigProgressAction = a: {1837 const zig_progress_action: EnvVarAction = a: {
1830 const fd = options.zig_progress_fd orelse break :a .nothing;1838 const fd = options.zig_progress_fd orelse break :a .nothing;
1831 const contains = map.get("ZIG_PROGRESS") != null;1839 const contains = map.get("ZIG_PROGRESS") != null;
1832 if (fd >= 0) {1840 if (fd >= 0) {
...@@ -1836,6 +1844,11 @@ pub fn createEnvironFromMap(...@@ -1836,6 +1844,11 @@ pub fn createEnvironFromMap(
1836 }1844 }
1837 break :a .nothing;1845 break :a .nothing;
1838 };1846 };
1847 const job_server_action: EnvVarAction = switch (options.job_server_path) {
1848 .unchanged => .nothing,
1849 .deleted => if (map.get("JOBSERVERV2") != null) .delete else .nothing,
1850 .updated => if (map.get("JOBSERVERV2") != null) .edit else .add,
1851 };
18391852
1840 const envp_count: usize = c: {1853 const envp_count: usize = c: {
1841 var count: usize = map.count();1854 var count: usize = map.count();
...@@ -1844,6 +1857,11 @@ pub fn createEnvironFromMap(...@@ -1844,6 +1857,11 @@ pub fn createEnvironFromMap(
1844 .delete => count -= 1,1857 .delete => count -= 1,
1845 .nothing, .edit => {},1858 .nothing, .edit => {},
1846 }1859 }
1860 switch (job_server_action) {
1861 .add => count += 1,
1862 .delete => count -= 1,
1863 .nothing, .edit => {},
1864 }
1847 break :c count;1865 break :c count;
1848 };1866 };
18491867
...@@ -1855,6 +1873,11 @@ pub fn createEnvironFromMap(...@@ -1855,6 +1873,11 @@ pub fn createEnvironFromMap(
1855 i += 1;1873 i += 1;
1856 }1874 }
18571875
1876 if (job_server_action == .add) {
1877 envp_buf[i] = try std.fmt.allocPrintZ(arena, "JOBSERVERV2={s}", .{options.job_server_path.updated});
1878 i += 1;
1879 }
1880
1858 {1881 {
1859 var it = map.iterator();1882 var it = map.iterator();
1860 while (it.next()) |pair| {1883 while (it.next()) |pair| {
...@@ -1871,6 +1894,19 @@ pub fn createEnvironFromMap(...@@ -1871,6 +1894,19 @@ pub fn createEnvironFromMap(
1871 .nothing => {},1894 .nothing => {},
1872 };1895 };
18731896
1897 if (mem.eql(u8, pair.key_ptr.*, "JOBSERVERV2")) switch (job_server_action) {
1898 .add => unreachable,
1899 .delete => continue,
1900 .edit => {
1901 envp_buf[i] = try std.fmt.allocPrintZ(arena, "{s}={s}", .{
1902 pair.key_ptr.*, options.job_server_path.updated,
1903 });
1904 i += 1;
1905 continue;
1906 },
1907 .nothing => {},
1908 };
1909
1874 envp_buf[i] = try std.fmt.allocPrintZ(arena, "{s}={s}", .{ pair.key_ptr.*, pair.value_ptr.* });1910 envp_buf[i] = try std.fmt.allocPrintZ(arena, "{s}={s}", .{ pair.key_ptr.*, pair.value_ptr.* });
1875 i += 1;1911 i += 1;
1876 }1912 }
...@@ -1887,16 +1923,19 @@ pub fn createEnvironFromExisting(...@@ -1887,16 +1923,19 @@ pub fn createEnvironFromExisting(
1887 existing: [*:null]const ?[*:0]const u8,1923 existing: [*:null]const ?[*:0]const u8,
1888 options: CreateEnvironOptions,1924 options: CreateEnvironOptions,
1889) Allocator.Error![:null]?[*:0]u8 {1925) Allocator.Error![:null]?[*:0]u8 {
1890 const existing_count, const contains_zig_progress = c: {1926 const existing_count, const contains_zig_progress, const contains_job_server = c: {
1891 var count: usize = 0;1927 var count: usize = 0;
1892 var contains = false;1928 var contains_zig_progress = false;
1929 var contains_job_server = false;
1893 while (existing[count]) |line| : (count += 1) {1930 while (existing[count]) |line| : (count += 1) {
1894 contains = contains or mem.eql(u8, mem.sliceTo(line, '='), "ZIG_PROGRESS");1931 const name = mem.sliceTo(line, '=');
1932 contains_zig_progress = contains_zig_progress or mem.eql(u8, name, "ZIG_PROGRESS");
1933 contains_job_server = contains_job_server or mem.eql(u8, name, "JOBSERVERV2");
1895 }1934 }
1896 break :c .{ count, contains };1935 break :c .{ count, contains_zig_progress, contains_job_server };
1897 };1936 };
1898 const ZigProgressAction = enum { nothing, edit, delete, add };1937 const EnvVarAction = enum { nothing, edit, delete, add };
1899 const zig_progress_action: ZigProgressAction = a: {1938 const zig_progress_action: EnvVarAction = a: {
1900 const fd = options.zig_progress_fd orelse break :a .nothing;1939 const fd = options.zig_progress_fd orelse break :a .nothing;
1901 if (fd >= 0) {1940 if (fd >= 0) {
1902 break :a if (contains_zig_progress) .edit else .add;1941 break :a if (contains_zig_progress) .edit else .add;
...@@ -1905,6 +1944,11 @@ pub fn createEnvironFromExisting(...@@ -1905,6 +1944,11 @@ pub fn createEnvironFromExisting(
1905 }1944 }
1906 break :a .nothing;1945 break :a .nothing;
1907 };1946 };
1947 const job_server_action: EnvVarAction = switch (options.job_server_path) {
1948 .unchanged => .nothing,
1949 .deleted => if (contains_job_server) .delete else .nothing,
1950 .updated => if (contains_job_server) .edit else .add,
1951 };
19081952
1909 const envp_count: usize = c: {1953 const envp_count: usize = c: {
1910 var count: usize = existing_count;1954 var count: usize = existing_count;
...@@ -1913,6 +1957,11 @@ pub fn createEnvironFromExisting(...@@ -1913,6 +1957,11 @@ pub fn createEnvironFromExisting(
1913 .delete => count -= 1,1957 .delete => count -= 1,
1914 .nothing, .edit => {},1958 .nothing, .edit => {},
1915 }1959 }
1960 switch (job_server_action) {
1961 .add => count += 1,
1962 .delete => count -= 1,
1963 .nothing, .edit => {},
1964 }
1916 break :c count;1965 break :c count;
1917 };1966 };
19181967
...@@ -1924,6 +1973,10 @@ pub fn createEnvironFromExisting(...@@ -1924,6 +1973,10 @@ pub fn createEnvironFromExisting(
1924 envp_buf[i] = try std.fmt.allocPrintZ(arena, "ZIG_PROGRESS={d}", .{options.zig_progress_fd.?});1973 envp_buf[i] = try std.fmt.allocPrintZ(arena, "ZIG_PROGRESS={d}", .{options.zig_progress_fd.?});
1925 i += 1;1974 i += 1;
1926 }1975 }
1976 if (job_server_action == .add) {
1977 envp_buf[i] = try std.fmt.allocPrintZ(arena, "JOBSERVERV2={s}", .{options.job_server_path.updated});
1978 i += 1;
1979 }
19271980
1928 while (existing[existing_index]) |line| : (existing_index += 1) {1981 while (existing[existing_index]) |line| : (existing_index += 1) {
1929 if (mem.eql(u8, mem.sliceTo(line, '='), "ZIG_PROGRESS")) switch (zig_progress_action) {1982 if (mem.eql(u8, mem.sliceTo(line, '='), "ZIG_PROGRESS")) switch (zig_progress_action) {
...@@ -1936,6 +1989,16 @@ pub fn createEnvironFromExisting(...@@ -1936,6 +1989,16 @@ pub fn createEnvironFromExisting(
1936 },1989 },
1937 .nothing => {},1990 .nothing => {},
1938 };1991 };
1992 if (mem.eql(u8, mem.sliceTo(line, '='), "JOBSERVERV2")) switch (job_server_action) {
1993 .add => unreachable,
1994 .delete => continue,
1995 .edit => {
1996 envp_buf[i] = try std.fmt.allocPrintZ(arena, "JOBSERVERV2={s}", .{options.job_server_path.updated});
1997 i += 1;
1998 continue;
1999 },
2000 .nothing => {},
2001 };
1939 envp_buf[i] = try arena.dupeZ(u8, mem.span(line));2002 envp_buf[i] = try arena.dupeZ(u8, mem.span(line));
1940 i += 1;2003 i += 1;
1941 }2004 }
lib/std/process/Child.zig+28
...@@ -103,6 +103,25 @@ resource_usage_statistics: ResourceUsageStatistics = .{},...@@ -103,6 +103,25 @@ resource_usage_statistics: ResourceUsageStatistics = .{},
103/// by substituting this node with the child's root node.103/// by substituting this node with the child's root node.
104progress_node: std.Progress.Node = std.Progress.Node.none,104progress_node: std.Progress.Node = std.Progress.Node.none,
105105
106/// When provided, ensures that the child process will have access to the
107/// jobserver provided by the thread pool.
108///
109/// If the thread pool represents the root process, the child process will be
110/// supplied with the `JOBSERVERV2` environment variable so that it can
111/// connect.
112///
113/// If the thread pool represents a client, its connection address will be
114/// passed into the `JOBSERVERV2` environment variable. This potentially
115/// overrides the global environment variable.
116///
117/// If the thread pool is in abstinance mode, any `JOBSERVERV2` environment
118/// variable will be elided from being passed down to the child. This differs
119/// from leaving the field as `null` in which case no modifications to
120/// jobserver environment variables will occur.
121///
122/// A provided thread pool must live longer than this `Child` instance.
123thread_pool: ?*std.Thread.Pool = null,
124
106pub const ResourceUsageStatistics = struct {125pub const ResourceUsageStatistics = struct {
107 rusage: @TypeOf(rusage_init) = rusage_init,126 rusage: @TypeOf(rusage_init) = rusage_init,
108127
...@@ -377,6 +396,7 @@ pub fn run(args: struct {...@@ -377,6 +396,7 @@ pub fn run(args: struct {
377 max_output_bytes: usize = 50 * 1024,396 max_output_bytes: usize = 50 * 1024,
378 expand_arg0: Arg0Expand = .no_expand,397 expand_arg0: Arg0Expand = .no_expand,
379 progress_node: std.Progress.Node = std.Progress.Node.none,398 progress_node: std.Progress.Node = std.Progress.Node.none,
399 thread_pool: ?*std.Thread.Pool = null,
380}) RunError!RunResult {400}) RunError!RunResult {
381 var child = ChildProcess.init(args.argv, args.allocator);401 var child = ChildProcess.init(args.argv, args.allocator);
382 child.stdin_behavior = .Ignore;402 child.stdin_behavior = .Ignore;
...@@ -387,6 +407,7 @@ pub fn run(args: struct {...@@ -387,6 +407,7 @@ pub fn run(args: struct {
387 child.env_map = args.env_map;407 child.env_map = args.env_map;
388 child.expand_arg0 = args.expand_arg0;408 child.expand_arg0 = args.expand_arg0;
389 child.progress_node = args.progress_node;409 child.progress_node = args.progress_node;
410 child.thread_pool = args.thread_pool;
390411
391 var stdout = std.ArrayList(u8).init(args.allocator);412 var stdout = std.ArrayList(u8).init(args.allocator);
392 var stderr = std.ArrayList(u8).init(args.allocator);413 var stderr = std.ArrayList(u8).init(args.allocator);
...@@ -616,19 +637,26 @@ fn spawnPosix(self: *ChildProcess) SpawnError!void {...@@ -616,19 +637,26 @@ fn spawnPosix(self: *ChildProcess) SpawnError!void {
616637
617 const envp: [*:null]const ?[*:0]const u8 = m: {638 const envp: [*:null]const ?[*:0]const u8 = m: {
618 const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;639 const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
640 const job_server_path: process.CreateEnvironOptions.String = if (self.thread_pool) |thread_pool| switch (thread_pool.job_server_options) {
641 .host, .connect => |addr| .{ .updated = mem.sliceTo(&addr.un.path, 0) },
642 .abstain => .deleted,
643 } else .unchanged;
619 if (self.env_map) |env_map| {644 if (self.env_map) |env_map| {
620 break :m (try process.createEnvironFromMap(arena, env_map, .{645 break :m (try process.createEnvironFromMap(arena, env_map, .{
621 .zig_progress_fd = prog_fd,646 .zig_progress_fd = prog_fd,
647 .job_server_path = job_server_path,
622 })).ptr;648 })).ptr;
623 } else if (builtin.link_libc) {649 } else if (builtin.link_libc) {
624 break :m (try process.createEnvironFromExisting(arena, std.c.environ, .{650 break :m (try process.createEnvironFromExisting(arena, std.c.environ, .{
625 .zig_progress_fd = prog_fd,651 .zig_progress_fd = prog_fd,
652 .job_server_path = job_server_path,
626 })).ptr;653 })).ptr;
627 } else if (builtin.output_mode == .Exe) {654 } else if (builtin.output_mode == .Exe) {
628 // Then we have Zig start code and this works.655 // Then we have Zig start code and this works.
629 // TODO type-safety for null-termination of `os.environ`.656 // TODO type-safety for null-termination of `os.environ`.
630 break :m (try process.createEnvironFromExisting(arena, @ptrCast(std.os.environ.ptr), .{657 break :m (try process.createEnvironFromExisting(arena, @ptrCast(std.os.environ.ptr), .{
631 .zig_progress_fd = prog_fd,658 .zig_progress_fd = prog_fd,
659 .job_server_path = job_server_path,
632 })).ptr;660 })).ptr;
633 } else {661 } else {
634 // TODO come up with a solution for this.662 // TODO come up with a solution for this.
lib/std/zig.zig+10-6
...@@ -689,7 +689,7 @@ pub const EnvVar = enum {...@@ -689,7 +689,7 @@ pub const EnvVar = enum {
689 CLICOLOR_FORCE,689 CLICOLOR_FORCE,
690 XDG_CACHE_HOME,690 XDG_CACHE_HOME,
691 HOME,691 HOME,
692 JOBSERVER2,692 JOBSERVERV2,
693693
694 pub fn isSet(comptime ev: EnvVar) bool {694 pub fn isSet(comptime ev: EnvVar) bool {
695 return std.process.hasEnvVarConstant(@tagName(ev));695 return std.process.hasEnvVarConstant(@tagName(ev));
...@@ -710,15 +710,17 @@ pub const EnvVar = enum {...@@ -710,15 +710,17 @@ pub const EnvVar = enum {
710};710};
711711
712pub const ThreadPoolOptions = struct {712pub const ThreadPoolOptions = struct {
713 allocator: Allocator,
713 n_jobs: ?u32 = null,714 n_jobs: ?u32 = null,
714 cache_directory: std.Build.Cache.Directory,715 cache_directory: std.Build.Cache.Directory,
715};716};
716717
717pub const cache_tmp_basename = "tmp";718pub const cache_tmp_basename = "tmp";
718719
719pub fn initThreadPool(gpa: Allocator, options: ThreadPoolOptions) !std.Thread.Pool {720pub fn initThreadPool(thread_pool: *std.Thread.Pool, options: ThreadPoolOptions) !void {
720 if (EnvVar.JOBSERVER2.getPosix()) |addr_string| {721 if (EnvVar.JOBSERVERV2.getPosix()) |addr_string| {
721 return std.Thread.Pool.init(gpa, .{722 return std.Thread.Pool.init(thread_pool, .{
723 .allocator = options.allocator,
722 .n_jobs = options.n_jobs,724 .n_jobs = options.n_jobs,
723 .job_server = .{ .connect = try std.net.Address.initUnix(addr_string) },725 .job_server = .{ .connect = try std.net.Address.initUnix(addr_string) },
724 });726 });
...@@ -744,13 +746,15 @@ pub fn initThreadPool(gpa: Allocator, options: ThreadPoolOptions) !std.Thread.Po...@@ -744,13 +746,15 @@ pub fn initThreadPool(gpa: Allocator, options: ThreadPoolOptions) !std.Thread.Po
744 @memcpy(addr.un.path[0..cache_dir.len], cache_dir);746 @memcpy(addr.un.path[0..cache_dir.len], cache_dir);
745 @memcpy(addr.un.path[cache_dir.len..][0..suffix.len], suffix);747 @memcpy(addr.un.path[cache_dir.len..][0..suffix.len], suffix);
746748
747 return std.Thread.Pool.init(gpa, .{749 return std.Thread.Pool.init(thread_pool, .{
750 .allocator = options.allocator,
748 .n_jobs = options.n_jobs,751 .n_jobs = options.n_jobs,
749 .job_server = .{ .host = addr },752 .job_server = .{ .host = addr },
750 }) catch |err| switch (err) {753 }) catch |err| switch (err) {
751 error.FileNotFound => {754 error.FileNotFound => {
752 try options.cache_directory.handle.makePath(cache_tmp_basename);755 try options.cache_directory.handle.makePath(cache_tmp_basename);
753 return std.Thread.Pool.init(gpa, .{756 return std.Thread.Pool.init(thread_pool, .{
757 .allocator = options.allocator,
754 .n_jobs = options.n_jobs,758 .n_jobs = options.n_jobs,
755 .job_server = .{ .host = addr },759 .job_server = .{ .host = addr },
756 });760 });
src/Compilation.zig+2
...@@ -4604,6 +4604,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr...@@ -4604,6 +4604,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr
4604 };4604 };
4605 if (std.process.can_spawn) {4605 if (std.process.can_spawn) {
4606 var child = std.process.Child.init(argv.items, arena);4606 var child = std.process.Child.init(argv.items, arena);
4607 child.thread_pool = comp.thread_pool;
4607 if (comp.clang_passthrough_mode) {4608 if (comp.clang_passthrough_mode) {
4608 child.stdin_behavior = .Inherit;4609 child.stdin_behavior = .Inherit;
4609 child.stdout_behavior = .Inherit;4610 child.stdout_behavior = .Inherit;
...@@ -4964,6 +4965,7 @@ fn spawnZigRc(...@@ -4964,6 +4965,7 @@ fn spawnZigRc(
4964 child.stdout_behavior = .Pipe;4965 child.stdout_behavior = .Pipe;
4965 child.stderr_behavior = .Pipe;4966 child.stderr_behavior = .Pipe;
4966 child.progress_node = child_progress_node;4967 child.progress_node = child_progress_node;
4968 child.thread_pool = comp.thread_pool;
49674969
4968 child.spawn() catch |err| {4970 child.spawn() catch |err| {
4969 return comp.failWin32Resource(win32_resource, "unable to spawn {s} rc: {s}", .{ argv[0], @errorName(err) });4971 return comp.failWin32Resource(win32_resource, "unable to spawn {s} rc: {s}", .{ argv[0], @errorName(err) });
src/link.zig+1
...@@ -1011,6 +1011,7 @@ pub fn spawnLld(...@@ -1011,6 +1011,7 @@ pub fn spawnLld(
1011 defer comp.gpa.free(stderr);1011 defer comp.gpa.free(stderr);
10121012
1013 var child = std.process.Child.init(argv, arena);1013 var child = std.process.Child.init(argv, arena);
1014 child.thread_pool = comp.thread_pool;
1014 const term = (if (comp.clang_passthrough_mode) term: {1015 const term = (if (comp.clang_passthrough_mode) term: {
1015 child.stdin_behavior = .Inherit;1016 child.stdin_behavior = .Inherit;
1016 child.stdout_behavior = .Inherit;1017 child.stdout_behavior = .Inherit;
src/main.zig+16-4
...@@ -3136,7 +3136,9 @@ fn buildOutputType(...@@ -3136,7 +3136,9 @@ fn buildOutputType(
3136 break :l global_cache_directory;3136 break :l global_cache_directory;
3137 };3137 };
31383138
3139 var thread_pool = try std.zig.initThreadPool(gpa, .{3139 var thread_pool: std.Thread.Pool = undefined;
3140 try std.zig.initThreadPool(&thread_pool, .{
3141 .allocator = gpa,
3140 .cache_directory = local_cache_directory,3142 .cache_directory = local_cache_directory,
3141 });3143 });
3142 defer thread_pool.deinit();3144 defer thread_pool.deinit();
...@@ -4250,6 +4252,7 @@ fn runOrTest(...@@ -4250,6 +4252,7 @@ fn runOrTest(
4250 child.stdin_behavior = .Inherit;4252 child.stdin_behavior = .Inherit;
4251 child.stdout_behavior = .Inherit;4253 child.stdout_behavior = .Inherit;
4252 child.stderr_behavior = .Inherit;4254 child.stderr_behavior = .Inherit;
4255 child.thread_pool = comp.thread_pool;
42534256
4254 // Here we release all the locks associated with the Compilation so4257 // Here we release all the locks associated with the Compilation so
4255 // that whatever this child process wants to do won't deadlock.4258 // that whatever this child process wants to do won't deadlock.
...@@ -4395,6 +4398,7 @@ fn runOrTestHotSwap(...@@ -4395,6 +4398,7 @@ fn runOrTestHotSwap(
4395 child.stdin_behavior = .Inherit;4398 child.stdin_behavior = .Inherit;
4396 child.stdout_behavior = .Inherit;4399 child.stdout_behavior = .Inherit;
4397 child.stderr_behavior = .Inherit;4400 child.stderr_behavior = .Inherit;
4401 child.thread_pool = comp.thread_pool;
43984402
4399 try child.spawn();4403 try child.spawn();
44004404
...@@ -4896,7 +4900,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -4896,7 +4900,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
48964900
4897 child_argv.items[argv_index_cache_dir] = local_cache_directory.path orelse cwd_path;4901 child_argv.items[argv_index_cache_dir] = local_cache_directory.path orelse cwd_path;
48984902
4899 var thread_pool = try std.zig.initThreadPool(gpa, .{4903 var thread_pool: std.Thread.Pool = undefined;
4904 try std.zig.initThreadPool(&thread_pool, .{
4905 .allocator = gpa,
4900 .cache_directory = local_cache_directory,4906 .cache_directory = local_cache_directory,
4901 });4907 });
4902 defer thread_pool.deinit();4908 defer thread_pool.deinit();
...@@ -5185,6 +5191,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5185,6 +5191,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5185 child.stdin_behavior = .Inherit;5191 child.stdin_behavior = .Inherit;
5186 child.stdout_behavior = .Inherit;5192 child.stdout_behavior = .Inherit;
5187 child.stderr_behavior = .Inherit;5193 child.stderr_behavior = .Inherit;
5194 child.thread_pool = &thread_pool;
51885195
5189 const term = t: {5196 const term = t: {
5190 std.debug.lockStdErr();5197 std.debug.lockStdErr();
...@@ -5331,7 +5338,9 @@ fn jitCmd(...@@ -5331,7 +5338,9 @@ fn jitCmd(
5331 };5338 };
5332 defer global_cache_directory.handle.close();5339 defer global_cache_directory.handle.close();
53335340
5334 var thread_pool = try std.zig.initThreadPool(gpa, .{5341 var thread_pool: std.Thread.Pool = undefined;
5342 try std.zig.initThreadPool(&thread_pool, .{
5343 .allocator = gpa,
5335 .cache_directory = global_cache_directory,5344 .cache_directory = global_cache_directory,
5336 });5345 });
5337 defer thread_pool.deinit();5346 defer thread_pool.deinit();
...@@ -5474,6 +5483,7 @@ fn jitCmd(...@@ -5474,6 +5483,7 @@ fn jitCmd(
5474 child.stdin_behavior = .Inherit;5483 child.stdin_behavior = .Inherit;
5475 child.stdout_behavior = if (options.capture == null) .Inherit else .Pipe;5484 child.stdout_behavior = if (options.capture == null) .Inherit else .Pipe;
5476 child.stderr_behavior = .Inherit;5485 child.stderr_behavior = .Inherit;
5486 child.thread_pool = &thread_pool;
54775487
5478 try child.spawn();5488 try child.spawn();
54795489
...@@ -6897,7 +6907,9 @@ fn cmdFetch(...@@ -6897,7 +6907,9 @@ fn cmdFetch(
6897 };6907 };
6898 defer global_cache_directory.handle.close();6908 defer global_cache_directory.handle.close();
68996909
6900 var thread_pool = try std.zig.initThreadPool(gpa, .{6910 var thread_pool: std.Thread.Pool = undefined;
6911 try std.zig.initThreadPool(&thread_pool, .{
6912 .allocator = gpa,
6901 .cache_directory = global_cache_directory,6913 .cache_directory = global_cache_directory,
6902 });6914 });
6903 defer thread_pool.deinit();6915 defer thread_pool.deinit();