authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-11-25 03:29:19-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-11-25 09:40:27-08:00
log0f1a6ae943367d0e6042c9e81cb860c96a84ecd8
treea1030b359616847060287b1aa8af607966b0bb68
parentd0ba6642b5407a904ff0bec5b8f05ab65652f428

fetch: update from std.Thread.Pool to std.Io

2 files changed, 33 insertions(+), 41 deletions(-)

src/Package/Fetch.zig+28-32
...@@ -38,15 +38,12 @@ const assert = std.debug.assert;...@@ -38,15 +38,12 @@ const assert = std.debug.assert;
38const ascii = std.ascii;38const ascii = std.ascii;
39const Allocator = std.mem.Allocator;39const Allocator = std.mem.Allocator;
40const Cache = std.Build.Cache;40const Cache = std.Build.Cache;
41const ThreadPool = std.Thread.Pool;
42const WaitGroup = std.Thread.WaitGroup;
43const git = @import("Fetch/git.zig");41const git = @import("Fetch/git.zig");
44const Package = @import("../Package.zig");42const Package = @import("../Package.zig");
45const Manifest = Package.Manifest;43const Manifest = Package.Manifest;
46const ErrorBundle = std.zig.ErrorBundle;44const ErrorBundle = std.zig.ErrorBundle;
4745
48arena: std.heap.ArenaAllocator,46arena: std.heap.ArenaAllocator,
49io: Io,
50location: Location,47location: Location,
51location_tok: std.zig.Ast.TokenIndex,48location_tok: std.zig.Ast.TokenIndex,
52hash_tok: std.zig.Ast.OptionalTokenIndex,49hash_tok: std.zig.Ast.OptionalTokenIndex,
...@@ -104,7 +101,8 @@ pub const LazyStatus = enum {...@@ -104,7 +101,8 @@ pub const LazyStatus = enum {
104101
105/// Contains shared state among all `Fetch` tasks.102/// Contains shared state among all `Fetch` tasks.
106pub const JobQueue = struct {103pub const JobQueue = struct {
107 mutex: std.Thread.Mutex = .{},104 io: Io,
105 mutex: Io.Mutex = .init,
108 /// It's an array hash map so that it can be sorted before rendering the106 /// It's an array hash map so that it can be sorted before rendering the
109 /// dependencies.zig source file.107 /// dependencies.zig source file.
110 /// Protected by `mutex`.108 /// Protected by `mutex`.
...@@ -115,8 +113,7 @@ pub const JobQueue = struct {...@@ -115,8 +113,7 @@ pub const JobQueue = struct {
115 all_fetches: std.ArrayList(*Fetch) = .empty,113 all_fetches: std.ArrayList(*Fetch) = .empty,
116114
117 http_client: *std.http.Client,115 http_client: *std.http.Client,
118 thread_pool: *ThreadPool,116 group: Io.Group = .init,
119 wait_group: WaitGroup = .{},
120 global_cache: Cache.Directory,117 global_cache: Cache.Directory,
121 /// If true then, no fetching occurs, and:118 /// If true then, no fetching occurs, and:
122 /// * The `global_cache` directory is assumed to be the direct parent119 /// * The `global_cache` directory is assumed to be the direct parent
...@@ -320,13 +317,14 @@ pub const Location = union(enum) {...@@ -320,13 +317,14 @@ pub const Location = union(enum) {
320317
321pub const RunError = error{318pub const RunError = error{
322 OutOfMemory,319 OutOfMemory,
320 Canceled,
323 /// This error code is intended to be handled by inspecting the321 /// This error code is intended to be handled by inspecting the
324 /// `error_bundle` field.322 /// `error_bundle` field.
325 FetchFailed,323 FetchFailed,
326};324};
327325
328pub fn run(f: *Fetch) RunError!void {326pub fn run(f: *Fetch) RunError!void {
329 const io = f.io;327 const io = f.job_queue.io;
330 const eb = &f.error_bundle;328 const eb = &f.error_bundle;
331 const arena = f.arena.allocator();329 const arena = f.arena.allocator();
332 const gpa = f.arena.child_allocator;330 const gpa = f.arena.child_allocator;
...@@ -488,7 +486,7 @@ fn runResource(...@@ -488,7 +486,7 @@ fn runResource(
488 resource: *Resource,486 resource: *Resource,
489 remote_hash: ?Package.Hash,487 remote_hash: ?Package.Hash,
490) RunError!void {488) RunError!void {
491 const io = f.io;489 const io = f.job_queue.io;
492 defer resource.deinit(io);490 defer resource.deinit(io);
493 const arena = f.arena.allocator();491 const arena = f.arena.allocator();
494 const eb = &f.error_bundle;492 const eb = &f.error_bundle;
...@@ -702,7 +700,8 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {...@@ -702,7 +700,8 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
702}700}
703701
704fn queueJobsForDeps(f: *Fetch) RunError!void {702fn queueJobsForDeps(f: *Fetch) RunError!void {
705 const io = f.io;703 const io = f.job_queue.io;
704
706 assert(f.job_queue.recursive);705 assert(f.job_queue.recursive);
707706
708 // If the package does not have a build.zig.zon file then there are no dependencies.707 // If the package does not have a build.zig.zon file then there are no dependencies.
...@@ -722,8 +721,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -722,8 +721,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
722 const prog_names = try parent_arena.alloc([]const u8, deps.len);721 const prog_names = try parent_arena.alloc([]const u8, deps.len);
723 var new_fetch_index: usize = 0;722 var new_fetch_index: usize = 0;
724723
725 f.job_queue.mutex.lock();724 try f.job_queue.mutex.lock(io);
726 defer f.job_queue.mutex.unlock();725 defer f.job_queue.mutex.unlock(io);
727726
728 try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);727 try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
729 try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));728 try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));
...@@ -792,7 +791,6 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -792,7 +791,6 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
792 f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);791 f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
793 }792 }
794 new_fetch.* = .{793 new_fetch.* = .{
795 .io = io,
796 .arena = std.heap.ArenaAllocator.init(gpa),794 .arena = std.heap.ArenaAllocator.init(gpa),
797 .location = location,795 .location = location,
798 .location_tok = dep.location_tok,796 .location_tok = dep.location_tok,
...@@ -830,11 +828,9 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -830,11 +828,9 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
830 break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] };828 break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] };
831 };829 };
832830
833 // Now it's time to give tasks to the thread pool.831 // Now it's time to dispatch tasks.
834 const thread_pool = f.job_queue.thread_pool;
835
836 for (new_fetches, prog_names) |*new_fetch, prog_name| {832 for (new_fetches, prog_names) |*new_fetch, prog_name| {
837 thread_pool.spawnWg(&f.job_queue.wait_group, workerRun, .{ new_fetch, prog_name });833 f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name });
838 }834 }
839}835}
840836
...@@ -848,6 +844,7 @@ pub fn workerRun(f: *Fetch, prog_name: []const u8) void {...@@ -848,6 +844,7 @@ pub fn workerRun(f: *Fetch, prog_name: []const u8) void {
848844
849 run(f) catch |err| switch (err) {845 run(f) catch |err| switch (err) {
850 error.OutOfMemory => f.oom_flag = true,846 error.OutOfMemory => f.oom_flag = true,
847 error.Canceled => {},
851 error.FetchFailed => {848 error.FetchFailed => {
852 // Nothing to do because the errors are already reported in `error_bundle`,849 // Nothing to do because the errors are already reported in `error_bundle`,
853 // and a reference is kept to the `Fetch` task inside `all_fetches`.850 // and a reference is kept to the `Fetch` task inside `all_fetches`.
...@@ -992,7 +989,7 @@ const FileType = enum {...@@ -992,7 +989,7 @@ const FileType = enum {
992const init_resource_buffer_size = git.Packet.max_data_length;989const init_resource_buffer_size = git.Packet.max_data_length;
993990
994fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void {991fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void {
995 const io = f.io;992 const io = f.job_queue.io;
996 const arena = f.arena.allocator();993 const arena = f.arena.allocator();
997 const eb = &f.error_bundle;994 const eb = &f.error_bundle;
998995
...@@ -1281,12 +1278,16 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: *Io.Reader) RunError!Unpack...@@ -1281,12 +1278,16 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: *Io.Reader) RunError!Unpack
1281 return res;1278 return res;
1282}1279}
12831280
1284fn unzip(f: *Fetch, out_dir: fs.Dir, reader: *Io.Reader) error{ ReadFailed, OutOfMemory, FetchFailed }!UnpackResult {1281fn unzip(
1282 f: *Fetch,
1283 out_dir: fs.Dir,
1284 reader: *Io.Reader,
1285) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult {
1285 // We write the entire contents to a file first because zip files1286 // We write the entire contents to a file first because zip files
1286 // must be processed back to front and they could be too large to1287 // must be processed back to front and they could be too large to
1287 // load into memory.1288 // load into memory.
12881289
1289 const io = f.io;1290 const io = f.job_queue.io;
1290 const cache_root = f.job_queue.global_cache;1291 const cache_root = f.job_queue.global_cache;
1291 const prefix = "tmp/";1292 const prefix = "tmp/";
1292 const suffix = ".zip";1293 const suffix = ".zip";
...@@ -1306,6 +1307,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: *Io.Reader) error{ ReadFailed, OutO...@@ -1306,6 +1307,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: *Io.Reader) error{ ReadFailed, OutO
1306 .read = true,1307 .read = true,
1307 }) catch |err| switch (err) {1308 }) catch |err| switch (err) {
1308 error.PathAlreadyExists => continue,1309 error.PathAlreadyExists => continue,
1310 error.Canceled => return error.Canceled,
1309 else => |e| return f.fail(1311 else => |e| return f.fail(
1310 f.location_tok,1312 f.location_tok,
1311 try eb.printString("failed to create temporary zip file: {t}", .{e}),1313 try eb.printString("failed to create temporary zip file: {t}", .{e}),
...@@ -1348,7 +1350,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: *Io.Reader) error{ ReadFailed, OutO...@@ -1348,7 +1350,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: *Io.Reader) error{ ReadFailed, OutO
1348}1350}
13491351
1350fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult {1352fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult {
1351 const io = f.io;1353 const io = f.job_queue.io;
1352 const arena = f.arena.allocator();1354 const arena = f.arena.allocator();
1353 // TODO don't try to get a gpa from an arena. expose this dependency higher up1355 // TODO don't try to get a gpa from an arena. expose this dependency higher up
1354 // because the backing of arena could be page allocator1356 // because the backing of arena could be page allocator
...@@ -1486,11 +1488,11 @@ const ComputedHash = struct {...@@ -1486,11 +1488,11 @@ const ComputedHash = struct {
1486/// hashed* and must not be present on the file system when calling this1488/// hashed* and must not be present on the file system when calling this
1487/// function.1489/// function.
1488fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {1490fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
1491 const io = f.job_queue.io;
1489 // All the path name strings need to be in memory for sorting.1492 // All the path name strings need to be in memory for sorting.
1490 const arena = f.arena.allocator();1493 const arena = f.arena.allocator();
1491 const gpa = f.arena.child_allocator;1494 const gpa = f.arena.child_allocator;
1492 const eb = &f.error_bundle;1495 const eb = &f.error_bundle;
1493 const thread_pool = f.job_queue.thread_pool;
1494 const root_dir = pkg_path.root_dir.handle;1496 const root_dir = pkg_path.root_dir.handle;
14951497
1496 // Collect all files, recursively, then sort.1498 // Collect all files, recursively, then sort.
...@@ -1514,10 +1516,8 @@ fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!Compute...@@ -1514,10 +1516,8 @@ fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!Compute
1514 {1516 {
1515 // The final hash will be a hash of each file hashed independently. This1517 // The final hash will be a hash of each file hashed independently. This
1516 // allows hashing in parallel.1518 // allows hashing in parallel.
1517 var wait_group: WaitGroup = .{};1519 var group: Io.Group = .init;
1518 // `computeHash` is called from a worker thread so there must not be1520 defer group.wait(io);
1519 // any waiting without working or a deadlock could occur.
1520 defer thread_pool.waitAndWork(&wait_group);
15211521
1522 while (walker.next() catch |err| {1522 while (walker.next() catch |err| {
1523 try eb.addRootErrorMessage(.{ .msg = try eb.printString(1523 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
...@@ -1542,7 +1542,7 @@ fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!Compute...@@ -1542,7 +1542,7 @@ fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!Compute
1542 .fs_path = fs_path,1542 .fs_path = fs_path,
1543 .failure = undefined, // to be populated by the worker1543 .failure = undefined, // to be populated by the worker
1544 };1544 };
1545 thread_pool.spawnWg(&wait_group, workerDeleteFile, .{ root_dir, deleted_file });1545 group.async(io, workerDeleteFile, .{ root_dir, deleted_file });
1546 try deleted_files.append(deleted_file);1546 try deleted_files.append(deleted_file);
1547 continue;1547 continue;
1548 }1548 }
...@@ -1570,7 +1570,7 @@ fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!Compute...@@ -1570,7 +1570,7 @@ fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!Compute
1570 .failure = undefined, // to be populated by the worker1570 .failure = undefined, // to be populated by the worker
1571 .size = undefined, // to be populated by the worker1571 .size = undefined, // to be populated by the worker
1572 };1572 };
1573 thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });1573 group.async(io, workerHashFile, .{ root_dir, hashed_file });
1574 try all_files.append(hashed_file);1574 try all_files.append(hashed_file);
1575 }1575 }
1576 }1576 }
...@@ -2241,7 +2241,6 @@ fn saveEmbedFile(comptime tarball_name: []const u8, dir: fs.Dir) !void {...@@ -2241,7 +2241,6 @@ fn saveEmbedFile(comptime tarball_name: []const u8, dir: fs.Dir) !void {
22412241
2242// Builds Fetch with required dependencies, clears dependencies on deinit().2242// Builds Fetch with required dependencies, clears dependencies on deinit().
2243const TestFetchBuilder = struct {2243const TestFetchBuilder = struct {
2244 thread_pool: ThreadPool,
2245 http_client: std.http.Client,2244 http_client: std.http.Client,
2246 global_cache_directory: Cache.Directory,2245 global_cache_directory: Cache.Directory,
2247 job_queue: Fetch.JobQueue,2246 job_queue: Fetch.JobQueue,
...@@ -2256,13 +2255,12 @@ const TestFetchBuilder = struct {...@@ -2256,13 +2255,12 @@ const TestFetchBuilder = struct {
2256 ) !*Fetch {2255 ) !*Fetch {
2257 const cache_dir = try cache_parent_dir.makeOpenPath("zig-global-cache", .{});2256 const cache_dir = try cache_parent_dir.makeOpenPath("zig-global-cache", .{});
22582257
2259 try self.thread_pool.init(.{ .allocator = allocator });
2260 self.http_client = .{ .allocator = allocator, .io = io };2258 self.http_client = .{ .allocator = allocator, .io = io };
2261 self.global_cache_directory = .{ .handle = cache_dir, .path = null };2259 self.global_cache_directory = .{ .handle = cache_dir, .path = null };
22622260
2263 self.job_queue = .{2261 self.job_queue = .{
2262 .io = io,
2264 .http_client = &self.http_client,2263 .http_client = &self.http_client,
2265 .thread_pool = &self.thread_pool,
2266 .global_cache = self.global_cache_directory,2264 .global_cache = self.global_cache_directory,
2267 .recursive = false,2265 .recursive = false,
2268 .read_only = false,2266 .read_only = false,
...@@ -2273,7 +2271,6 @@ const TestFetchBuilder = struct {...@@ -2273,7 +2271,6 @@ const TestFetchBuilder = struct {
22732271
2274 self.fetch = .{2272 self.fetch = .{
2275 .arena = std.heap.ArenaAllocator.init(allocator),2273 .arena = std.heap.ArenaAllocator.init(allocator),
2276 .io = io,
2277 .location = .{ .path_or_url = path_or_url },2274 .location = .{ .path_or_url = path_or_url },
2278 .location_tok = 0,2275 .location_tok = 0,
2279 .hash_tok = .none,2276 .hash_tok = .none,
...@@ -2309,7 +2306,6 @@ const TestFetchBuilder = struct {...@@ -2309,7 +2306,6 @@ const TestFetchBuilder = struct {
2309 self.fetch.prog_node.end();2306 self.fetch.prog_node.end();
2310 self.global_cache_directory.handle.close();2307 self.global_cache_directory.handle.close();
2311 self.http_client.deinit();2308 self.http_client.deinit();
2312 self.thread_pool.deinit();
2313 }2309 }
23142310
2315 fn packageDir(self: *TestFetchBuilder) !fs.Dir {2311 fn packageDir(self: *TestFetchBuilder) !fs.Dir {
src/main.zig+5-9
...@@ -5139,8 +5139,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8)...@@ -5139,8 +5139,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8)
5139 defer fetch_prog_node.end();5139 defer fetch_prog_node.end();
51405140
5141 var job_queue: Package.Fetch.JobQueue = .{5141 var job_queue: Package.Fetch.JobQueue = .{
5142 .io = io,
5142 .http_client = &http_client,5143 .http_client = &http_client,
5143 .thread_pool = &thread_pool,
5144 .global_cache = dirs.global_cache,5144 .global_cache = dirs.global_cache,
5145 .read_only = false,5145 .read_only = false,
5146 .recursive = true,5146 .recursive = true,
...@@ -5173,7 +5173,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8)...@@ -5173,7 +5173,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8)
51735173
5174 var fetch: Package.Fetch = .{5174 var fetch: Package.Fetch = .{
5175 .arena = std.heap.ArenaAllocator.init(gpa),5175 .arena = std.heap.ArenaAllocator.init(gpa),
5176 .io = io,
5177 .location = .{ .relative_path = phantom_package_root },5176 .location = .{ .relative_path = phantom_package_root },
5178 .location_tok = 0,5177 .location_tok = 0,
5179 .hash_tok = .none,5178 .hash_tok = .none,
...@@ -5207,10 +5206,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8)...@@ -5207,10 +5206,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8)
5207 &fetch,5206 &fetch,
5208 );5207 );
52095208
5210 job_queue.thread_pool.spawnWg(&job_queue.wait_group, Package.Fetch.workerRun, .{5209 job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });
5211 &fetch, "root",5210 job_queue.group.wait(io);
5212 });
5213 job_queue.wait_group.wait();
52145211
5215 try job_queue.consolidateErrors();5212 try job_queue.consolidateErrors();
52165213
...@@ -6899,8 +6896,8 @@ fn cmdFetch(...@@ -6899,8 +6896,8 @@ fn cmdFetch(
6899 defer global_cache_directory.handle.close();6896 defer global_cache_directory.handle.close();
69006897
6901 var job_queue: Package.Fetch.JobQueue = .{6898 var job_queue: Package.Fetch.JobQueue = .{
6899 .io = io,
6902 .http_client = &http_client,6900 .http_client = &http_client,
6903 .thread_pool = &thread_pool,
6904 .global_cache = global_cache_directory,6901 .global_cache = global_cache_directory,
6905 .recursive = false,6902 .recursive = false,
6906 .read_only = false,6903 .read_only = false,
...@@ -6912,7 +6909,6 @@ fn cmdFetch(...@@ -6912,7 +6909,6 @@ fn cmdFetch(
69126909
6913 var fetch: Package.Fetch = .{6910 var fetch: Package.Fetch = .{
6914 .arena = std.heap.ArenaAllocator.init(gpa),6911 .arena = std.heap.ArenaAllocator.init(gpa),
6915 .io = io,
6916 .location = .{ .path_or_url = path_or_url },6912 .location = .{ .path_or_url = path_or_url },
6917 .location_tok = 0,6913 .location_tok = 0,
6918 .hash_tok = .none,6914 .hash_tok = .none,
...@@ -6942,7 +6938,7 @@ fn cmdFetch(...@@ -6942,7 +6938,7 @@ fn cmdFetch(
6942 defer fetch.deinit();6938 defer fetch.deinit();
69436939
6944 fetch.run() catch |err| switch (err) {6940 fetch.run() catch |err| switch (err) {
6945 error.OutOfMemory => fatal("out of memory", .{}),6941 error.OutOfMemory, error.Canceled => |e| return e,
6946 error.FetchFailed => {}, // error bundle checked below6942 error.FetchFailed => {}, // error bundle checked below
6947 };6943 };
69486944