authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-06 00:29:31-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-06 12:36:29-08:00
log3d33735d73de269278fb87d2c9ae8fb7d83977be
tree629df23a332d0b78bcd9b374ae51152980a99672
parent36b65ab59e5e514ad06a11cde96b87c565d86ac8

zig build: add --fork CLI argument

closes #31124

5 files changed, 221 insertions(+), 69 deletions(-)

lib/compiler/build_runner.zig+26-23
......@@ -1573,6 +1573,23 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
15731573 \\ -fsys=[name] Enable a system integration
15741574 \\ -fno-sys=[name] Disable a system integration
15751575 \\
1576 \\ -fdarling, -fno-darling Integration with system-installed Darling to
1577 \\ execute macOS programs on Linux hosts
1578 \\ (default: no)
1579 \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute
1580 \\ foreign-architecture programs on Linux hosts
1581 \\ (default: no)
1582 \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc
1583 \\ (e.g. glibc or musl) built for multiple foreign
1584 \\ architectures, allowing execution of non-native
1585 \\ programs that link with libc.
1586 \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on
1587 \\ ARM64 macOS hosts. (default: no)
1588 \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to
1589 \\ execute WASI binaries. (default: no)
1590 \\ -fwine, -fno-wine Integration with system-installed Wine to execute
1591 \\ Windows programs on Linux hosts. (default: no)
1592 \\
15761593 \\ Available System Integrations: Enabled:
15771594 \\
15781595 );
......@@ -1592,33 +1609,16 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
15921609 try w.writeAll(
15931610 \\
15941611 \\General Options:
1612 \\ -h, --help Print this help and exit
1613 \\ -l, --list-steps Print available steps
1614 \\
15951615 \\ -p, --prefix [path] Where to install files (default: zig-out)
15961616 \\ --prefix-lib-dir [path] Where to install libraries
15971617 \\ --prefix-exe-dir [path] Where to install executables
15981618 \\ --prefix-include-dir [path] Where to install C header files
1599 \\
16001619 \\ --release[=mode] Request release mode, optionally specifying a
16011620 \\ preferred optimization mode: fast, safe, small
16021621 \\
1603 \\ -fdarling, -fno-darling Integration with system-installed Darling to
1604 \\ execute macOS programs on Linux hosts
1605 \\ (default: no)
1606 \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute
1607 \\ foreign-architecture programs on Linux hosts
1608 \\ (default: no)
1609 \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc
1610 \\ (e.g. glibc or musl) built for multiple foreign
1611 \\ architectures, allowing execution of non-native
1612 \\ programs that link with libc.
1613 \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on
1614 \\ ARM64 macOS hosts. (default: no)
1615 \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to
1616 \\ execute WASI binaries. (default: no)
1617 \\ -fwine, -fno-wine Integration with system-installed Wine to execute
1618 \\ Windows programs on Linux hosts. (default: no)
1619 \\
1620 \\ -h, --help Print this help and exit
1621 \\ -l, --list-steps Print available steps
16221622 \\ --verbose Print commands before executing them
16231623 \\ --color [auto|off|on] Enable or disable colored error messages
16241624 \\ --error-style [style] Control how build errors are printed
......@@ -1641,9 +1641,6 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
16411641 \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss
16421642 \\ --test-timeout <timeout> Limit execution time of unit tests, terminating if exceeded.
16431643 \\ The timeout must include a unit: ns, us, ms, s, m, h
1644 \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit
1645 \\ needed (Default) Lazy dependencies are fetched as needed
1646 \\ all Lazy dependencies are always fetched
16471644 \\ --watch Continuously rebuild when source files are modified
16481645 \\ --debounce <ms> Delay before rebuilding after changed file detected
16491646 \\ --webui[=ip] Enable the web interface on the given IP address
......@@ -1656,6 +1653,12 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
16561653 \\ -fincremental Enable incremental compilation
16571654 \\ -fno-incremental Disable incremental compilation
16581655 \\
1656 \\Package Management Options:
1657 \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit
1658 \\ needed (Default) Lazy dependencies are fetched as needed
1659 \\ all Lazy dependencies are always fetched
1660 \\ --fork=[path] Override one or more packages from dependency tree
1661 \\
16591662 \\Advanced Options:
16601663 \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error
16611664 \\ -fno-reference-trace Disable reference trace
src/Package.zig+37
......@@ -137,6 +137,43 @@ pub const Hash = struct {
137137 _ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable;
138138 return result;
139139 }
140
141 pub fn projectId(hash: *const Hash) ProjectId {
142 const name = std.mem.sliceTo(&hash.bytes, '-');
143 const hashplus = hash.bytes[std.mem.findScalarLast(u8, &hash.bytes, '-').? + 1 ..];
144 var decoded: [6]u8 = undefined;
145 std.base64.url_safe_no_pad.Decoder.decode(&decoded, hashplus[0..8]) catch unreachable;
146 const fingerprint_id = std.mem.readInt(u32, decoded[0..4], .little);
147 return .init(name, fingerprint_id);
148 }
149
150 test projectId {
151 const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw");
152 const project_id = hash.projectId();
153
154 var expected_name: [32]u8 = @splat(0);
155 expected_name[0.."pulseaudio".len].* = "pulseaudio".*;
156 try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
157
158 try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id);
159 }
160};
161
162/// Minimum information required to identify whether a package is an artifact
163/// of a given project.
164pub const ProjectId = struct {
165 /// Bytes after name.len are set to zero.
166 padded_name: [32]u8,
167 fingerprint_id: u32,
168
169 pub fn init(name: []const u8, fingerprint_id: u32) ProjectId {
170 var padded_name: [32]u8 = @splat(0);
171 @memcpy(padded_name[0..name.len], name);
172 return .{
173 .padded_name = padded_name,
174 .fingerprint_id = fingerprint_id,
175 };
176 }
140177};
141178
142179pub const MultihashFunction = enum(u16) {
src/Package/Fetch.zig+16-29
......@@ -142,6 +142,8 @@ pub const JobQueue = struct {
142142 /// Set of hashes that will be additionally fetched even if they are marked
143143 /// as lazy.
144144 unlazy_set: UnlazySet = .{},
145 /// Identifies paths that override all packages in the tree matching
146 fork_set: ForkSet = .{},
145147
146148 pub const Mode = enum {
147149 /// Non-lazy dependencies are always fetched.
......@@ -152,6 +154,7 @@ pub const JobQueue = struct {
152154 };
153155 pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch);
154156 pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void);
157 pub const ForkSet = std.AutoArrayHashMapUnmanaged(Package.ProjectId, Cache.Path);
155158
156159 pub fn deinit(jq: *JobQueue) void {
157160 const io = jq.io;
......@@ -801,45 +804,29 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
801804 const io = f.job_queue.io;
802805 const eb = &f.error_bundle;
803806 const arena = f.arena.allocator();
804 const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions(
807 const manifest_path = try pkg_root.join(arena, Manifest.basename);
808
809 f.manifest = @as(Manifest, undefined);
810
811 Manifest.load(
805812 io,
806 try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }),
807813 arena,
808 .limited(Manifest.max_bytes),
809 .@"1",
810 0,
814 manifest_path,
815 &f.manifest_ast,
816 eb,
817 &f.manifest.?,
818 f.allow_missing_paths_field,
811819 ) catch |err| switch (err) {
812820 error.FileNotFound => return,
821 error.Canceled => |e| return e,
822 error.ErrorsBundled => return error.FetchFailed,
813823 else => |e| {
814 const file_path = try pkg_root.join(arena, Manifest.basename);
815824 try eb.addRootErrorMessage(.{
816 .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ file_path, e }),
825 .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }),
817826 });
818827 return error.FetchFailed;
819828 },
820829 };
821
822 const ast = &f.manifest_ast;
823 ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
824
825 if (ast.errors.len > 0) {
826 const file_path = try std.fmt.allocPrint(arena, "{f}" ++ fs.path.sep_str ++ Manifest.basename, .{pkg_root});
827 try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, eb);
828 return error.FetchFailed;
829 }
830
831 const rng: std.Random.IoSource = .{ .io = io };
832
833 f.manifest = try Manifest.parse(arena, ast.*, rng.interface(), .{
834 .allow_missing_paths_field = f.allow_missing_paths_field,
835 });
836 const manifest = &f.manifest.?;
837
838 if (manifest.errors.len > 0) {
839 const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename });
840 try manifest.copyErrorsIntoBundle(ast.*, src_path, eb);
841 return error.FetchFailed;
842 }
843830}
844831
845832fn queueJobsForDeps(f: *Fetch) RunError!void {
src/Package/Manifest.zig+44-2
......@@ -1,10 +1,13 @@
11const Manifest = @This();
2
23const std = @import("std");
4const Io = std.Io;
35const mem = std.mem;
46const Allocator = std.mem.Allocator;
57const assert = std.debug.assert;
68const Ast = std.zig.Ast;
79const testing = std.testing;
10
811const Package = @import("../Package.zig");
912
1013pub const max_bytes = 10 * 1024 * 1024;
......@@ -53,7 +56,7 @@ pub const ParseOptions = struct {
5356
5457pub const Error = Allocator.Error;
5558
56pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) Error!Manifest {
59pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest {
5760 const main_node_index = ast.nodeData(.root).node;
5861
5962 var arena_instance = std.heap.ArenaAllocator.init(gpa);
......@@ -61,7 +64,7 @@ pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) E
6164
6265 var p: Parse = .{
6366 .gpa = gpa,
64 .ast = ast,
67 .ast = ast.*,
6568 .arena = arena_instance.allocator(),
6669 .errors = .{},
6770
......@@ -578,6 +581,45 @@ const Parse = struct {
578581 }
579582};
580583
584pub fn load(
585 io: Io,
586 arena: Allocator,
587 manifest_path: std.Build.Cache.Path,
588 ast: *std.zig.Ast,
589 error_bundle: *std.zig.ErrorBundle.Wip,
590 manifest: *Manifest,
591 allow_missing_paths_field: bool,
592) !void {
593 const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions(
594 io,
595 manifest_path.sub_path,
596 arena,
597 .limited(max_bytes),
598 .@"1",
599 0,
600 );
601
602 ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
603
604 if (ast.errors.len > 0) {
605 const file_path = try manifest_path.joinString(arena, "");
606 try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle);
607 return error.ErrorsBundled;
608 }
609
610 const rng: std.Random.IoSource = .{ .io = io };
611
612 manifest.* = try parse(arena, ast, rng.interface(), .{
613 .allow_missing_paths_field = allow_missing_paths_field,
614 });
615
616 if (manifest.errors.len > 0) {
617 const src_path = try error_bundle.printString("{f}", .{manifest_path});
618 try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle);
619 return error.ErrorsBundled;
620 }
621}
622
581623test "basic" {
582624 const gpa = testing.allocator;
583625
src/main.zig+98-15
......@@ -4896,7 +4896,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
48964896 var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
48974897 var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
48984898 var override_build_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map);
4899 var child_argv = std.array_list.Managed([]const u8).init(arena);
4899 var child_argv: std.ArrayList([]const u8) = .empty;
4900 var forks: std.ArrayList(Fork) = .empty;
49004901 var reference_trace: ?u32 = null;
49014902 var debug_compile_errors = false;
49024903 var verbose_link = (native_os != .wasi or builtin.link_libc) and
......@@ -4917,24 +4918,24 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
49174918 var debug_libc_paths_file: ?[]const u8 = null;
49184919
49194920 const argv_index_exe = child_argv.items.len;
4920 _ = try child_argv.addOne();
4921 _ = try child_argv.addOne(arena);
49214922
49224923 const self_exe_path = try process.executablePathAlloc(io, arena);
4923 try child_argv.append(self_exe_path);
4924 try child_argv.append(arena, self_exe_path);
49244925
49254926 const argv_index_zig_lib_dir = child_argv.items.len;
4926 _ = try child_argv.addOne();
4927 _ = try child_argv.addOne(arena);
49274928
49284929 const argv_index_build_file = child_argv.items.len;
4929 _ = try child_argv.addOne();
4930 _ = try child_argv.addOne(arena);
49304931
49314932 const argv_index_cache_dir = child_argv.items.len;
4932 _ = try child_argv.addOne();
4933 _ = try child_argv.addOne(arena);
49334934
49344935 const argv_index_global_cache_dir = child_argv.items.len;
4935 _ = try child_argv.addOne();
4936 _ = try child_argv.addOne(arena);
49364937
4937 try child_argv.appendSlice(&.{
4938 try child_argv.appendSlice(arena, &.{
49384939 "--seed",
49394940 try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}),
49404941 });
......@@ -4955,7 +4956,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
49554956 // read this file in the parent to obtain the results, in the case the child
49564957 // exits with code 3.
49574958 const results_tmp_file_nonce = std.fmt.hex(randInt(io, u64));
4958 try child_argv.append("-Z" ++ results_tmp_file_nonce);
4959 try child_argv.append(arena, "-Z" ++ results_tmp_file_nonce);
49594960
49604961 var color: Color = .auto;
49614962 var n_jobs: ?u32 = null;
......@@ -5000,11 +5001,19 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
50005001 fatal("expected [needed|all] after '--fetch=', found '{s}'", .{
50015002 sub_arg,
50025003 });
5004 } else if (mem.cutPrefix(u8, arg, "--fork=")) |sub_arg| {
5005 try forks.append(arena, .{
5006 .project_id = undefined,
5007 .path = .{
5008 .root_dir = .cwd(),
5009 .sub_path = sub_arg,
5010 },
5011 });
50035012 } else if (mem.eql(u8, arg, "--system")) {
50045013 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
50055014 i += 1;
50065015 system_pkg_dir_path = args[i];
5007 try child_argv.append("--system");
5016 try child_argv.append(arena, "--system");
50085017 continue;
50095018 } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| {
50105019 reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| {
......@@ -5014,7 +5023,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
50145023 reference_trace = null;
50155024 } else if (mem.eql(u8, arg, "--debug-log")) {
50165025 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5017 try child_argv.appendSlice(args[i .. i + 2]);
5026 try child_argv.appendSlice(arena, args[i .. i + 2]);
50185027 i += 1;
50195028 if (!build_options.enable_logging) {
50205029 warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{});
......@@ -5070,7 +5079,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
50705079 color = std.meta.stringToEnum(Color, args[i]) orelse {
50715080 fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] });
50725081 };
5073 try child_argv.appendSlice(&.{ arg, args[i] });
5082 try child_argv.appendSlice(arena, &.{ arg, args[i] });
50745083 continue;
50755084 } else if (mem.cutPrefix(u8, arg, "-j")) |str| {
50765085 const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| {
......@@ -5090,11 +5099,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
50905099 } else if (mem.eql(u8, arg, "--")) {
50915100 // The rest of the args are supposed to get passed onto
50925101 // build runner's `build.args`
5093 try child_argv.appendSlice(args[i..]);
5102 try child_argv.appendSlice(arena, args[i..]);
50945103 break;
50955104 }
50965105 }
5097 try child_argv.append(arg);
5106 try child_argv.append(arena, arg);
50985107 }
50995108 }
51005109
......@@ -5182,6 +5191,25 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
51825191 defer http_client.deinit();
51835192
51845193 var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
5194 var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
5195
5196 {
5197 // Populate fork_set.
5198 var group: Io.Group = .init;
5199 defer group.cancel(io);
5200
5201 for (forks.items) |*fork|
5202 group.async(io, loadFork, .{ io, gpa, fork, color });
5203
5204 try group.await(io);
5205
5206 for (forks.items) |*fork| {
5207 const project_id = fork.project_id catch |err| switch (err) {
5208 error.AlreadyReported => process.exit(1),
5209 };
5210 try fork_set.put(arena, project_id, fork.path);
5211 }
5212 }
51855213
51865214 // This loop is re-evaluated when the build script exits with an indication that it
51875215 // could not continue due to missing lazy dependencies.
......@@ -5518,6 +5546,61 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
55185546 }
55195547}
55205548
5549const Fork = struct {
5550 path: Path,
5551 project_id: error{AlreadyReported}!Package.ProjectId,
5552};
5553
5554fn loadFork(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void {
5555 fork.project_id = loadForkFallible(io, gpa, fork, color) catch |err| switch (err) {
5556 error.Canceled => |e| return e,
5557 error.AlreadyReported => |e| e,
5558 else => |e| e: {
5559 std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e });
5560 break :e error.AlreadyReported;
5561 },
5562 };
5563}
5564
5565fn loadForkFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !Package.ProjectId {
5566 var arena_instance = std.heap.ArenaAllocator.init(gpa);
5567 defer arena_instance.deinit();
5568 const arena = arena_instance.allocator();
5569
5570 var error_bundle: std.zig.ErrorBundle.Wip = undefined;
5571 try error_bundle.init(gpa);
5572 defer error_bundle.deinit();
5573
5574 const manifest_path = try fork.path.join(arena, Package.Manifest.basename);
5575
5576 var manifest_ast: std.zig.Ast = undefined;
5577 var manifest: Package.Manifest = undefined;
5578
5579 Package.Manifest.load(
5580 io,
5581 arena,
5582 manifest_path,
5583 &manifest_ast,
5584 &error_bundle,
5585 &manifest,
5586 true,
5587 ) catch |err| switch (err) {
5588 error.Canceled => |e| return e,
5589 error.ErrorsBundled => {
5590 assert(error_bundle.root_list.items.len > 0);
5591 var errors = try error_bundle.toOwnedBundle("");
5592 errors.renderToStderr(io, .{}, color) catch {};
5593 return error.AlreadyReported;
5594 },
5595 else => |e| {
5596 std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e });
5597 return error.AlreadyReported;
5598 },
5599 };
5600
5601 return .init(manifest.name, manifest.id);
5602}
5603
55215604const JitCmdOptions = struct {
55225605 cmd_name: []const u8,
55235606 root_src_path: []const u8,
......@@ -7410,7 +7493,7 @@ fn loadManifest(
74107493 process.exit(2);
74117494 }
74127495
7413 var manifest = try Package.Manifest.parse(gpa, ast, rng.interface(), .{});
7496 var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{});
74147497 errdefer manifest.deinit(gpa);
74157498
74167499 if (manifest.errors.len > 0) {