From e10cbf08eeed53561b5efe87a8cae0d7827f7301 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 12 Feb 2026 22:42:34 -0800 Subject: [PATCH 001/179] configure/make phase process separation sketch `zig build` CLI kicks off async task to compile optimized make runner executable, does fetch, compiles configure process in debug mode, then checks cache for the CLI options that affect configuration only. On hit, skips building/running the configure script. On miss, runs it, saves result in cache. The cached artifact is a "configuration" file - a serialized build step graph, which also includes unlazy package dependencies and additional file system dependencies. Next, awaits task for compiling optimized make runner executable, passes configuration file to it. Make runner is responsible for the CLI after that point. For the use case of detecting when `git describe` needs to be rerun, we can allow the configure process to manually add a file system mtime dependencies, in this case it would be on `.git/index` and `.git/HEAD`. This will enable two optimizations: 1. The bulk of the build system will not be rebuilt when user changes their configure script. 2. The user logic can be completely bypassed when the CLI options provided do not affect the configure phase - even if they affect the make phase. Remaining tasks in the branch: * some stuff in `zig build` CLI is `@panic("TODO")`. * configure runner needs to implement serialization of build graph using std.zig.Configuration * build runner needs to be transformed into make runner, consuming configuration file as input and deserializing the step graph. * introduce depending only on a file's metadata and *not* its contents into the cache system, and add a std.Build API for using it. --- lib/compiler/configure_runner.zig | 336 ++++++++++++++++++++++++++ lib/std/Build/Cache.zig | 6 + lib/std/zig.zig | 2 + lib/std/zig/Configuration.zig | 83 +++++++ lib/std/zig/LibCInstallation.zig | 18 +- src/Compilation.zig | 14 +- src/main.zig | 385 ++++++++++++++++++++---------- src/print_env.zig | 4 + 8 files changed, 713 insertions(+), 135 deletions(-) create mode 100644 lib/compiler/configure_runner.zig create mode 100644 lib/std/zig/Configuration.zig diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig new file mode 100644 index 0000000000000000000000000000000000000000..b1eafdb9ae7282fabf4102ee3eb3ec8c214383b0 --- /dev/null +++ b/lib/compiler/configure_runner.zig @@ -0,0 +1,336 @@ +const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; +const assert = std.debug.assert; +const fmt = std.fmt; +const mem = std.mem; +const process = std.process; +const File = std.Io.File; +const Step = std.Build.Step; +const Watch = std.Build.Watch; +const WebServer = std.Build.WebServer; +const Allocator = std.mem.Allocator; +const fatal = std.process.fatal; +const Writer = std.Io.Writer; +const Color = std.zig.Color; + +pub const root = @import("@build"); +pub const dependencies = @import("@dependencies"); + +pub const std_options: std.Options = .{ + .side_channels_mitigations = .none, + .http_disable_tls = true, +}; + +pub fn main(init: process.Init.Minimal) !void { + // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not + // always the case. So, we do need a true gpa for some things. + var debug_gpa_state: std.heap.DebugAllocator(.{}) = .init; + defer _ = debug_gpa_state.deinit(); + const gpa = debug_gpa_state.allocator(); + + var threaded: std.Io.Threaded = .init(gpa, .{ + .environ = init.environ, + .argv0 = .init(init.args), + }); + defer threaded.deinit(); + const io = threaded.io(); + + // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. + var arena_allocator: std.heap.ArenaAllocator = .init(std.heap.page_allocator); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); + + const args = try init.args.toSlice(arena); + + // skip my own exe name + var arg_idx: usize = 1; + + const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); + const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); + const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); + const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); + const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + + const cwd: Io.Dir = .cwd(); + + const zig_lib_directory: std.Build.Cache.Directory = .{ + .path = zig_lib_dir, + .handle = try cwd.openDir(io, zig_lib_dir, .{}), + }; + + const build_root_directory: std.Build.Cache.Directory = .{ + .path = build_root, + .handle = try cwd.openDir(io, build_root, .{}), + }; + + const local_cache_directory: std.Build.Cache.Directory = .{ + .path = cache_root, + .handle = try cwd.createDirPathOpen(io, cache_root, .{}), + }; + + const global_cache_directory: std.Build.Cache.Directory = .{ + .path = global_cache_root, + .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), + }; + + var graph: std.Build.Graph = .{ + .io = io, + .arena = arena, + .cache = .{ + .io = io, + .gpa = gpa, + .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), + .cwd = try process.currentPathAlloc(io, arena), + }, + .zig_exe = zig_exe, + .environ_map = try init.environ.createMap(arena), + .global_cache_root = global_cache_directory, + .zig_lib_directory = zig_lib_directory, + .host = .{ + .query = .{}, + .result = try std.zig.system.resolveTargetQuery(io, .{}), + }, + .time_report = false, + }; + + graph.cache.addPrefix(.{ .path = null, .handle = cwd }); + graph.cache.addPrefix(build_root_directory); + graph.cache.addPrefix(local_cache_directory); + graph.cache.addPrefix(global_cache_directory); + graph.cache.hash.addBytes(builtin.zig_version_string); + + const builder = try std.Build.create( + &graph, + build_root_directory, + local_cache_directory, + dependencies.root_deps, + ); + + var error_style: ErrorStyle = .verbose; + var multiline_errors: MultilineErrors = .indent; + var color: Color = .auto; + + if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { + if (std.meta.stringToEnum(ErrorStyle, str)) |style| { + error_style = style; + } + } + + if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { + if (std.meta.stringToEnum(MultilineErrors, str)) |style| { + multiline_errors = style; + } + } + + while (nextArg(args, &arg_idx)) |arg| { + if (mem.cutPrefix(u8, arg, "-D")) |option_contents| { + if (option_contents.len == 0) + fatalWithHint("expected option name after '-D'", .{}); + if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { + const option_name = option_contents[0..name_end]; + const option_value = option_contents[name_end + 1 ..]; + if (try builder.addUserInputOption(option_name, option_value)) + fatal(" access the help menu with 'zig build -h'", .{}); + } else { + if (try builder.addUserInputFlag(option_contents)) + fatal(" access the help menu with 'zig build -h'", .{}); + } + } else if (mem.eql(u8, arg, "--verbose")) { + builder.verbose = true; + } else if (mem.startsWith(u8, arg, "-fsys=")) { + const name = arg["-fsys=".len..]; + graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); + } else if (mem.startsWith(u8, arg, "-fno-sys=")) { + const name = arg["-fno-sys=".len..]; + graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); + } else if (mem.eql(u8, arg, "--release")) { + builder.release_mode = .any; + } else if (mem.startsWith(u8, arg, "--release=")) { + const text = arg["--release=".len..]; + builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { + fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ + arg, text, + }); + }; + } else if (mem.eql(u8, arg, "--search-prefix")) { + const search_prefix = nextArgOrFatal(args, &arg_idx); + builder.addSearchPrefix(search_prefix); + } else if (mem.eql(u8, arg, "--libc")) { + builder.libc_file = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--color")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); + color = std.meta.stringToEnum(Color, next_arg) orelse { + fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ + arg, next_arg, + }); + }; + } else if (mem.eql(u8, arg, "--error-style")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected style after '{s}'", .{arg}); + error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { + fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + }; + } else if (mem.eql(u8, arg, "--multiline-errors")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected style after '{s}'", .{arg}); + multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { + fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + }; + } else if (mem.eql(u8, arg, "--seed")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u32 after '{s}'", .{arg}); + graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { + fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ + next_arg, @errorName(err), + }); + }; + } else if (mem.eql(u8, arg, "--build-id")) { + builder.build_id = .fast; + } else if (mem.startsWith(u8, arg, "--build-id=")) { + const style = arg["--build-id=".len..]; + builder.build_id = std.zig.BuildId.parse(style) catch |err| { + fatal("unable to parse --build-id style '{s}': {s}", .{ + style, @errorName(err), + }); + }; + } else if (mem.eql(u8, arg, "--debug-pkg-config")) { + builder.debug_pkg_config = true; + } else if (mem.eql(u8, arg, "--debug-rt")) { + graph.debug_compiler_runtime_libs = true; + } else if (mem.eql(u8, arg, "--debug-compile-errors")) { + builder.debug_compile_errors = true; + } else if (mem.eql(u8, arg, "--debug-incremental")) { + builder.debug_incremental = true; + } else if (mem.eql(u8, arg, "--system")) { + // The usage text shows another argument after this parameter + // but it is handled by the parent process. The build runner + // only sees this flag. + graph.system_package_mode = true; + } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { + // --glibc-runtimes was the old name of the flag; kept for compatibility for now. + builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--verbose-link")) { + builder.verbose_link = true; + } else if (mem.eql(u8, arg, "--verbose-air")) { + builder.verbose_air = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { + builder.verbose_llvm_ir = "-"; + } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { + builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; + } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { + builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; + } else if (mem.eql(u8, arg, "--verbose-cimport")) { + builder.verbose_cimport = true; + } else if (mem.eql(u8, arg, "--verbose-cc")) { + builder.verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { + builder.verbose_llvm_cpu_features = true; + } else if (mem.eql(u8, arg, "-fincremental")) { + graph.incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + graph.incremental = false; + } else if (mem.eql(u8, arg, "-fwine")) { + builder.enable_wine = true; + } else if (mem.eql(u8, arg, "-fno-wine")) { + builder.enable_wine = false; + } else if (mem.eql(u8, arg, "-fqemu")) { + builder.enable_qemu = true; + } else if (mem.eql(u8, arg, "-fno-qemu")) { + builder.enable_qemu = false; + } else if (mem.eql(u8, arg, "-fwasmtime")) { + builder.enable_wasmtime = true; + } else if (mem.eql(u8, arg, "-fno-wasmtime")) { + builder.enable_wasmtime = false; + } else if (mem.eql(u8, arg, "-frosetta")) { + builder.enable_rosetta = true; + } else if (mem.eql(u8, arg, "-fno-rosetta")) { + builder.enable_rosetta = false; + } else if (mem.eql(u8, arg, "-fdarling")) { + builder.enable_darling = true; + } else if (mem.eql(u8, arg, "-fno-darling")) { + builder.enable_darling = false; + } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { + graph.allow_so_scripts = true; + } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { + graph.allow_so_scripts = false; + } else if (mem.eql(u8, arg, "-freference-trace")) { + builder.reference_trace = 256; + } else if (mem.startsWith(u8, arg, "-freference-trace=")) { + const num = arg["-freference-trace=".len..]; + builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); + process.exit(1); + }; + } else if (mem.eql(u8, arg, "-fno-reference-trace")) { + builder.reference_trace = null; + } else if (mem.cutPrefix(u8, arg, "-j")) |text| { + const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| + fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); + if (n < 1) fatal("number of jobs must be at least 1", .{}); + threaded.setAsyncLimit(.limited(n)); + } else if (mem.eql(u8, arg, "--")) { + builder.args = argsRest(args, arg_idx); + break; + } else { + fatalWithHint("unrecognized argument: '{s}'", .{arg}); + } + } + + const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); + const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); + + graph.stderr_mode = switch (color) { + .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), + .on => .escape_codes, + .off => .no_color, + }; + + try builder.runBuild(root); +} + +fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { + if (idx.* >= args.len) return null; + defer idx.* += 1; + return args[idx.*]; +} + +fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { + return nextArg(args, idx) orelse { + std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); + process.exit(1); + }; +} + +fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { + if (idx >= args.len) return null; + return args[idx..]; +} + +const ErrorStyle = enum { + verbose, + minimal, + verbose_clear, + minimal_clear, + fn verboseContext(s: ErrorStyle) bool { + return switch (s) { + .verbose, .verbose_clear => true, + .minimal, .minimal_clear => false, + }; + } + fn clearOnUpdate(s: ErrorStyle) bool { + return switch (s) { + .verbose, .minimal => false, + .verbose_clear, .minimal_clear => true, + }; + } +}; +const MultilineErrors = enum { indent, newline, none }; +const Summary = enum { all, new, failures, line, none }; + +fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { + std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); + process.exit(1); +} diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 86682aac19abb50adf8165769330fe400beabc16..3384e154cef1db389088441f7563ab17af44167f 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1024,6 +1024,12 @@ pub const Manifest = struct { try self.populateFileHash(gop.key_ptr); } + pub fn addPathPost(man: *Manifest, path: Path) !void { + _ = man; + _ = path; + @panic("TODO"); + } + /// Like `addFilePost` but when the file contents have already been loaded from disk. pub fn addFilePostContents( self: *Manifest, diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 75ef9c9b63195335b78c5cfa04d4d75742532905..9c2c956582d70a6582a268641702f3b41f7c08c7 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -11,6 +11,8 @@ const Writer = std.Io.Writer; const tokenizer = @import("zig/tokenizer.zig"); +/// The serialized output of configure phase ingested by make phase. +pub const Configuration = @import("zig/Configuration.zig"); pub const ErrorBundle = @import("zig/ErrorBundle.zig"); pub const Server = @import("zig/Server.zig"); pub const Client = @import("zig/Client.zig"); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig new file mode 100644 index 0000000000000000000000000000000000000000..86bf82fa3d0c73117dc85755b36a240e76fbc8c1 --- /dev/null +++ b/lib/std/zig/Configuration.zig @@ -0,0 +1,83 @@ +const Configuration = @This(); + +const std = @import("../std.zig"); +const Io = std.Io; +const Allocator = std.mem.Allocator; + +string_bytes: []u8, +steps: []Step, +path_deps_base: []Path.Base, +path_deps_sub: []String, +unlazy_deps: []String, + +pub const Header = extern struct { + string_bytes_len: u32, + steps_len: u32, + path_deps_len: u32, + unlazy_deps_len: u32, +}; + +pub const Step = extern struct { + name: String, +}; + +pub const Path = extern struct { + base: Base, + sub: String, + + pub const Base = enum(u8) { + cwd, + global_cache, + local_cache, + build_root, + }; + + pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path { + _ = c; + _ = arena; + _ = path; + @panic("TODO"); + } +}; + +pub const String = enum(u32) { + _, + + pub fn slice(index: String, c: *const Configuration) [:0]const u8 { + const start_slice = c.string_bytes[@intFromEnum(index)..]; + return start_slice[0..std.mem.indexOfScalar(u8, start_slice, 0).? :0]; + } +}; + +pub const LoadError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; + +pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration { + var buffer: [2000]u8 = undefined; + var fr = file.reader(io, &buffer); + const header = fr.interface.takeStruct(Header, .little) catch |err| switch (err) { + error.ReadFailed => return fr.err.?, + else => |e| return e, + }; + + var result: Configuration = .{ + .string_bytes = try arena.alloc(u8, header.string_bytes_len), + .steps = try arena.alloc(Step, header.steps_len), + .path_deps_sub = try arena.alloc(String, header.path_deps_len), + .path_deps_base = try arena.alloc(Path.Base, header.path_deps_len), + .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), + }; + + var vecs = [_][]u8{ + result.string_bytes, + @ptrCast(result.steps), + @ptrCast(result.path_deps_base), + @ptrCast(result.path_deps_sub), + @ptrCast(result.unlazy_deps), + }; + fr.interface.readVecAll(&vecs) catch |err| switch (err) { + error.ReadFailed => return fr.err.?, + else => |e| return e, + }; + + return result; +} diff --git a/lib/std/zig/LibCInstallation.zig b/lib/std/zig/LibCInstallation.zig index 9d7995a69af007b2bf8dd58cb58290ee25618082..dd45c96407019bb3e65022c18d4941091aac04f3 100644 --- a/lib/std/zig/LibCInstallation.zig +++ b/lib/std/zig/LibCInstallation.zig @@ -13,6 +13,7 @@ const Target = std.Target; const fs = std.fs; const Allocator = std.mem.Allocator; const Path = std.Build.Cache.Path; +const Cache = std.Build.Cache; const log = std.log.scoped(.libc_installation); const Environ = std.process.Environ; @@ -990,7 +991,7 @@ pub fn resolveCrtPaths( target: *const std.Target, ) error{ OutOfMemory, LibCInstallationMissingCrtDir }!CrtPaths { const crt_dir_path: Path = .{ - .root_dir = std.Build.Cache.Directory.cwd(), + .root_dir = Cache.Directory.cwd(), .sub_path = lci.crt_dir orelse return error.LibCInstallationMissingCrtDir, }; switch (target.os.tag) { @@ -1016,7 +1017,7 @@ pub fn resolveCrtPaths( }, .haiku, .serenity => { const gcc_dir_path: Path = .{ - .root_dir = std.Build.Cache.Directory.cwd(), + .root_dir = Cache.Directory.cwd(), .sub_path = lci.gcc_dir orelse return error.LibCInstallationMissingCrtDir, }; return .{ @@ -1038,3 +1039,16 @@ pub fn resolveCrtPaths( }, } } + +pub fn addToHash(opt_lci: ?*const LibCInstallation, hh: *Cache.HashHelper, abi: std.Target.Abi) void { + const lci = opt_lci orelse return hh.add(false); + hh.add(true); + hh.addOptionalBytes(lci.crt_dir); + switch (abi) { + .msvc, .itanium => { + hh.addOptionalBytes(lci.msvc_lib_dir); + hh.addOptionalBytes(lci.kernel32_lib_dir); + }, + else => {}, + } +} diff --git a/src/Compilation.zig b/src/Compilation.zig index d69813fd1120bf1384b4e16cd62dfc3679ac8f6f..9e9b66f03bb9c81dab21b305901efaee158fb8b9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -752,13 +752,10 @@ pub const Directories = struct { else => []const u8, }, environ_map: *const std.process.Environ.Map, + cwd: []const u8, ) Directories { const wasi = builtin.target.os.tag == .wasi; - const cwd = introspect.getResolvedCwd(io, arena) catch |err| { - fatal("unable to get cwd: {t}", .{err}); - }; - const zig_lib: Cache.Directory = d: { if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib"); if (wasi) break :d getPreopen(preopens, "/lib"); @@ -3528,14 +3525,7 @@ fn addNonIncrementalStuffToCacheManifest( man.hash.addListOfBytes(opts.rpath_list); man.hash.addListOfBytes(opts.symbol_wrap_set.keys()); if (comp.config.link_libc) { - man.hash.add(comp.libc_installation != null); - if (comp.libc_installation) |libc_installation| { - man.hash.addOptionalBytes(libc_installation.crt_dir); - if (target.abi == .msvc or target.abi == .itanium) { - man.hash.addOptionalBytes(libc_installation.msvc_lib_dir); - man.hash.addOptionalBytes(libc_installation.kernel32_lib_dir); - } - } + LibCInstallation.addToHash(comp.libc_installation, &man.hash, target.abi); man.hash.addOptionalBytes(target.dynamic_linker.get()); } man.hash.add(opts.repro); diff --git a/src/main.zig b/src/main.zig index 16df63526f3654bacfdfbf116d193ec7a0edd2b2..7648cadeeaa7a19581beac9e2f6175ddec486157 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3166,6 +3166,8 @@ fn buildOutputType( else => process.executablePathAlloc(io, arena) catch |err| fatal("unable to find zig self exe path: {t}", .{err}), }; + const cwd_path = try introspect.getResolvedCwd(io, arena); + // This `init` calls `fatal` on error. var dirs: Compilation.Directories = .init( arena, @@ -3182,6 +3184,7 @@ fn buildOutputType( preopens, self_exe_path, environ_map, + cwd_path, ); defer dirs.deinit(io); @@ -4936,16 +4939,21 @@ test sanitizeExampleName { try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); } -fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, environ_map: *process.Environ.Map) !void { - dev.check(.build_command); - +fn cmdBuild( + gpa: Allocator, + arena: Allocator, + io: Io, + args: []const []const u8, + environ_map: *process.Environ.Map, +) !void { var build_file: ?[]const u8 = null; var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); - var override_build_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map); - var child_argv: std.ArrayList([]const u8) = .empty; + var override_make_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map); + var configure_argv: std.ArrayList([]const u8) = .empty; + var make_argv: std.ArrayList([]const u8) = .empty; var forks: std.ArrayList(Fork) = .empty; var reference_trace: ?u32 = null; var debug_compile_errors = false; @@ -4965,46 +4973,32 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, var debug_target: ?[]const u8 = null; var debug_libc_paths_file: ?[]const u8 = null; - const argv_index_exe = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_exe = configure_argv.items.len; + _ = try configure_argv.addOne(arena); const self_exe_path = try process.executablePathAlloc(io, arena); - try child_argv.append(arena, self_exe_path); + try configure_argv.append(arena, self_exe_path); - const argv_index_zig_lib_dir = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_zig_lib_dir = configure_argv.items.len; + _ = try configure_argv.addOne(arena); - const argv_index_build_file = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_build_file = configure_argv.items.len; + _ = try configure_argv.addOne(arena); - const argv_index_cache_dir = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_cache_dir = configure_argv.items.len; + _ = try configure_argv.addOne(arena); - const argv_index_global_cache_dir = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_global_cache_dir = configure_argv.items.len; + _ = try configure_argv.addOne(arena); - try child_argv.appendSlice(arena, &.{ + try configure_argv.appendSlice(arena, &.{ "--seed", try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}), }); - const argv_index_seed = child_argv.items.len - 1; + const argv_index_seed = configure_argv.items.len - 1; - // This parent process needs a way to obtain results from the configuration - // phase of the child process. In the future, the make phase will be - // executed in a separate process than the configure phase, and we can then - // use stdout from the configuration phase for this purpose. - // - // However, currently, both phases are in the same process, and Run Step - // provides API for making the runned subprocesses inherit stdout and stderr - // which means these streams are not available for passing metadata back - // to the parent. - // - // Until make and configure phases are separated into different processes, - // the strategy is to choose a temporary file name ahead of time, and then - // read this file in the parent to obtain the results, in the case the child - // exits with code 3. - const results_tmp_file_nonce = std.fmt.hex(randInt(io, u64)); - try child_argv.append(arena, "-Z" ++ results_tmp_file_nonce); + const argv_index_configuration_file = make_argv.items.len; + _ = try make_argv.addOne(arena); var color: Color = .auto; var n_jobs: ?u32 = null; @@ -5027,7 +5021,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } else if (mem.eql(u8, arg, "--build-runner")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; - override_build_runner = args[i]; + override_make_runner = args[i]; continue; } else if (mem.eql(u8, arg, "--cache-dir")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); @@ -5071,7 +5065,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; system_pkg_dir_path = args[i]; - try child_argv.append(arena, "--system"); + try configure_argv.append(arena, "--system"); continue; } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { @@ -5081,7 +5075,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, reference_trace = null; } else if (mem.eql(u8, arg, "--debug-log")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - try child_argv.appendSlice(arena, args[i .. i + 2]); + try make_argv.appendSlice(arena, args[i .. i + 2]); i += 1; try addDebugLog(arena, args[i]); continue; @@ -5131,7 +5125,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, color = std.meta.stringToEnum(Color, args[i]) orelse { fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] }); }; - try child_argv.appendSlice(arena, &.{ arg, args[i] }); + try configure_argv.appendSlice(arena, &.{ arg, args[i] }); continue; } else if (mem.cutPrefix(u8, arg, "-j")) |str| { const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| { @@ -5146,25 +5140,85 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } else if (mem.eql(u8, arg, "--seed")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; - child_argv.items[argv_index_seed] = args[i]; + configure_argv.items[argv_index_seed] = args[i]; continue; } else if (mem.eql(u8, arg, "--")) { // The rest of the args are supposed to get passed onto // build runner's `build.args` - try child_argv.appendSlice(arena, args[i..]); + try configure_argv.appendSlice(arena, args[i..]); break; } } - try child_argv.append(arena, arg); + try make_argv.append(arena, arg); } } const root_prog_node = std.Progress.start(io, .{ .disable_printing = (color == .off), - .root_name = "Compile Build Script", + .root_name = "", }); defer root_prog_node.end(); + process.raiseFileDescriptorLimit(); + + const cwd_path = introspect.getResolvedCwd(io, arena) catch |err| + fatal("failed to get current directory path: {t}", .{err}); + + const build_root = try findBuildRoot(arena, io, .{ + .cwd_path = cwd_path, + .build_file = build_file, + }); + + // This `init` calls `fatal` on error. + var dirs: Compilation.Directories = .init( + arena, + io, + override_lib_dir, + override_global_cache_dir, + .{ .override = path: { + if (override_local_cache_dir) |d| break :path d; + break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename}); + } }, + .empty, + self_exe_path, + environ_map, + cwd_path, + ); + defer dirs.deinit(io); + + const thread_limit = @min( + @max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), + std.math.maxInt(Zcu.PerThread.IdBacking), + ); + try setThreadLimit(arena, thread_limit); + + // Kick off an optimized compilation of the make runner. + var make_runner_task = io.async(compileMakeRunner, .{ io, .{ + .dirs = &dirs, + .optimize = .ReleaseSafe, + .parent_prog_node = root_prog_node, + } }); + defer if (make_runner_task.cancel(io)) |mr| mr.deinit(io) else |_| {}; + + // Cache lookup for configure options. If we get a match, we can skip + // execution of the configure script. If not, we get the file path to pass + // to the configure process. + var local_cache: Cache = .{ + .gpa = gpa, + .io = io, + .manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}), + .cwd = cwd_path, + }; + local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + local_cache.addPrefix(dirs.zig_lib); + local_cache.addPrefix(dirs.local_cache); + local_cache.addPrefix(dirs.global_cache); + defer local_cache.manifest_dir.close(io); + + var config_man = local_cache.obtain(); + defer config_man.deinit(); + config_man.hash.addBytes(build_options.version); + // Normally the build runner is compiled for the host target but here is // some code to help when debugging edits to the build runner so that you // can make sure it compiles successfully on other targets. @@ -5174,6 +5228,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, const target_query = try std.Target.Query.parse(.{ .arch_os_abi = triple, }); + config_man.hash.addBytes(triple); break :t .{ .result = std.zig.resolveTargetQueryOrFatal(io, target_query), .is_native_os = false, @@ -5189,49 +5244,21 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, .is_explicit_dynamic_linker = false, }; }; + // Likewise, `--debug-libc` allows overriding the libc installation. const libc_installation: ?*const LibCInstallation = lci: { const paths_file = debug_libc_paths_file orelse break :lci null; if (!build_options.enable_debug_extensions) unreachable; const lci = try arena.create(LibCInstallation); lci.* = try .parse(arena, io, paths_file, &resolved_target.result); + LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi); break :lci lci; }; - process.raiseFileDescriptorLimit(); - - const cwd_path = try introspect.getResolvedCwd(io, arena); - const build_root = try findBuildRoot(arena, io, .{ - .cwd_path = cwd_path, - .build_file = build_file, - }); - - // This `init` calls `fatal` on error. - var dirs: Compilation.Directories = .init( - arena, - io, - override_lib_dir, - override_global_cache_dir, - .{ .override = path: { - if (override_local_cache_dir) |d| break :path d; - break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename}); - } }, - .empty, - self_exe_path, - environ_map, - ); - defer dirs.deinit(io); - - child_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; - child_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; - child_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; - child_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; - - const thread_limit = @min( - @max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), - std.math.maxInt(Zcu.PerThread.IdBacking), - ); - try setThreadLimit(arena, thread_limit); + configure_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; + configure_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; + configure_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; + configure_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; // Dummy http client that is not actually used when fetch_command is unsupported. // Prevents bootstrap from depending on a bunch of unnecessary stuff. @@ -5269,11 +5296,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, // This loop is re-evaluated when the build script exits with an indication that it // could not continue due to missing lazy dependencies. - while (true) { + const configuration_path: Path = cp: while (true) { // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. { - const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_build_runner) |runner| .{ + const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_make_runner) |runner| .{ .root = try .fromUnresolved(arena, dirs, &.{fs.path.dirname(runner) orelse "."}), .root_src_path = fs.path.basename(runner), } else .{ @@ -5497,6 +5524,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, config, ); + const compile_prog_node = root_prog_node.start("Compile Configure Script", 0); + defer compile_prog_node.end(); + try root_mod.deps.put(arena, "@build", build_mod); var create_diag: Compilation.CreateDiagnostic = undefined; @@ -5528,7 +5558,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, }; defer comp.destroy(); - updateModule(comp, color, root_prog_node) catch |err| switch (err) { + updateModule(comp, color, compile_prog_node) catch |err| switch (err) { error.CompileErrorsReported => process.exit(2), else => |e| return e, }; @@ -5536,52 +5566,74 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, // Since incremental compilation isn't done yet, we use cache_mode = whole // above, and thus the output file is already closed. //try comp.makeBinFileExecutable(); - child_argv.items[argv_index_exe] = try dirs.local_cache.join(arena, &.{ - "o", - &Cache.binToHex(comp.digest.?), - comp.emit_bin.?, - }); + const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); + const exe_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), + }; + _ = try config_man.addFilePath(exe_path, null); + configure_argv.items[argv_index_exe] = try exe_path.toString(arena); + + if (try config_man.hit()) { + const digest = config_man.final(); + break :cp .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + }; + } } if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", child_argv.items); + const cmd = try std.mem.join(arena, " ", configure_argv.items); fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); } + + const rand_int = randInt(io, u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); + const config_tmp_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = tmp_dir_sub_path, + }; + const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( + io, + config_tmp_path.sub_path, + .{ .read = true, .exclusive = true }, + ); + defer config_tmp_file.close(io); + switch (term: { - _ = try io.lockStderr(&.{}, .no_color); - defer io.unlockStderr(); + const child_node = root_prog_node.start("Run Configure Script", 0); + defer child_node.end(); var child = std.process.spawn(io, .{ - .argv = child_argv.items, - }) catch |err| fatal("failed to spawn build runner {s}: {t}", .{ child_argv.items[0], err }); + .argv = configure_argv.items, + .stdout = .{ .file = config_tmp_file }, + .progress_node = child_node, + }) catch |err| fatal("failed to spawn configure script {s}: {t}", .{ configure_argv.items[0], err }); defer child.kill(io); break :term child.wait(io) catch |err| - fatal("failed to wait build runner {s}: {t}", .{ child_argv.items[0], err }); + fatal("failed to wait configure script {s}: {t}", .{ configure_argv.items[0], err }); }) { .exited => |code| { - if (code == 0) return cleanExit(io); - // Indicates that the build runner has reported compile errors - // and this parent process does not need to report any further - // diagnostics. - if (code == 2) process.exit(2); + if (code != 0) { + // Failure to produce the configuration file. + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following configure command failed with exit code {d}:\n{s}", .{ code, cmd }); + } + // Even though the file is designed to be sent directly to make + // runner, we must load it now because: + // * If it contains additional file dependencies, we need to + // add them to `config_man` before obtaining the final digest. + // * If it contains a set of lazy packages that need to be + // fetched, we need to fetch those now and re-run configure. + var configuration = std.zig.Configuration.load(arena, io, config_tmp_file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); - if (code == 3) { - if (!dev.env.supports(.fetch_command)) process.exit(3); - // Indicates the configure phase failed due to missing lazy - // dependencies and stdout contains the hashes of the ones - // that are missing. - const s = fs.path.sep_str; - const tmp_sub_path = "tmp" ++ s ++ results_tmp_file_nonce; - const stdout = dirs.local_cache.handle.readFileAlloc(io, tmp_sub_path, arena, .limited(50 * 1024 * 1024)) catch |err| { - fatal("unable to read results of configure phase from '{f}{s}': {t}", .{ - dirs.local_cache, tmp_sub_path, err, - }); - }; - dirs.local_cache.handle.deleteFile(io, tmp_sub_path) catch {}; - - var it = mem.splitScalar(u8, stdout, '\n'); + if (configuration.unlazy_deps.len != 0) { + if (!dev.env.supports(.fetch_command)) process.exit(1); var any_errors = false; - while (it.next()) |hash| { - if (hash.len == 0) continue; + for (configuration.unlazy_deps) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); if (hash.len > Package.Hash.max_len) { std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ hash.len, hash, @@ -5591,10 +5643,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } try unlazy_set.put(arena, .fromSlice(hash), {}); } - if (any_errors) process.exit(3); + if (any_errors) process.exit(1); if (system_pkg_dir_path) |p| { // In this mode, the system needs to provide these packages; they // cannot be fetched by Zig. + const s = fs.path.sep_str; for (unlazy_set.keys()) |*hash| { std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice(), @@ -5602,28 +5655,115 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } std.log.info("remote package fetching disabled due to --system mode", .{}); std.log.info("dependencies might be avoidable depending on build configuration", .{}); - process.exit(3); + process.exit(1); } - continue; + continue :cp; } - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); + for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { + const conf_path: std.zig.Configuration.Path = .{ .base = base, .sub = sub }; + try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); + } + + const digest = config_man.final(); + const final_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + }; + Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + ) catch |err| { + fatal("failed to rename configuration file from {f} into {f}: {t}", .{ + config_tmp_path, final_path, err, + }); + }; + config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + + break :cp final_path; }, .signal => |sig| { - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command terminated with signal {t}:\n{s}", .{ sig, cmd }); + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following configure command terminated with signal {t}:\n{s}", .{ sig, cmd }); }, .stopped => |sig| { - const cmd = try std.mem.join(arena, " ", child_argv.items); + const cmd = try std.mem.join(arena, " ", configure_argv.items); fatal("the following build command stopped with signal {t}:\n{s}", .{ sig, cmd }); }, .unknown => { - const cmd = try std.mem.join(arena, " ", child_argv.items); + const cmd = try std.mem.join(arena, " ", configure_argv.items); fatal("the following build command crashed:\n{s}", .{cmd}); }, } + }; + + { + // Release all file system locks just before running the maker process. + var configuration_lock = config_man.toOwnedLock(); + defer configuration_lock.release(io); + + const make_runner = make_runner_task.await(io) catch |err| + fatal("failed to compile maker: {t}", .{err}); + defer make_runner.deinit(io); + + make_argv.items[0] = try make_runner.exe_path.toString(arena); + make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); } + + if (!process.can_spawn) { + const cmd = try std.mem.join(arena, " ", make_argv.items); + fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); + } + + switch (term: { + _ = try io.lockStderr(&.{}, .no_color); + defer io.unlockStderr(); + var child = std.process.spawn(io, .{ + .argv = make_argv.items, + }) catch |err| fatal("failed to spawn maker {s}: {t}", .{ make_argv.items[0], err }); + defer child.kill(io); + break :term child.wait(io) catch |err| + fatal("failed to wait maker {s}: {t}", .{ make_argv.items[0], err }); + }) { + .exited => |code| { + if (code == 0) return cleanExit(io); + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following maker command failed with exit code {d}:\n{s}", .{ code, cmd }); + }, + .signal => |sig| { + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following maker command terminated with signal {t}:\n{s}", .{ sig, cmd }); + }, + else => { + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following maker command crashed:\n{s}", .{cmd}); + }, + } +} + +const MakeRunner = struct { + exe_path: Path, + + const Options = struct { + dirs: *Compilation.Directories, + optimize: std.builtin.OptimizeMode, + parent_prog_node: std.Progress.Node, + }; + + fn deinit(mr: MakeRunner, io: Io) void { + _ = mr; + _ = io; + @panic("TODO"); + } +}; + +fn compileMakeRunner(io: Io, options: MakeRunner.Options) !MakeRunner { + _ = io; + _ = options; + @panic("TODO"); } const Fork = struct { @@ -5749,6 +5889,8 @@ fn jitCmdInner( const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); + const cwd_path = try introspect.getResolvedCwd(io, arena); + // This `init` calls `fatal` on error. var dirs: Compilation.Directories = .init( arena, @@ -5759,6 +5901,7 @@ fn jitCmdInner( preopens, self_exe_path, environ_map, + cwd_path, ); defer dirs.deinit(io); diff --git a/src/print_env.zig b/src/print_env.zig index 163648321e41dd6c8368c6e0874fcffd13fe183c..ec7eadd7095e3b6ac511640996d0537bd8dcbc02 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -8,6 +8,7 @@ const fatal = std.process.fatal; const build_options = @import("build_options"); const Compilation = @import("Compilation.zig"); +const introspect = @import("introspect.zig"); pub fn cmdEnv( arena: Allocator, @@ -28,6 +29,8 @@ pub fn cmdEnv( }, }; + const cwd_path = try introspect.getResolvedCwd(io, arena); + var dirs: Compilation.Directories = .init( arena, io, @@ -37,6 +40,7 @@ pub fn cmdEnv( preopens, if (builtin.target.os.tag != .wasi) self_exe_path, environ_map, + cwd_path, ); defer dirs.deinit(io); -- 2.54.0 From 2e88ac8842a6a5bbd9fe654292ab416cadfaf8bf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Feb 2026 16:03:37 -0800 Subject: [PATCH 002/179] zig build: configure runner basics implemented --- lib/compiler/configure_runner.zig | 256 +- lib/compiler/{build_runner.zig => maker.zig} | 410 ++-- lib/{std/Build => compiler/maker}/Fuzz.zig | 2 +- lib/compiler/maker/Graph.zig | 92 + lib/compiler/maker/Package.zig | 12 + lib/compiler/maker/Step.zig | 850 +++++++ lib/compiler/maker/Step/Compile.zig | 1074 +++++++++ lib/compiler/maker/Step/InstallArtifact.zig | 96 + lib/compiler/maker/Step/Run.zig | 2127 +++++++++++++++++ lib/compiler/maker/Step/WriteFile.zig | 206 ++ lib/{std/Build => compiler/maker}/Watch.zig | 0 .../maker}/Watch/FsEvents.zig | 0 .../Build => compiler/maker}/WebServer.zig | 0 lib/std/Build.zig | 198 +- lib/std/Build/Step.zig | 913 +------ lib/std/Build/Step/CheckFile.zig | 4 +- lib/std/Build/Step/Compile.zig | 1078 +-------- lib/std/Build/Step/ConfigHeader.zig | 4 +- lib/std/Build/Step/Fail.zig | 4 +- lib/std/Build/Step/Fmt.zig | 4 +- lib/std/Build/Step/InstallArtifact.zig | 101 +- lib/std/Build/Step/InstallDir.zig | 4 +- lib/std/Build/Step/InstallFile.zig | 4 +- lib/std/Build/Step/ObjCopy.zig | 4 +- lib/std/Build/Step/Options.zig | 4 +- lib/std/Build/Step/Run.zig | 2116 +--------------- lib/std/Build/Step/TranslateC.zig | 4 +- lib/std/Build/Step/UpdateSourceFiles.zig | 4 +- lib/std/Build/Step/WriteFile.zig | 211 +- lib/std/zig.zig | 2 - lib/std/zig/Configuration.zig | 240 +- src/main.zig | 157 +- 32 files changed, 5192 insertions(+), 4989 deletions(-) rename lib/compiler/{build_runner.zig => maker.zig} (83%) rename lib/{std/Build => compiler/maker}/Fuzz.zig (99%) create mode 100644 lib/compiler/maker/Graph.zig create mode 100644 lib/compiler/maker/Package.zig create mode 100644 lib/compiler/maker/Step.zig create mode 100644 lib/compiler/maker/Step/Compile.zig create mode 100644 lib/compiler/maker/Step/InstallArtifact.zig create mode 100644 lib/compiler/maker/Step/Run.zig create mode 100644 lib/compiler/maker/Step/WriteFile.zig rename lib/{std/Build => compiler/maker}/Watch.zig (100%) rename lib/{std/Build => compiler/maker}/Watch/FsEvents.zig (100%) rename lib/{std/Build => compiler/maker}/WebServer.zig (100%) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index b1eafdb9ae7282fabf4102ee3eb3ec8c214383b0..78d3dff4c399392b1519d2d3d78368b5b4e2ffcd 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -8,12 +8,11 @@ const mem = std.mem; const process = std.process; const File = std.Io.File; const Step = std.Build.Step; -const Watch = std.Build.Watch; -const WebServer = std.Build.WebServer; const Allocator = std.mem.Allocator; const fatal = std.process.fatal; const Writer = std.Io.Writer; const Color = std.zig.Color; +const Configuration = std.Build.Configuration; pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); @@ -26,7 +25,11 @@ pub const std_options: std.Options = .{ pub fn main(init: process.Init.Minimal) !void { // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not // always the case. So, we do need a true gpa for some things. - var debug_gpa_state: std.heap.DebugAllocator(.{}) = .init; + var debug_gpa_state: std.heap.DebugAllocator(.{ + // We'd rather have `zig build` run faster than catch harmless leaks in + // the user's build.zig script. + .stack_trace_frames = 0, + }) = .init; defer _ = debug_gpa_state.deinit(); const gpa = debug_gpa_state.allocator(); @@ -47,11 +50,11 @@ pub fn main(init: process.Init.Minimal) !void { // skip my own exe name var arg_idx: usize = 1; - const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); - const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); - const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); - const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); - const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); + const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); + const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); + const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); + const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); const cwd: Io.Dir = .cwd(); @@ -66,8 +69,8 @@ pub fn main(init: process.Init.Minimal) !void { }; const local_cache_directory: std.Build.Cache.Directory = .{ - .path = cache_root, - .handle = try cwd.createDirPathOpen(io, cache_root, .{}), + .path = local_cache_root, + .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), }; const global_cache_directory: std.Build.Cache.Directory = .{ @@ -137,8 +140,6 @@ pub fn main(init: process.Init.Minimal) !void { if (try builder.addUserInputFlag(option_contents)) fatal(" access the help menu with 'zig build -h'", .{}); } - } else if (mem.eql(u8, arg, "--verbose")) { - builder.verbose = true; } else if (mem.startsWith(u8, arg, "-fsys=")) { const name = arg["-fsys=".len..]; graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); @@ -146,19 +147,14 @@ pub fn main(init: process.Init.Minimal) !void { const name = arg["-fno-sys=".len..]; graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); } else if (mem.eql(u8, arg, "--release")) { - builder.release_mode = .any; + graph.release_mode = .any; } else if (mem.startsWith(u8, arg, "--release=")) { const text = arg["--release=".len..]; - builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { + graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ arg, text, }); }; - } else if (mem.eql(u8, arg, "--search-prefix")) { - const search_prefix = nextArgOrFatal(args, &arg_idx); - builder.addSearchPrefix(search_prefix); - } else if (mem.eql(u8, arg, "--libc")) { - builder.libc_file = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--color")) { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); @@ -196,8 +192,6 @@ pub fn main(init: process.Init.Minimal) !void { style, @errorName(err), }); }; - } else if (mem.eql(u8, arg, "--debug-pkg-config")) { - builder.debug_pkg_config = true; } else if (mem.eql(u8, arg, "--debug-rt")) { graph.debug_compiler_runtime_libs = true; } else if (mem.eql(u8, arg, "--debug-compile-errors")) { @@ -209,71 +203,11 @@ pub fn main(init: process.Init.Minimal) !void { // but it is handled by the parent process. The build runner // only sees this flag. graph.system_package_mode = true; - } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { - // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--verbose-link")) { - builder.verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - builder.verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - builder.verbose_llvm_ir = "-"; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { - builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { - builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; - } else if (mem.eql(u8, arg, "--verbose-cimport")) { - builder.verbose_cimport = true; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - builder.verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - builder.verbose_llvm_cpu_features = true; - } else if (mem.eql(u8, arg, "-fincremental")) { - graph.incremental = true; - } else if (mem.eql(u8, arg, "-fno-incremental")) { - graph.incremental = false; - } else if (mem.eql(u8, arg, "-fwine")) { - builder.enable_wine = true; - } else if (mem.eql(u8, arg, "-fno-wine")) { - builder.enable_wine = false; - } else if (mem.eql(u8, arg, "-fqemu")) { - builder.enable_qemu = true; - } else if (mem.eql(u8, arg, "-fno-qemu")) { - builder.enable_qemu = false; - } else if (mem.eql(u8, arg, "-fwasmtime")) { - builder.enable_wasmtime = true; - } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - builder.enable_wasmtime = false; - } else if (mem.eql(u8, arg, "-frosetta")) { - builder.enable_rosetta = true; - } else if (mem.eql(u8, arg, "-fno-rosetta")) { - builder.enable_rosetta = false; - } else if (mem.eql(u8, arg, "-fdarling")) { - builder.enable_darling = true; - } else if (mem.eql(u8, arg, "-fno-darling")) { - builder.enable_darling = false; - } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { - graph.allow_so_scripts = true; - } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { - graph.allow_so_scripts = false; - } else if (mem.eql(u8, arg, "-freference-trace")) { - builder.reference_trace = 256; - } else if (mem.startsWith(u8, arg, "-freference-trace=")) { - const num = arg["-freference-trace=".len..]; - builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); - process.exit(1); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - builder.reference_trace = null; } else if (mem.cutPrefix(u8, arg, "-j")) |text| { const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); if (n < 1) fatal("number of jobs must be at least 1", .{}); threaded.setAsyncLimit(.limited(n)); - } else if (mem.eql(u8, arg, "--")) { - builder.args = argsRest(args, arg_idx); - break; } else { fatalWithHint("unrecognized argument: '{s}'", .{arg}); } @@ -289,6 +223,150 @@ pub fn main(init: process.Init.Minimal) !void { }; try builder.runBuild(root); + + var wc: Configuration.Wip = .init(gpa); + defer wc.deinit(); + + var stdout_buffer: [1024]u8 = undefined; + var file_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); + serialize(builder, &wc, &file_writer.interface) catch |err| switch (err) { + error.WriteFailed => fatal("failed to write configuration output: {t}", .{file_writer.err.?}), + error.OutOfMemory => |e| return e, + }; + + // This executable is short-lived and run in Debug mode, so we'd rather + // have `zig build` run faster than catch resource leaks in the user's + // build.zig script (or, frankly, this configure runner), therefore we call + // exit directly here rather than cleanExit. + process.exit(0); +} + +fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { + const graph = b.graph; + const arena = graph.arena; + const gpa = wc.gpa; + + // Starting from all top-level steps in `b`, traverse the entire step graph + // and add all step dependencies implied by module graphs. + const top_level_steps = b.top_level_steps.values(); + // Index corresponds to `Configuration.steps` index. + var step_map: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; + try step_map.ensureUnusedCapacity(arena, top_level_steps.len); + for (top_level_steps) |tls| { + step_map.putAssumeCapacityNoClobber(&tls.step, {}); + } + { + while (wc.steps.items.len < step_map.count()) { + const step = step_map.keys()[wc.steps.items.len]; + + // Set up any implied dependencies for this step. It's important that we do this first, so + // that the loop below discovers steps implied by the module graph. + try createModuleDependenciesForStep(step); + + try step_map.ensureUnusedCapacity(arena, step.dependencies.items.len); + for (step.dependencies.items) |other_step| { + step_map.putAssumeCapacity(other_step, {}); + } + + // Add and then de-duplicate dependencies. + const deps = d: { + const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len); + for (try wc.prepareDeps(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| + dep.* = @intCast(step_map.getIndex(dep_step).?); + break :d try wc.dedupeDeps(deps); + }; + + try wc.steps.ensureTotalCapacity(gpa, step_map.entries.capacity); + wc.steps.appendAssumeCapacity(.{ + .name = try wc.addString(step.name), + .flags = .{ .tag = step.tag }, + .deps = deps, + .extra_index = switch (step.tag) { + .top_level => e: { + const top_level: *Step.TopLevel = @fieldParentPtr("step", step); + break :e try wc.addExtra(@as(Configuration.Step.TopLevel, .{ + .description = try wc.addString(top_level.description), + })); + }, + .compile => @panic("TODO"), + .install_artifact => @panic("TODO"), + .install_file => @panic("TODO"), + .install_dir => @panic("TODO"), + .remove_dir => @panic("TODO"), + .fail => @panic("TODO"), + .fmt => @panic("TODO"), + .translate_c => @panic("TODO"), + .write_file => @panic("TODO"), + .update_source_files => @panic("TODO"), + .run => @panic("TODO"), + .check_file => @panic("TODO"), + .check_object => @panic("TODO"), + .config_header => @panic("TODO"), + .objcopy => @panic("TODO"), + .options => @panic("TODO"), + }, + }); + } + } + + try wc.unlazy_deps.ensureUnusedCapacity(gpa, graph.needed_lazy_dependencies.keys().len); + for (graph.needed_lazy_dependencies.keys()) |k| { + wc.unlazy_deps.appendAssumeCapacity(try wc.addString(k)); + } + + try wc.write(writer, .{ + .default_step = @intCast(step_map.getIndex(b.default_step).?), + }); +} + +/// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which +/// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. +fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { + const root_module = if (step.cast(Step.Compile)) |cs| root: { + break :root cs.root_module; + } else return; // not a compile step so no module dependencies + + // Starting from `root_module`, discover all modules in this graph. + const modules = root_module.getGraph().modules; + + // For each of those modules, set up the implied step dependencies. + for (modules) |mod| { + if (mod.root_source_file) |lp| lp.addStepDependencies(step); + for (mod.include_dirs.items) |include_dir| switch (include_dir) { + .path, + .path_system, + .path_after, + .framework_path, + .framework_path_system, + .embed_path, + => |lp| lp.addStepDependencies(step), + + .other_step => |other| { + other.getEmittedIncludeTree().addStepDependencies(step); + step.dependOn(&other.step); + }, + + .config_header_step => |other| step.dependOn(&other.step), + }; + for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); + for (mod.rpaths.items) |rpath| switch (rpath) { + .lazy_path => |lp| lp.addStepDependencies(step), + .special => {}, + }; + for (mod.link_objects.items) |link_object| switch (link_object) { + .static_path, + .assembly_file, + => |lp| lp.addStepDependencies(step), + .other_step => |other| step.dependOn(&other.step), + .system_lib => {}, + .c_source_file => |source| source.file.addStepDependencies(step), + .c_source_files => |source_files| source_files.root.addStepDependencies(step), + .win32_resource_file => |rc_source| { + rc_source.file.addStepDependencies(step); + for (rc_source.include_paths) |lp| lp.addStepDependencies(step); + }, + }; + } } fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { @@ -299,14 +377,17 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { return nextArg(args, idx) orelse { - std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); - process.exit(1); + fatal("expected argument after {q}\n access the help menu with \"zig build -h\"", .{ + args[idx.* - 1], + }); }; } -fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { - if (idx >= args.len) return null; - return args[idx..]; +fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { + const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); + if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); + const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); + return arg; } const ErrorStyle = enum { @@ -331,6 +412,5 @@ const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { - std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); - process.exit(1); + fatal(f ++ "\n access the help menu with \"zig build -h\"", args); } diff --git a/lib/compiler/build_runner.zig b/lib/compiler/maker.zig similarity index 83% rename from lib/compiler/build_runner.zig rename to lib/compiler/maker.zig index 439e46bb6e82bf93bf53652cc506c129781da740..44fe7170ab3b6043303880acb05b129c168d63d1 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/maker.zig @@ -1,4 +1,3 @@ -const runner = @This(); const builtin = @import("builtin"); const std = @import("std"); @@ -8,15 +7,17 @@ const fmt = std.fmt; const mem = std.mem; const process = std.process; const File = std.Io.File; -const Step = std.Build.Step; -const Watch = std.Build.Watch; -const WebServer = std.Build.WebServer; const Allocator = std.mem.Allocator; const fatal = std.process.fatal; const Writer = std.Io.Writer; +const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; -pub const root = @import("@build"); -pub const dependencies = @import("@dependencies"); +const Fuzz = @import("maker/Fuzz.zig"); +const Graph = @import("maker/Graph.zig"); +const Step = @import("maker/Step.zig"); +const Watch = @import("maker/Watch.zig"); +const WebServer = @import("maker/WebServer.zig"); pub const std_options: std.Options = .{ .side_channels_mitigations = .none, @@ -47,35 +48,36 @@ pub fn main(init: process.Init.Minimal) !void { // skip my own exe name var arg_idx: usize = 1; - const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); - const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); - const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); - const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); - const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + const zig_exe = cutArgPrefixOrFatal(args, &arg_idx, "--zig="); + const zig_lib_dir = cutArgPrefixOrFatal(args, &arg_idx, "--lib="); + const build_root = cutArgPrefixOrFatal(args, &arg_idx, "--build-root="); + const local_cache_root = cutArgPrefixOrFatal(args, &arg_idx, "--local-cache="); + const global_cache_root = cutArgPrefixOrFatal(args, &arg_idx, "--global-cache="); + const configure_path = cutArgPrefixOrFatal(args, &arg_idx, "--configure="); const cwd: Io.Dir = .cwd(); - const zig_lib_directory: std.Build.Cache.Directory = .{ + const zig_lib_directory: Cache.Directory = .{ .path = zig_lib_dir, .handle = try cwd.openDir(io, zig_lib_dir, .{}), }; - const build_root_directory: std.Build.Cache.Directory = .{ + const build_root_directory: Cache.Directory = .{ .path = build_root, .handle = try cwd.openDir(io, build_root, .{}), }; - const local_cache_directory: std.Build.Cache.Directory = .{ - .path = cache_root, - .handle = try cwd.createDirPathOpen(io, cache_root, .{}), + const local_cache_directory: Cache.Directory = .{ + .path = local_cache_root, + .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), }; - const global_cache_directory: std.Build.Cache.Directory = .{ + const global_cache_directory: Cache.Directory = .{ .path = global_cache_root, .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), }; - var graph: std.Build.Graph = .{ + var graph: Graph = .{ .io = io, .arena = arena, .cache = .{ @@ -101,18 +103,11 @@ pub fn main(init: process.Init.Minimal) !void { graph.cache.addPrefix(global_cache_directory); graph.cache.hash.addBytes(builtin.zig_version_string); - const builder = try std.Build.create( - &graph, - build_root_directory, - local_cache_directory, - dependencies.root_deps, - ); - var targets = std.array_list.Managed([]const u8).init(arena); var debug_log_scopes = std.array_list.Managed([]const u8).init(arena); var install_prefix: ?[]const u8 = null; - var dir_list = std.Build.DirList{}; + var dir_list: std.Build.DirList = .{}; var error_style: ErrorStyle = .verbose; var multiline_errors: MultilineErrors = .indent; var summary: ?Summary = null; @@ -122,11 +117,28 @@ pub fn main(init: process.Init.Minimal) !void { var color: Color = .auto; var help_menu = false; var steps_menu = false; - var output_tmp_nonce: ?[16]u8 = null; var watch = false; - var fuzz: ?std.Build.Fuzz.Mode = null; + var fuzz: ?Fuzz.Mode = null; var debounce_interval_ms: u16 = 50; var webui_listen: ?Io.net.IpAddress = null; + var verbose = false; + var sysroot: ?[]const u8 = null; + var search_prefixes: std.ArrayList([]const u8) = .empty; + var libc_file: ?[]const u8 = null; + var debug_pkg_config: bool = false; + // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, + // this will be the directory $glibc-build-dir/install/glibcs + // Given the example of the aarch64 target, this is the directory + // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. + // Also works for dynamic musl. + var libc_runtimes_dir: ?[]const u8 = null; + var enable_wine = false; + var enable_qemu = false; + var enable_wasmtime = false; + var enable_darling = false; + var enable_rosetta = false; + var reference_trace: ?u32 = null; + var run_args: ?[]const []const u8 = null; if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { if (std.meta.stringToEnum(ErrorStyle, str)) |style| { @@ -140,26 +152,23 @@ pub fn main(init: process.Init.Minimal) !void { } } + var configuration: Configuration = undefined; + { + var file = cwd.openFile(io, configure_path, .{}) catch |err| + fatal("failed to open configuration file {f}: {t}", .{ configure_path, err }); + defer file.close(io); + configuration = Configuration.load(arena, io, file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ configure_path, err }); + } + graph.configuration = &configuration; + graph.scanConfiguration(); + + std.log.err("TODO handle user -D options", .{}); + while (nextArg(args, &arg_idx)) |arg| { - if (mem.startsWith(u8, arg, "-Z")) { - if (arg.len != 18) fatalWithHint("bad argument: '{s}'", .{arg}); - output_tmp_nonce = arg[2..18].*; - } else if (mem.startsWith(u8, arg, "-D")) { - const option_contents = arg[2..]; - if (option_contents.len == 0) - fatalWithHint("expected option name after '-D'", .{}); - if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { - const option_name = option_contents[0..name_end]; - const option_value = option_contents[name_end + 1 ..]; - if (try builder.addUserInputOption(option_name, option_value)) - fatal(" access the help menu with 'zig build -h'", .{}); - } else { - if (try builder.addUserInputFlag(option_contents)) - fatal(" access the help menu with 'zig build -h'", .{}); - } - } else if (mem.startsWith(u8, arg, "-")) { + if (mem.startsWith(u8, arg, "-")) { if (mem.eql(u8, arg, "--verbose")) { - builder.verbose = true; + verbose = true; } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { help_menu = true; } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { @@ -172,15 +181,6 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.startsWith(u8, arg, "-fno-sys=")) { const name = arg["-fno-sys=".len..]; graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); - } else if (mem.eql(u8, arg, "--release")) { - builder.release_mode = .any; - } else if (mem.startsWith(u8, arg, "--release=")) { - const text = arg["--release=".len..]; - builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { - fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ - arg, text, - }); - }; } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { dir_list.lib_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { @@ -188,7 +188,7 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--prefix-include-dir")) { dir_list.include_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--sysroot")) { - builder.sysroot = nextArgOrFatal(args, &arg_idx); + sysroot = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--maxrss")) { const max_rss_text = nextArgOrFatal(args, &arg_idx); max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { @@ -235,10 +235,9 @@ pub fn main(init: process.Init.Minimal) !void { ); test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); } else if (mem.eql(u8, arg, "--search-prefix")) { - const search_prefix = nextArgOrFatal(args, &arg_idx); - builder.addSearchPrefix(search_prefix); + try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); } else if (mem.eql(u8, arg, "--libc")) { - builder.libc_file = nextArgOrFatal(args, &arg_idx); + libc_file = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--color")) { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); @@ -275,15 +274,6 @@ pub fn main(init: process.Init.Minimal) !void { next_arg, @errorName(err), }); }; - } else if (mem.eql(u8, arg, "--build-id")) { - builder.build_id = .fast; - } else if (mem.startsWith(u8, arg, "--build-id=")) { - const style = arg["--build-id=".len..]; - builder.build_id = std.zig.BuildId.parse(style) catch |err| { - fatal("unable to parse --build-id style '{s}': {s}", .{ - style, @errorName(err), - }); - }; } else if (mem.eql(u8, arg, "--debounce")) { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected u16 after '{s}'", .{arg}); @@ -304,17 +294,12 @@ pub fn main(init: process.Init.Minimal) !void { const next_arg = nextArgOrFatal(args, &arg_idx); try debug_log_scopes.append(next_arg); } else if (mem.eql(u8, arg, "--debug-pkg-config")) { - builder.debug_pkg_config = true; + debug_pkg_config = true; } else if (mem.eql(u8, arg, "--debug-rt")) { graph.debug_compiler_runtime_libs = .Debug; } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { - graph.debug_compiler_runtime_libs = - std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse - fatal("unrecognized optimization mode: '{s}'", .{rest}); - } else if (mem.eql(u8, arg, "--debug-compile-errors")) { - builder.debug_compile_errors = true; - } else if (mem.eql(u8, arg, "--debug-incremental")) { - builder.debug_incremental = true; + graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse + fatal("unrecognized optimization mode: {s}", .{rest}); } else if (mem.eql(u8, arg, "--system")) { // The usage text shows another argument after this parameter // but it is handled by the parent process. The build runner @@ -322,21 +307,7 @@ pub fn main(init: process.Init.Minimal) !void { graph.system_package_mode = true; } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--verbose-link")) { - builder.verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - builder.verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - builder.verbose_llvm_ir = "-"; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { - builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { - builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - builder.verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - builder.verbose_llvm_cpu_features = true; + libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--watch")) { watch = true; } else if (mem.eql(u8, arg, "--time-report")) { @@ -384,39 +355,39 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "-fno-incremental")) { graph.incremental = false; } else if (mem.eql(u8, arg, "-fwine")) { - builder.enable_wine = true; + enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { - builder.enable_wine = false; + enable_wine = false; } else if (mem.eql(u8, arg, "-fqemu")) { - builder.enable_qemu = true; + enable_qemu = true; } else if (mem.eql(u8, arg, "-fno-qemu")) { - builder.enable_qemu = false; + enable_qemu = false; } else if (mem.eql(u8, arg, "-fwasmtime")) { - builder.enable_wasmtime = true; + enable_wasmtime = true; } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - builder.enable_wasmtime = false; + enable_wasmtime = false; } else if (mem.eql(u8, arg, "-frosetta")) { - builder.enable_rosetta = true; + enable_rosetta = true; } else if (mem.eql(u8, arg, "-fno-rosetta")) { - builder.enable_rosetta = false; + enable_rosetta = false; } else if (mem.eql(u8, arg, "-fdarling")) { - builder.enable_darling = true; + enable_darling = true; } else if (mem.eql(u8, arg, "-fno-darling")) { - builder.enable_darling = false; + enable_darling = false; } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { graph.allow_so_scripts = true; } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { graph.allow_so_scripts = false; } else if (mem.eql(u8, arg, "-freference-trace")) { - builder.reference_trace = 256; + reference_trace = 256; } else if (mem.startsWith(u8, arg, "-freference-trace=")) { const num = arg["-freference-trace=".len..]; - builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); process.exit(1); }; } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - builder.reference_trace = null; + reference_trace = null; } else if (mem.cutPrefix(u8, arg, "-j")) |text| { const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); @@ -424,7 +395,7 @@ pub fn main(init: process.Init.Minimal) !void { threaded.setAsyncLimit(.limited(n)); graph.max_jobs = n; } else if (mem.eql(u8, arg, "--")) { - builder.args = argsRest(args, arg_idx); + run_args = argsRest(args, arg_idx); break; } else { fatalWithHint("unrecognized argument: '{s}'", .{arg}); @@ -453,51 +424,24 @@ pub fn main(init: process.Init.Minimal) !void { }); defer main_progress_node.end(); - builder.debug_log_scopes = debug_log_scopes.items; - builder.resolveInstallPrefix(install_prefix, dir_list); - { - var prog_node = main_progress_node.start("Configure", 0); - defer prog_node.end(); - try builder.runBuild(root); - createModuleDependencies(builder) catch @panic("OOM"); - } + graph.resolveInstallPrefix(install_prefix, dir_list); - if (graph.needed_lazy_dependencies.entries.len != 0) { - var buffer: std.ArrayList(u8) = .empty; - for (graph.needed_lazy_dependencies.keys()) |k| { - try buffer.appendSlice(arena, k); - try buffer.append(arena, '\n'); - } - const s = std.fs.path.sep_str; - const tmp_sub_path = "tmp" ++ s ++ (output_tmp_nonce orelse fatal("missing -Z arg", .{})); - local_cache_directory.handle.writeFile(io, .{ - .sub_path = tmp_sub_path, - .data = buffer.items, - .flags = .{ .exclusive = true }, - }) catch |err| { - fatal("unable to write configuration results to '{f}{s}': {s}", .{ - local_cache_directory, tmp_sub_path, @errorName(err), - }); - }; - process.exit(3); // Indicate configure phase failed with meaningful stdout. - } - - if (builder.validateUserInputDidItFail()) { + if (graph.validateUserInputDidItFail()) { fatal(" access the help menu with 'zig build -h'", .{}); } - validateSystemLibraryOptions(builder); + validateSystemLibraryOptions(&graph); if (help_menu) { var w = initStdoutWriter(io); - printUsage(builder, w) catch return stdout_writer_allocation.err.?; + printUsage(&graph, w) catch return stdout_writer_allocation.err.?; w.flush() catch return stdout_writer_allocation.err.?; return; } if (steps_menu) { var w = initStdoutWriter(io); - printSteps(builder, w) catch return stdout_writer_allocation.err.?; + printSteps(&graph, w) catch return stdout_writer_allocation.err.?; w.flush() catch return stdout_writer_allocation.err.?; return; } @@ -530,7 +474,7 @@ pub fn main(init: process.Init.Minimal) !void { run.max_rss_is_default = true; } - prepare(arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { + prepare(arena, &graph, targets.items, &run) catch |err| switch (err) { error.DependencyLoopDetected, error.InsufficientMemory => { // Perhaps in the future there could be an Advanced Options flag // such as --debug-build-runner-leaks which would make this code @@ -573,13 +517,7 @@ pub fn main(init: process.Init.Minimal) !void { }) { if (run.web_server) |*ws| ws.startBuild(); - try runStepNames( - builder, - targets.items, - main_progress_node, - &run, - fuzz, - ); + try runStepNames(graph, targets.items, main_progress_node, &run, fuzz); if (run.web_server) |*web_server| { if (fuzz) |mode| if (mode != .forever) fatal( @@ -678,23 +616,19 @@ const Run = struct { summary: Summary, }; -fn prepare( - arena: Allocator, - b: *std.Build, - step_names: []const []const u8, - run: *Run, - seed: u32, -) !void { +fn prepare(graph: *Graph, step_names: []const []const u8, run: *Run) !void { + const arena = graph.arena; + const seed: u32 = graph.random_seed; const gpa = run.gpa; const step_stack = &run.step_stack; if (step_names.len == 0) { - try step_stack.put(gpa, b.default_step, {}); + try step_stack.put(gpa, graph.configuration.default_step, {}); } else { try step_stack.ensureUnusedCapacity(gpa, step_names.len); for (0..step_names.len) |i| { const step_name = step_names[step_names.len - i - 1]; - const s = b.top_level_steps.get(step_name) orelse { + const s = graph.top_level_steps.get(step_name) orelse { std.log.info("access the help menu with \"zig build -h\"", .{}); fatal("no step named '{s}'", .{step_name}); }; @@ -709,7 +643,7 @@ fn prepare( rand.shuffle(*Step, starting_steps); for (starting_steps) |s| { - try constructGraphAndCheckForDependencyLoop(gpa, b, s, &run.step_stack, rand); + try constructGraphAndCheckForDependencyLoop(gpa, s, &run.step_stack, rand); } { @@ -745,14 +679,13 @@ fn prepare( } fn runStepNames( - b: *std.Build, + graph: *Graph, step_names: []const []const u8, parent_prog_node: std.Progress.Node, run: *Run, - fuzz: ?std.Build.Fuzz.Mode, + fuzz: ?Fuzz.Mode, ) !void { const gpa = run.gpa; - const graph = b.graph; const io = graph.io; const step_stack = &run.step_stack; @@ -775,7 +708,7 @@ fn runStepNames( var group: Io.Group = .init; defer group.cancel(io); // Start working on all of the initial steps... - for (initial_set.items) |s| try stepReady(&group, b, s, step_prog, run); + for (initial_set.items) |s| try stepReady(&group, s, step_prog, run); // ...and `makeStep` will trigger every other step when their last dependency finishes. try group.await(io); } @@ -848,7 +781,7 @@ fn runStepNames( } assert(mode == .limit); - var f = std.Build.Fuzz.init( + var f = Fuzz.init( gpa, io, step_stack.keys(), @@ -938,13 +871,13 @@ fn runStepNames( var print_node: PrintNode = .{ .parent = null }; if (step_names.len == 0) { print_node.last = true; - printTreeStep(b, b.default_step, run, t, &print_node, &step_stack_copy) catch {}; + printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {}; } else { - const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: { + const last_index = if (run.summary == .all) graph.top_level_steps.count() else blk: { var i: usize = step_names.len; while (i > 0) { i -= 1; - const step = b.top_level_steps.get(step_names[i]).?.step; + const step = graph.top_level_steps.get(step_names[i]).?.step; const found = switch (run.summary) { .all, .line, .none => unreachable, .failures => step.state != .success, @@ -952,12 +885,12 @@ fn runStepNames( }; if (found) break :blk i; } - break :blk b.top_level_steps.count(); + break :blk graph.top_level_steps.count(); }; for (step_names, 0..) |step_name, i| { - const tls = b.top_level_steps.get(step_name).?; + const tls = graph.top_level_steps.get(step_name).?; print_node.last = i + 1 == last_index; - printTreeStep(b, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; + printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; } } w.writeByte('\n') catch {}; @@ -1173,7 +1106,7 @@ fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void { } fn printTreeStep( - b: *std.Build, + graph: *Graph, s: *Step, run: *const Run, stderr: Io.Terminal, @@ -1231,7 +1164,7 @@ fn printTreeStep( .parent = parent_node, .last = i == last_index, }; - try printTreeStep(b, dep, run, stderr, &print_node, step_stack); + try printTreeStep(graph, dep, run, stderr, &print_node, step_stack); } } else { if (s.dependencies.items.len == 0) { @@ -1258,7 +1191,6 @@ fn printTreeStep( /// random order fn constructGraphAndCheckForDependencyLoop( gpa: Allocator, - b: *std.Build, s: *Step, step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), rand: std.Random, @@ -1282,8 +1214,8 @@ fn constructGraphAndCheckForDependencyLoop( for (deps) |dep| { try step_stack.put(gpa, dep, {}); - try dep.dependants.append(b.allocator, s); - constructGraphAndCheckForDependencyLoop(gpa, b, dep, step_stack, rand) catch |err| { + try dep.dependants.append(gpa, s); + constructGraphAndCheckForDependencyLoop(gpa, dep, step_stack, rand) catch |err| { if (err == error.DependencyLoopDetected) { std.debug.print(" {s}\n", .{s.name}); } @@ -1311,13 +1243,12 @@ fn constructGraphAndCheckForDependencyLoop( /// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked /// steps after "make" completes for `s`. fn makeStep( + graph: *Graph, group: *Io.Group, - b: *std.Build, s: *Step, root_prog_node: std.Progress.Node, run: *Run, ) Io.Cancelable!void { - const graph = b.graph; const io = graph.io; const gpa = run.gpa; @@ -1404,26 +1335,26 @@ fn makeStep( } } for (dispatch_set.items) |candidate| { - group.async(io, makeStep, .{ group, b, candidate, root_prog_node, run }); + group.async(io, makeStep, .{ graph, group, candidate, root_prog_node, run }); } } for (s.dependants.items) |dependant| { // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { - try stepReady(group, b, dependant, root_prog_node, run); + try stepReady(graph, group, dependant, root_prog_node, run); } } } fn stepReady( + graph: *Graph, group: *Io.Group, - b: *std.Build, s: *Step, root_prog_node: std.Progress.Node, run: *Run, ) !void { - const io = b.graph.io; + const io = graph.io; if (s.max_rss != 0) { try run.max_rss_mutex.lock(io); defer run.max_rss_mutex.unlock(io); @@ -1434,7 +1365,7 @@ fn stepReady( } run.available_rss -= s.max_rss; } - group.async(io, makeStep, .{ group, b, s, root_prog_node, run }); + group.async(io, makeStep, .{ graph, group, s, root_prog_node, run }); } pub fn printErrorMessages( @@ -1523,10 +1454,10 @@ pub fn printErrorMessages( try writer.writeByte('\n'); } -fn printSteps(builder: *std.Build, w: *Writer) !void { - const arena = builder.graph.arena; - for (builder.top_level_steps.values()) |top_level_step| { - const name = if (&top_level_step.step == builder.default_step) +fn printSteps(graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + for (graph.top_level_steps.values()) |top_level_step| { + const name = if (&top_level_step.step == graph.default_step) try fmt.allocPrint(arena, "{s} (default)", .{top_level_step.step.name}) else top_level_step.step.name; @@ -1534,26 +1465,26 @@ fn printSteps(builder: *std.Build, w: *Writer) !void { } } -fn printUsage(b: *std.Build, w: *Writer) !void { - const arena = b.graph.arena; +fn printUsage(graph: *Graph, w: *Writer) !void { + const arena = graph.arena; try w.print( \\Usage: {s} build [steps] [options] \\ \\Steps: \\ - , .{b.graph.zig_exe}); - try printSteps(b, w); + , .{graph.zig_exe}); + try printSteps(graph, w); try w.writeAll( \\ \\Project-Specific Options: \\ ); - if (b.available_options_list.items.len == 0) { + if (graph.available_options_list.items.len == 0) { try w.print(" (none)\n", .{}); } else { - for (b.available_options_list.items) |option| { + for (graph.available_options_list.items) |option| { const name = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ option.name, option.type_id }); try w.print("{s:<30} {s}\n", .{ name, option.description }); if (option.enum_options) |enum_options| { @@ -1597,10 +1528,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void { \\ Available System Integrations: Enabled: \\ ); - if (b.graph.system_library_options.entries.len == 0) { + if (graph.system_library_options.entries.len == 0) { try w.writeAll(" (none) -\n"); } else { - for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { + for (graph.system_library_options.keys(), graph.system_library_options.values()) |k, v| { const status = switch (v) { .declared_enabled => "yes", .declared_disabled => "no", @@ -1702,10 +1633,17 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { } fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); - process.exit(1); - }; + return nextArg(args, idx) orelse + fatal("expected argument after {q}\n access the help menu with \"zig build -h\"", .{args[idx.* - 1]}); +} + +fn cutArgPrefixOrFatal(args: []const [:0]const u8, idx: *usize, prefix: []const u8) []const u8 { + if (nextArg(args, idx)) |next_arg| { + if (mem.cutPrefix(u8, next_arg, prefix)) |arg| { + return arg; + } + } + fatal("expected argument after {q} to start with {q}", .{ args[idx.* - 1], prefix }); } fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { @@ -1740,9 +1678,9 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { process.exit(1); } -fn validateSystemLibraryOptions(b: *std.Build) void { +fn validateSystemLibraryOptions(graph: *Graph) void { var bad = false; - for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { + for (graph.system_library_options.keys(), graph.system_library_options.values()) |k, v| { switch (v) { .user_disabled, .user_enabled => { // The user tried to enable or disable a system library integration, but @@ -1759,84 +1697,6 @@ fn validateSystemLibraryOptions(b: *std.Build) void { } } -/// Starting from all top-level steps in `b`, traverses the entire step graph -/// and adds all step dependencies implied by module graphs. -fn createModuleDependencies(b: *std.Build) Allocator.Error!void { - const arena = b.graph.arena; - - var all_steps: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; - var next_step_idx: usize = 0; - - try all_steps.ensureUnusedCapacity(arena, b.top_level_steps.count()); - for (b.top_level_steps.values()) |tls| { - all_steps.putAssumeCapacityNoClobber(&tls.step, {}); - } - - while (next_step_idx < all_steps.count()) { - const step = all_steps.keys()[next_step_idx]; - next_step_idx += 1; - - // Set up any implied dependencies for this step. It's important that we do this first, so - // that the loop below discovers steps implied by the module graph. - try createModuleDependenciesForStep(step); - - try all_steps.ensureUnusedCapacity(arena, step.dependencies.items.len); - for (step.dependencies.items) |other_step| { - all_steps.putAssumeCapacity(other_step, {}); - } - } -} - -/// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which -/// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. -fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { - const root_module = if (step.cast(Step.Compile)) |cs| root: { - break :root cs.root_module; - } else return; // not a compile step so no module dependencies - - // Starting from `root_module`, discover all modules in this graph. - const modules = root_module.getGraph().modules; - - // For each of those modules, set up the implied step dependencies. - for (modules) |mod| { - if (mod.root_source_file) |lp| lp.addStepDependencies(step); - for (mod.include_dirs.items) |include_dir| switch (include_dir) { - .path, - .path_system, - .path_after, - .framework_path, - .framework_path_system, - .embed_path, - => |lp| lp.addStepDependencies(step), - - .other_step => |other| { - other.getEmittedIncludeTree().addStepDependencies(step); - step.dependOn(&other.step); - }, - - .config_header_step => |other| step.dependOn(&other.step), - }; - for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); - for (mod.rpaths.items) |rpath| switch (rpath) { - .lazy_path => |lp| lp.addStepDependencies(step), - .special => {}, - }; - for (mod.link_objects.items) |link_object| switch (link_object) { - .static_path, - .assembly_file, - => |lp| lp.addStepDependencies(step), - .other_step => |other| step.dependOn(&other.step), - .system_lib => {}, - .c_source_file => |source| source.file.addStepDependencies(step), - .c_source_files => |source_files| source_files.root.addStepDependencies(step), - .win32_resource_file => |rc_source| { - rc_source.file.addStepDependencies(step); - for (rc_source.include_paths) |lp| lp.addStepDependencies(step); - }, - }; - } -} - var stdio_buffer_allocation: [256]u8 = undefined; var stdout_writer_allocation: Io.File.Writer = undefined; @@ -1847,7 +1707,7 @@ fn initStdoutWriter(io: Io) *Writer { fn cleanTmpFiles(io: Io, steps: []const *Step) void { for (steps) |step| { - const wf = step.cast(std.Build.Step.WriteFile) orelse continue; + const wf = step.cast(Step.WriteFile) orelse continue; if (wf.mode != .tmp) continue; const path = wf.generated_directory.path orelse continue; Io.Dir.cwd().deleteTree(io, path) catch |err| { diff --git a/lib/std/Build/Fuzz.zig b/lib/compiler/maker/Fuzz.zig similarity index 99% rename from lib/std/Build/Fuzz.zig rename to lib/compiler/maker/Fuzz.zig index 9d3551a565a76ebfcae3f21e3e753cb553ef3cf2..a9b01e8111c0bf6034d740461d654f455e557813 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/compiler/maker/Fuzz.zig @@ -1,4 +1,4 @@ -const std = @import("../std.zig"); +const std = @import("Std"); const Io = std.Io; const Build = std.Build; const Cache = Build.Cache; diff --git a/lib/compiler/maker/Graph.zig b/lib/compiler/maker/Graph.zig new file mode 100644 index 0000000000000000000000000000000000000000..ed5e297b59339ce09271cf7a4ec49b4df4046a97 --- /dev/null +++ b/lib/compiler/maker/Graph.zig @@ -0,0 +1,92 @@ +//! Shared maker state among all steps. +const Graph = @This(); + +const std = @import("std"); +const Io = std.Io; +const Allocator = std.mem.Allocator; +const Configuration = std.Build.Configuration; + +const Step = @import("Step.zig"); +const Package = @import("Package.zig"); + +io: Io, +/// Process lifetime. +arena: Allocator, +system_library_options: std.StringArrayHashMapUnmanaged(std.Build.SystemLibraryMode), +system_package_mode: bool, +debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, +cache: std.Build.Cache, +zig_exe: [:0]const u8, +environ_map: std.process.Environ.Map, +global_cache_root: std.Build.Cache.Directory, +zig_lib_directory: std.Build.Cache.Directory, +incremental: ?bool, +random_seed: u32, +allow_so_scripts: ?bool, +time_report: bool, +/// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also +/// respects the '--color' flag. +stderr_mode: ?Io.Terminal.Mode, + +configuration: *const Configuration, +top_level_steps: std.AutoArrayHashMapUnmanaged(Configuration.String, Configuration.Step.Index), + +pub const DirList = struct { + lib_dir: ?[]const u8 = null, + exe_dir: ?[]const u8 = null, + include_dir: ?[]const u8 = null, +}; + +/// This function is intended to be called by lib/build_runner.zig, not a build.zig file. +pub fn resolveInstallPrefix(graph: *Graph, p: *Package, install_prefix: ?[]const u8, dir_list: DirList) !void { + if (p.dest_dir) |dest_dir| { + p.install_prefix = install_prefix orelse "/usr"; + p.install_path = b.pathJoin(&.{ dest_dir, p.install_prefix }); + } else { + p.install_prefix = install_prefix orelse + (p.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); + b.install_path = b.install_prefix; + } + + var lib_list = [_][]const u8{ b.install_path, "lib" }; + var exe_list = [_][]const u8{ b.install_path, "bin" }; + var h_list = [_][]const u8{ b.install_path, "include" }; + + if (dir_list.lib_dir) |dir| { + if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; + lib_list[1] = dir; + } + + if (dir_list.exe_dir) |dir| { + if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; + exe_list[1] = dir; + } + + if (dir_list.include_dir) |dir| { + if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; + h_list[1] = dir; + } + + b.lib_dir = b.pathJoin(&lib_list); + b.exe_dir = b.pathJoin(&exe_list); + b.h_dir = b.pathJoin(&h_list); +} + +fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { + // Create an installation directory local to this package. This will be used when + // dependant packages require a standard prefix, such as include directories for C headers. + var hash = b.graph.cache.hash; + // Random bytes to make unique. Refresh this with new random bytes when + // implementation is modified in a non-backwards-compatible way. + hash.add(@as(u32, 0xd8cb0056)); + hash.addBytes(b.dep_prefix); + + var wyhash = std.hash.Wyhash.init(0); + hashUserInputOptionsMap(b.allocator, b.user_input_options, &wyhash); + hash.add(wyhash.final()); + + const digest = hash.final(); + const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); + b.resolveInstallPrefix(install_prefix, .{}); +} + diff --git a/lib/compiler/maker/Package.zig b/lib/compiler/maker/Package.zig new file mode 100644 index 0000000000000000000000000000000000000000..e3ad442f69d6fa215b7dde6394211bc9cda283bf --- /dev/null +++ b/lib/compiler/maker/Package.zig @@ -0,0 +1,12 @@ +const Package = @This(); + +const std = @import("std"); + +install_prefix: []const u8, +install_path: []const u8, +dest_dir: ?[]const u8, +lib_dir: []const u8, +exe_dir: []const u8, +h_dir: []const u8, +/// Path to the directory containing build.zig. +build_root: std.Build.Cache.Path, diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/maker/Step.zig new file mode 100644 index 0000000000000000000000000000000000000000..295993ec245d79d5edde3444f8ed7aec7e2c6b5b --- /dev/null +++ b/lib/compiler/maker/Step.zig @@ -0,0 +1,850 @@ +const Step = @This(); + +const std = @import("std"); +const Io = std.Io; +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const assert = std.debug.assert; + +const WebServer = @import("WebServer.zig"); + +pub const Compile = @import("Step/Compile.zig"); +pub const Run = @import("Step/Run.zig"); + +state: State, +makeFn: MakeFn, +dependants: std.ArrayList(*Step), +/// Collects the set of files that retrigger this step to run. +/// +/// This is used by the build system's implementation of `--watch` but it can +/// also be potentially useful for IDEs to know what effects editing a +/// particular file has. +/// +/// Populated within `make`. Implementation may choose to clear and repopulate, +/// retain previous value, or update. +inputs: Inputs = .init, +pending_deps: u32, + +result_error_msgs: std.ArrayList([]const u8), +result_error_bundle: std.zig.ErrorBundle, +result_stderr: []const u8, +result_cached: bool, +result_duration_ns: ?u64, +/// 0 means unavailable or not reported. +result_peak_rss: usize, +/// If the step is failed and this field is populated, this is the command which failed. +/// This field may be populated even if the step succeeded. +result_failed_command: ?[]const u8, +test_results: TestResults, + + +pub const State = enum { + precheck_unstarted, + precheck_started, + /// This is also used to indicate "dirty" steps that have been modified + /// after a previous build completed, in which case, the step may or may + /// not have been completed before. Either way, one or more of its direct + /// file system inputs have been modified, meaning that the step needs to + /// be re-evaluated. + precheck_done, + dependency_failure, + success, + failure, + /// This state indicates that the step did not complete, however, it also did not fail, + /// and it is safe to continue executing its dependencies. + skipped, + /// This step was skipped because it specified a max_rss that exceeded the runner's maximum. + /// It is not safe to run its dependencies. + skipped_oom, +}; + +pub const Inputs = struct { + table: Table, + + pub const init: Inputs = .{ + .table = .{}, + }; + + pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, Files, Cache.Path.TableAdapter, false); + /// The special file name "." means any changes inside the directory. + pub const Files = std.ArrayList([]const u8); + + pub fn populated(inputs: *Inputs) bool { + return inputs.table.count() != 0; + } + + pub fn clear(inputs: *Inputs, gpa: Allocator) void { + for (inputs.table.values()) |*files| files.deinit(gpa); + inputs.table.clearRetainingCapacity(); + } +}; + +pub const TestResults = struct { + /// The total number of tests in the step. Every test has a "status" from the following: + /// * passed + /// * skipped + /// * failed cleanly + /// * crashed + /// * timed out + test_count: u32 = 0, + + /// The number of tests which were skipped (`error.SkipZigTest`). + skip_count: u32 = 0, + /// The number of tests which failed cleanly. + fail_count: u32 = 0, + /// The number of tests which terminated unexpectedly, i.e. crashed. + crash_count: u32 = 0, + /// The number of tests which timed out. + timeout_count: u32 = 0, + + /// The number of detected memory leaks. The associated test may still have passed; indeed, *all* + /// individual tests may have passed. However, the step as a whole fails if any test has leaks. + leak_count: u32 = 0, + /// The number of detected error logs. The associated test may still have passed; indeed, *all* + /// individual tests may have passed. However, the step as a whole fails if any test logs errors. + log_err_count: u32 = 0, + + pub fn isSuccess(tr: TestResults) bool { + // all steps are success or skip + return tr.fail_count == 0 and + tr.crash_count == 0 and + tr.timeout_count == 0 and + // no (otherwise successful) step leaked memory or logged errors + tr.leak_count == 0 and + tr.log_err_count == 0; + } + + /// Computes the number of tests which passed from the other values. + pub fn passCount(tr: TestResults) u32 { + return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count; + } +}; + +pub const MakeOptions = struct { + progress_node: std.Progress.Node, + watch: bool, + web_server: ?*WebServer, + /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. + unit_test_timeout_ns: ?u64, + /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. + gpa: Allocator, +}; + +pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; + +/// If the Step's `make` function reports `error.MakeFailed`, it indicates they +/// have already reported the error. Otherwise, we add a simple error report +/// here. +pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { + const arena = s.owner.allocator; + const graph = s.owner.graph; + const io = graph.io; + + var start_ts: ?Io.Timestamp = t: { + if (!graph.time_report) break :t null; + if (s.id == .compile) break :t null; + if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; + break :t Io.Clock.awake.now(io); + }; + const make_result = s.makeFn(s, options); + if (start_ts) |*ts| { + const duration = ts.untilNow(io, .awake); + options.web_server.?.updateTimeReportGeneric(s, duration); + } + + make_result catch |err| switch (err) { + error.MakeFailed, error.MakeSkipped => |e| return e, + else => { + s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); + return error.MakeFailed; + }, + }; + + if (!s.test_results.isSuccess()) { + return error.MakeFailed; + } + + if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { + const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ + s.result_peak_rss, s.max_rss, + }) catch @panic("OOM"); + s.result_error_msgs.append(arena, msg) catch @panic("OOM"); + } +} + +fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { + _ = options; + + var all_cached = true; + + for (step.dependencies.items) |dep| { + all_cached = all_cached and dep.result_cached; + } + + step.result_cached = all_cached; +} + +/// Implementation detail of file watching. Prepares the step for being re-evaluated. +/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. +pub fn invalidateResult(step: *Step, gpa: Allocator) bool { + if (step.state == .precheck_done) return false; + assert(step.pending_deps == 0); + step.state = .precheck_done; + step.reset(gpa); + for (step.dependants.items) |dependant| { + _ = dependant.invalidateResult(gpa); + dependant.pending_deps += 1; + } + return true; +} + +/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. +pub fn reset(step: *Step, gpa: Allocator) void { + assert(step.state == .precheck_done); + + if (step.result_failed_command) |cmd| gpa.free(cmd); + + step.result_error_msgs.clearRetainingCapacity(); + step.result_stderr = ""; + step.result_cached = false; + step.result_duration_ns = null; + step.result_peak_rss = 0; + step.result_failed_command = null; + step.test_results = .{}; + step.clearWatchInputs(); + + step.result_error_bundle.deinit(gpa); + step.result_error_bundle = std.zig.ErrorBundle.empty; +} + +/// Populates `s.result_failed_command`. +pub fn captureChildProcess( + s: *Step, + gpa: Allocator, + progress_node: std.Progress.Node, + argv: []const []const u8, +) !std.process.RunResult { + const graph = s.owner.graph; + const arena = graph.arena; + const io = graph.io; + + // If an error occurs, it's happened in this command: + assert(s.result_failed_command == null); + s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); + + try handleChildProcUnsupported(s); + try handleVerbose(s.owner, .inherit, argv); + + const result = std.process.run(arena, io, .{ + .argv = argv, + .environ_map = &graph.environ_map, + .progress_node = progress_node, + }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); + + if (result.stderr.len > 0) { + try s.result_error_msgs.append(arena, result.stderr); + } + + return result; +} + +pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { + try step.addError(fmt, args); + return error.MakeFailed; +} + +pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { + const arena = step.owner.allocator; + const msg = try std.fmt.allocPrint(arena, fmt, args); + try step.result_error_msgs.append(arena, msg); +} + +pub const ZigProcess = struct { + child: std.process.Child, + multi_reader_buffer: Io.File.MultiReader.Buffer(2), + multi_reader: Io.File.MultiReader, + progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn, + + pub const StreamEnum = enum { stdout, stderr }; + + pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void { + zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null; + } + + pub fn deinit(zp: *ZigProcess, io: Io) void { + zp.child.kill(io); + zp.multi_reader.deinit(); + zp.* = undefined; + } +}; + +/// Assumes that argv contains `--listen=-` and that the process being spawned +/// is the zig compiler - the same version that compiled the build runner. +/// Populates `s.result_failed_command`. +pub fn evalZigProcess( + s: *Step, + argv: []const []const u8, + prog_node: std.Progress.Node, + watch: bool, + web_server: ?*WebServer, + gpa: Allocator, +) !?Cache.Path { + const b = s.owner; + const io = b.graph.io; + + // If an error occurs, it's happened in this command: + assert(s.result_failed_command == null); + s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); + + if (s.getZigProcess()) |zp| update: { + assert(watch); + if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index); + zp.progress_ipc_index = null; + var exited = false; + defer if (exited) { + s.cast(Compile).?.zig_process = null; + zp.deinit(io); + gpa.destroy(zp); + } else zp.saveState(prog_node); + const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { + error.BrokenPipe, error.EndOfStream => |reason| { + std.log.info("{s} restart required: {t}", .{ argv[0], reason }); + // Process restart required. + const term = zp.child.wait(io) catch |e| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); + }; + _ = term; + exited = true; + break :update; + }, + else => |e| return e, + }; + + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + } + + if (s.result_error_msgs.items.len > 0 and result == null) { + // Crash detected. + const term = zp.child.wait(io) catch |e| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + exited = true; + try handleChildProcessTerm(s, term); + return error.MakeFailed; + } + + return result; + } + assert(argv.len != 0); + + try handleChildProcUnsupported(s); + try handleVerbose(s.owner, .inherit, argv); + + const zp = try gpa.create(ZigProcess); + defer if (!watch) gpa.destroy(zp); + + zp.child = std.process.spawn(io, .{ + .argv = argv, + .environ_map = &b.graph.environ_map, + .stdin = .pipe, + .stdout = .pipe, + .stderr = .pipe, + .request_resource_usage_statistics = true, + .progress_node = prog_node, + }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); + + zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ + zp.child.stdout.?, zp.child.stderr.?, + }); + if (watch) s.cast(Compile).?.zig_process = zp; + defer if (!watch) zp.deinit(io); + + const result = result: { + defer if (watch) zp.saveState(prog_node); + break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); + }; + + if (!watch) { + // Send EOF to stdin. + zp.child.stdin.?.close(io); + zp.child.stdin = null; + + const term = zp.child.wait(io) catch |err| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + + // Special handling for Compile step that is expecting compile errors. + if (s.cast(Compile)) |compile| switch (term) { + .exited => { + // Note that the exit code may be 0 in this case due to the + // compiler server protocol. + if (compile.expect_errors != null) { + return error.NeedCompileErrorCheck; + } + }, + else => {}, + }; + + try handleChildProcessTerm(s, term); + } + + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + } + + return result; +} + +/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. +pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { + const b = s.owner; + const io = b.graph.io; + const src_path = src_lazy_path.getPath3(b, s); + try handleVerbose(b, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); + return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| + return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); +} + +/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. +pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { + const b = s.owner; + const io = b.graph.io; + try handleVerbose(b, .inherit, &.{ "install", "-d", dest_path }); + return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| + return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); +} + +fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebServer, gpa: Allocator) !?Path { + const b = s.owner; + const arena = b.allocator; + const io = b.graph.io; + + const start_ts = Io.Clock.awake.now(io); + + try sendMessage(io, zp.child.stdin.?, .update); + if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); + + var result: ?Path = null; + var eos_err: error{EndOfStream}!void = {}; + + const stdout = zp.multi_reader.fileReader(0); + + while (true) { + const Header = std.zig.Server.Message.Header; + const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) { + error.EndOfStream => break, + error.ReadFailed => return stdout.err.?, + }; + const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) { + error.EndOfStream => |e| { + // Better to report the crash with stderr below, but we set + // this in case the child exits successfully while violating + // this protocol. + eos_err = e; + break; + }, + error.ReadFailed => return stdout.err.?, + }; + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return s.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + } + }, + .error_bundle => { + s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); + // This message indicates the end of the update. + if (watch) break; + }, + .emit_digest => { + const EmitDigest = std.zig.Server.Message.EmitDigest; + const emit_digest: *align(1) const EmitDigest = @ptrCast(body); + s.result_cached = emit_digest.flags.cache_hit; + const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; + result = .{ + .root_dir = b.cache_root, + .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), + }; + }, + .file_system_inputs => { + s.clearWatchInputs(); + var it = std.mem.splitScalar(u8, body, 0); + while (it.next()) |prefixed_path| { + const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); + const sub_path = try arena.dupe(u8, prefixed_path[1..]); + const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; + switch (prefix_index) { + .cwd => { + const path: Cache.Path = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .zig_lib => zl: { + if (s.cast(Step.Compile)) |compile| { + if (compile.zig_lib_dir) |zig_lib_dir| { + const lp = try zig_lib_dir.join(arena, sub_path); + try addWatchInput(s, lp); + break :zl; + } + } + const path: Cache.Path = .{ + .root_dir = s.owner.graph.zig_lib_directory, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .local_cache => { + const path: Cache.Path = .{ + .root_dir = b.cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .global_cache => { + const path: Cache.Path = .{ + .root_dir = s.owner.graph.global_cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + } + } + }, + .time_report => if (web_server) |ws| { + const TimeReport = std.zig.Server.Message.TimeReport; + const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); + ws.updateTimeReportCompile(.{ + .compile = s.cast(Step.Compile).?, + .use_llvm = tr.flags.use_llvm, + .stats = tr.stats, + .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), + .llvm_pass_timings_len = tr.llvm_pass_timings_len, + .files_len = tr.files_len, + .decls_len = tr.decls_len, + .trailing = body[@sizeOf(TimeReport)..], + }); + }, + else => {}, // ignore other messages + } + } + + s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()); + + const stderr_contents = zp.multi_reader.reader(1).buffered(); + if (stderr_contents.len > 0) { + try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); + } + + try eos_err; + + return result; +} + +pub fn getZigProcess(s: *Step) ?*ZigProcess { + return switch (s.id) { + .compile => s.cast(Compile).?.zig_process, + else => null, + }; +} + +fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + var w = file.writer(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +pub fn handleVerbose( + b: *Build, + cwd: std.process.Child.Cwd, + argv: []const []const u8, +) error{OutOfMemory}!void { + return handleVerbose2(b, cwd, null, argv); +} + +pub fn handleVerbose2( + b: *Build, + cwd: std.process.Child.Cwd, + opt_env: ?*const std.process.Environ.Map, + argv: []const []const u8, +) error{OutOfMemory}!void { + if (b.verbose) { + const graph = b.graph; + // Intention of verbose is to print all sub-process command lines to + // stderr before spawning them. + const text = try allocPrintCmd(b.allocator, cwd, if (opt_env) |env| .{ + .child = env, + .parent = &graph.environ_map, + } else null, argv); + std.debug.print("{s}\n", .{text}); + } +} + +/// Asserts that the caller has already populated `s.result_failed_command`. +pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { + if (!std.process.can_spawn) { + return s.fail("unable to spawn process: host cannot spawn child processes", .{}); + } +} + +/// Asserts that the caller has already populated `s.result_failed_command`. +pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { + assert(s.result_failed_command != null); + return switch (term) { + .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), + .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), + .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), + .unknown => s.fail("process terminated unexpectedly", .{}), + }; +} + +/// Prefer `cacheHitAndWatch` unless you already added watch inputs +/// separately from using the cache system. +pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool { + s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); + return s.result_cached; +} + +/// Clears previous watch inputs, if any, and then populates watch inputs from +/// the full set of files picked up by the cache manifest. +/// +/// Must be accompanied with `writeManifestAndWatch`. +pub fn cacheHitAndWatch(s: *Step, man: *Cache.Manifest) !bool { + const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); + s.result_cached = is_hit; + // The above call to hit() populates the manifest with files, so in case of + // a hit, we need to populate watch inputs. + if (is_hit) try setWatchInputsFromManifest(s, man); + return is_hit; +} + +fn failWithCacheError( + s: *Step, + man: *const Cache.Manifest, + err: Cache.Manifest.HitError, +) error{ OutOfMemory, Canceled, MakeFailed } { + switch (err) { + error.CacheCheckFailed => switch (man.diagnostic) { + .none => unreachable, + .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ + man.diagnostic, e, + }), + .file_open, .file_stat, .file_read, .file_hash => |op| { + const pp = man.files.keys()[op.file_index].prefixed_path; + const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; + return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ + prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, + }); + }, + }, + error.OutOfMemory, error.Canceled => |e| return e, + error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), + } +} + +/// Prefer `writeManifestAndWatch` unless you already added watch inputs +/// separately from using the cache system. +pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { + if (s.test_results.isSuccess()) { + man.writeManifest() catch |err| { + try s.addError("unable to write cache manifest: {t}", .{err}); + }; + } +} + +/// Clears previous watch inputs, if any, and then populates watch inputs from +/// the full set of files picked up by the cache manifest. +/// +/// Must be accompanied with `cacheHitAndWatch`. +pub fn writeManifestAndWatch(s: *Step, man: *Cache.Manifest) !void { + try writeManifest(s, man); + try setWatchInputsFromManifest(s, man); +} + +fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void { + const arena = s.owner.allocator; + const prefixes = man.cache.prefixes(); + clearWatchInputs(s); + for (man.files.keys()) |file| { + // The file path data is freed when the cache manifest is cleaned up at the end of `make`. + const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); + try addWatchInputFromPath(s, .{ + .root_dir = prefixes[file.prefixed_path.prefix], + .sub_path = std.fs.path.dirname(sub_path) orelse "", + }, std.fs.path.basename(sub_path)); + } +} + +/// For steps that have a single input that never changes when re-running `make`. +pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { + if (!step.inputs.populated()) try step.addWatchInput(lazy_path); +} + +pub fn clearWatchInputs(step: *Step) void { + const gpa = step.owner.allocator; + step.inputs.clear(gpa); +} + +/// Places a *file* dependency on the path. +pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!void { + switch (lazy_file) { + .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), + .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), + .cwd_relative => |path_string| { + try addWatchInputFromPath(step, .{ + .root_dir = .{ + .path = null, + .handle = Io.Dir.cwd(), + }, + .sub_path = std.fs.path.dirname(path_string) orelse "", + }, std.fs.path.basename(path_string)); + }, + // Nothing to watch because this dependency edge is modeled instead via `dependants`. + .generated => {}, + } +} + +/// Any changes inside the directory will trigger invalidation. +/// +/// See also `addDirectoryWatchInputFromPath` which takes a `Cache.Path` instead. +/// +/// Paths derived from this directory should also be manually added via +/// `addDirectoryWatchInputFromPath` if and only if this function returns +/// `true`. +pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool { + switch (lazy_directory) { + .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), + .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), + .cwd_relative => |path_string| { + try addDirectoryWatchInputFromPath(step, .{ + .root_dir = .{ + .path = null, + .handle = Io.Dir.cwd(), + }, + .sub_path = path_string, + }); + }, + // Nothing to watch because this dependency edge is modeled instead via `dependants`. + .generated => return false, + } + return true; +} + +/// Any changes inside the directory will trigger invalidation. +/// +/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. +/// +/// This function should only be called when it has been verified that the +/// dependency on `path` is not already accounted for by a `Step` dependency. +/// In other words, before calling this function, first check that the +/// `Build.LazyPath` which this `path` is derived from is not `generated`. +pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void { + return addWatchInputFromPath(step, path, "."); +} + +fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { + return addWatchInputFromPath(step, .{ + .root_dir = builder.build_root, + .sub_path = std.fs.path.dirname(sub_path) orelse "", + }, std.fs.path.basename(sub_path)); +} + +fn addDirectoryWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { + return addDirectoryWatchInputFromPath(step, .{ + .root_dir = builder.build_root, + .sub_path = sub_path, + }); +} + +fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !void { + const gpa = step.owner.allocator; + const gop = try step.inputs.table.getOrPut(gpa, path); + if (!gop.found_existing) gop.value_ptr.* = .empty; + try gop.value_ptr.append(gpa, basename); +} + +pub fn allocPrintCmd( + gpa: Allocator, + cwd: std.process.Child.Cwd, + opt_env: ?struct { + child: *const std.process.Environ.Map, + parent: *const std.process.Environ.Map, + }, + argv: []const []const u8, +) Allocator.Error![]u8 { + const shell = struct { + fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { + for (string) |c| { + if (switch (c) { + else => true, + '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, + '=' => is_argv0, + }) break; + } else return writer.writeAll(string); + + try writer.writeByte('"'); + for (string) |c| { + if (switch (c) { + std.ascii.control_code.nul => break, + '!', '"', '$', '\\', '`' => true, + else => !std.ascii.isPrint(c), + }) try writer.writeByte('\\'); + switch (c) { + std.ascii.control_code.nul => unreachable, + std.ascii.control_code.bel => try writer.writeByte('a'), + std.ascii.control_code.bs => try writer.writeByte('b'), + std.ascii.control_code.ht => try writer.writeByte('t'), + std.ascii.control_code.lf => try writer.writeByte('n'), + std.ascii.control_code.vt => try writer.writeByte('v'), + std.ascii.control_code.ff => try writer.writeByte('f'), + std.ascii.control_code.cr => try writer.writeByte('r'), + std.ascii.control_code.esc => try writer.writeByte('E'), + ' '...'~' => try writer.writeByte(c), + else => try writer.print("{o:0>3}", .{c}), + } + } + try writer.writeByte('"'); + } + }; + + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + const writer = &aw.writer; + switch (cwd) { + .inherit => {}, + .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, + .dir => @panic("TODO"), + } + if (opt_env) |env| { + var it = env.child.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + if (env.parent.get(key)) |process_value| { + if (std.mem.eql(u8, value, process_value)) continue; + } + writer.print("{s}=", .{key}) catch return error.OutOfMemory; + shell.escape(writer, value, false) catch return error.OutOfMemory; + writer.writeByte(' ') catch return error.OutOfMemory; + } + } + shell.escape(writer, argv[0], true) catch return error.OutOfMemory; + for (argv[1..]) |arg| { + writer.writeByte(' ') catch return error.OutOfMemory; + shell.escape(writer, arg, false) catch return error.OutOfMemory; + } + return aw.toOwnedSlice(); +} + diff --git a/lib/compiler/maker/Step/Compile.zig b/lib/compiler/maker/Step/Compile.zig new file mode 100644 index 0000000000000000000000000000000000000000..e3727025e25d189080e18cce93e48b13e0a226d1 --- /dev/null +++ b/lib/compiler/maker/Step/Compile.zig @@ -0,0 +1,1074 @@ +/// Populated during the make phase when there is a long-lived compiler process. +/// Managed by the build runner, not user build script. +zig_process: ?*Step.ZigProcess, + +fn make(step: *Step, options: Step.MakeOptions) !void { + const b = step.owner; + const compile: *Compile = @fieldParentPtr("step", step); + + const zig_args = try getZigArgs(compile, false); + + const maybe_output_dir = step.evalZigProcess( + zig_args, + options.progress_node, + (b.graph.incremental == true) and (options.watch or options.web_server != null), + options.web_server, + options.gpa, + ) catch |err| switch (err) { + error.NeedCompileErrorCheck => { + assert(compile.expect_errors != null); + try checkCompileErrors(compile); + return; + }, + else => |e| return e, + }; + + // Update generated files + if (maybe_output_dir) |output_dir| { + if (compile.emit_directory) |lp| { + lp.path = b.fmt("{f}", .{output_dir}); + } + + // zig fmt: off + if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); + if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); + // hack for stage2_x86_64 + coff + if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); + if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); + if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); + if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); + if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); + if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); + if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); + // zig fmt: on + } + + if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and + compile.version != null and compile.generated_bin != null and + std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) + { + try doAtomicSymLinks( + step, + compile.getEmittedBin().getPath2(b, step), + compile.major_only_filename.?, + compile.name_only_filename.?, + ); + } +} + +fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { + const step = &compile.step; + const b = step.owner; + const arena = b.allocator; + + var zig_args = std.array_list.Managed([]const u8).init(arena); + defer zig_args.deinit(); + + try zig_args.append(b.graph.zig_exe); + + const cmd = switch (compile.kind) { + .lib => "build-lib", + .exe => "build-exe", + .obj => "build-obj", + .@"test" => "test", + .test_obj => "test-obj", + }; + try zig_args.append(cmd); + + if (b.reference_trace) |some| { + try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); + } + try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); + + try addFlag(&zig_args, "llvm", compile.use_llvm); + try addFlag(&zig_args, "lld", compile.use_lld); + try addFlag(&zig_args, "new-linker", compile.use_new_linker); + + if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { + try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); + } + + switch (compile.entry) { + .default => {}, + .disabled => try zig_args.append("-fno-entry"), + .enabled => try zig_args.append("-fentry"), + .symbol_name => |entry_name| { + try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); + }, + } + + { + var symbol_it = compile.force_undefined_symbols.keyIterator(); + while (symbol_it.next()) |symbol_name| { + try zig_args.append("--force_undefined"); + try zig_args.append(symbol_name.*); + } + } + + if (compile.stack_size) |stack_size| { + try zig_args.append("--stack"); + try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); + } + + if (fuzz) { + try zig_args.append("-ffuzz"); + } + + { + // Stores system libraries that have already been seen for at least one + // module, along with any arguments that need to be passed to the + // compiler for each module individually. + var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; + var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; + + var prev_has_cflags = false; + var prev_has_rcflags = false; + var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; + var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; + // Track the number of positional arguments so that a nice error can be + // emitted if there is nothing to link. + var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); + + // Fully recursive iteration including dynamic libraries to detect + // libc and libc++ linkage. + for (compile.getCompileDependencies(true)) |some_compile| { + for (some_compile.root_module.getGraph().modules) |mod| { + if (mod.link_libc == true) compile.is_linking_libc = true; + if (mod.link_libcpp == true) compile.is_linking_libcpp = true; + } + } + + var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); + + // For this loop, don't chase dynamic libraries because their link + // objects are already linked. + for (compile.getCompileDependencies(false)) |dep_compile| { + for (dep_compile.root_module.getGraph().modules) |mod| { + // While walking transitive dependencies, if a given link object is + // already included in a library, it should not redundantly be + // placed on the linker line of the dependee. + const my_responsibility = dep_compile == compile; + const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); + + // Inherit dependencies on darwin frameworks. + if (!already_linked) { + for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { + try frameworks.put(arena, name, info); + } + } + + // Inherit dependencies on system libraries and static libraries. + for (mod.link_objects.items) |link_object| { + switch (link_object) { + .static_path => |static_path| { + if (my_responsibility) { + try zig_args.append(static_path.getPath2(mod.owner, step)); + total_linker_objects += 1; + } + }, + .system_lib => |system_lib| { + const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); + if (system_lib_gop.found_existing) { + try zig_args.appendSlice(system_lib_gop.value_ptr.*); + continue; + } else { + system_lib_gop.value_ptr.* = &.{}; + } + + if (already_linked) + continue; + + if ((system_lib.search_strategy != prev_search_strategy or + system_lib.preferred_link_mode != prev_preferred_link_mode) and + compile.linkage != .static) + { + switch (system_lib.search_strategy) { + .no_fallback => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_dylibs_only"), + .static => try zig_args.append("-search_static_only"), + }, + .paths_first => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_paths_first"), + .static => try zig_args.append("-search_paths_first_static"), + }, + .mode_first => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_dylibs_first"), + .static => try zig_args.append("-search_static_first"), + }, + } + prev_search_strategy = system_lib.search_strategy; + prev_preferred_link_mode = system_lib.preferred_link_mode; + } + + const prefix: []const u8 = prefix: { + if (system_lib.needed) break :prefix "-needed-l"; + if (system_lib.weak) break :prefix "-weak-l"; + break :prefix "-l"; + }; + switch (system_lib.use_pkg_config) { + .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), + .yes, .force => { + if (compile.runPkgConfig(system_lib.name)) |result| { + try zig_args.appendSlice(result.cflags); + try zig_args.appendSlice(result.libs); + try seen_system_libs.put(arena, system_lib.name, result.cflags); + } else |err| switch (err) { + error.PkgConfigInvalidOutput, + error.PkgConfigCrashed, + error.PkgConfigFailed, + error.PkgConfigNotInstalled, + error.PackageNotFound, + => switch (system_lib.use_pkg_config) { + .yes => { + // pkg-config failed, so fall back to linking the library + // by name directly. + try zig_args.append(b.fmt("{s}{s}", .{ + prefix, + system_lib.name, + })); + }, + .force => { + panic("pkg-config failed for library {s}", .{system_lib.name}); + }, + .no => unreachable, + }, + + else => |e| return e, + } + }, + } + }, + .other_step => |other| { + switch (other.kind) { + .exe => return step.fail("cannot link with an executable build artifact", .{}), + .@"test" => return step.fail("cannot link with a test", .{}), + .obj, .test_obj => { + const included_in_lib_or_obj = !my_responsibility and + (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); + if (!already_linked and !included_in_lib_or_obj) { + try zig_args.append(other.getEmittedBin().getPath2(b, step)); + total_linker_objects += 1; + } + }, + .lib => l: { + const other_produces_implib = other.producesImplib(); + const other_is_static = other_produces_implib or other.isStaticLibrary(); + + if (compile.isStaticLibrary() and other_is_static) { + // Avoid putting a static library inside a static library. + break :l; + } + + // For DLLs, we must link against the implib. + // For everything else, we directly link + // against the library file. + const full_path_lib = if (other_produces_implib) + try other.getGeneratedFilePath("generated_implib", &compile.step) + else + try other.getGeneratedFilePath("generated_bin", &compile.step); + + try zig_args.append(full_path_lib); + total_linker_objects += 1; + + if (other.linkage == .dynamic and + compile.rootModuleTarget().os.tag != .windows) + { + if (fs.path.dirname(full_path_lib)) |dirname| { + try zig_args.append("-rpath"); + try zig_args.append(dirname); + } + } + }, + } + }, + .assembly_file => |asm_file| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags) { + try zig_args.append("-cflags"); + try zig_args.append("--"); + prev_has_cflags = false; + } + try zig_args.append(asm_file.getPath2(mod.owner, step)); + total_linker_objects += 1; + }, + + .c_source_file => |c_source_file| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags or c_source_file.flags.len != 0) { + try zig_args.append("-cflags"); + for (c_source_file.flags) |arg| { + try zig_args.append(arg); + } + try zig_args.append("--"); + } + prev_has_cflags = (c_source_file.flags.len != 0); + + if (c_source_file.language) |lang| { + try zig_args.append("-x"); + try zig_args.append(lang.internalIdentifier()); + } + + try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); + + if (c_source_file.language != null) { + try zig_args.append("-x"); + try zig_args.append("none"); + } + total_linker_objects += 1; + }, + + .c_source_files => |c_source_files| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags or c_source_files.flags.len != 0) { + try zig_args.append("-cflags"); + for (c_source_files.flags) |arg| { + try zig_args.append(arg); + } + try zig_args.append("--"); + } + prev_has_cflags = (c_source_files.flags.len != 0); + + if (c_source_files.language) |lang| { + try zig_args.append("-x"); + try zig_args.append(lang.internalIdentifier()); + } + + const root_path = c_source_files.root.getPath2(mod.owner, step); + for (c_source_files.files) |file| { + try zig_args.append(b.pathJoin(&.{ root_path, file })); + } + + if (c_source_files.language != null) { + try zig_args.append("-x"); + try zig_args.append("none"); + } + + total_linker_objects += c_source_files.files.len; + }, + + .win32_resource_file => |rc_source_file| l: { + if (!my_responsibility) break :l; + + if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { + if (prev_has_rcflags) { + try zig_args.append("-rcflags"); + try zig_args.append("--"); + prev_has_rcflags = false; + } + } else { + try zig_args.append("-rcflags"); + for (rc_source_file.flags) |arg| { + try zig_args.append(arg); + } + for (rc_source_file.include_paths) |include_path| { + try zig_args.append("/I"); + try zig_args.append(include_path.getPath2(mod.owner, step)); + } + try zig_args.append("--"); + prev_has_rcflags = true; + } + try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); + total_linker_objects += 1; + }, + } + } + + // We need to emit the --mod argument here so that the above link objects + // have the correct parent module, but only if the module is part of + // this compilation. + if (!my_responsibility) continue; + if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { + const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; + try mod.appendZigProcessFlags(&zig_args, step); + + // --dep arguments + try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); + for (mod.import_table.keys(), mod.import_table.values()) |name, import| { + const import_index = cli_named_modules.modules.getIndex(import).?; + const import_cli_name = cli_named_modules.names.keys()[import_index]; + zig_args.appendAssumeCapacity("--dep"); + if (std.mem.eql(u8, import_cli_name, name)) { + zig_args.appendAssumeCapacity(import_cli_name); + } else { + zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); + } + } + + // When the CLI sees a -M argument, it determines whether it + // implies the existence of a Zig compilation unit based on + // whether there is a root source file. If there is no root + // source file, then this is not a zig compilation unit - it is + // perhaps a set of linker objects, or C source files instead. + // Linker objects are added to the CLI globally, while C source + // files must have a module parent. + if (mod.root_source_file) |lp| { + const src = lp.getPath2(mod.owner, step); + try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); + } else if (moduleNeedsCliArg(mod)) { + try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); + } + } + } + } + + if (total_linker_objects == 0) { + return step.fail("the linker needs one or more objects to link", .{}); + } + + for (frameworks.keys(), frameworks.values()) |name, info| { + if (info.needed) { + try zig_args.append("-needed_framework"); + } else if (info.weak) { + try zig_args.append("-weak_framework"); + } else { + try zig_args.append("-framework"); + } + try zig_args.append(name); + } + + if (compile.is_linking_libcpp) { + try zig_args.append("-lc++"); + } + + if (compile.is_linking_libc) { + try zig_args.append("-lc"); + } + } + + if (compile.win32_manifest) |manifest_file| { + try zig_args.append(manifest_file.getPath2(b, step)); + } + + if (compile.win32_module_definition) |module_file| { + try zig_args.append(module_file.getPath2(b, step)); + } + + if (compile.image_base) |image_base| { + try zig_args.append("--image-base"); + try zig_args.append(b.fmt("0x{x}", .{image_base})); + } + + for (compile.filters) |filter| { + try zig_args.append("--test-filter"); + try zig_args.append(filter); + } + + if (compile.test_runner) |test_runner| { + try zig_args.append("--test-runner"); + try zig_args.append(test_runner.path.getPath2(b, step)); + } + + for (b.debug_log_scopes) |log_scope| { + try zig_args.append("--debug-log"); + try zig_args.append(log_scope); + } + + if (b.debug_compile_errors) { + try zig_args.append("--debug-compile-errors"); + } + + if (b.debug_incremental) { + try zig_args.append("--debug-incremental"); + } + + if (b.verbose_air) try zig_args.append("--verbose-air"); + if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); + if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); + if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); + if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); + if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); + if (b.graph.time_report) try zig_args.append("--time-report"); + + if (compile.generated_asm != null) try zig_args.append("-femit-asm"); + if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); + if (compile.generated_docs != null) try zig_args.append("-femit-docs"); + if (compile.generated_implib != null) try zig_args.append("-femit-implib"); + if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); + if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); + if (compile.generated_h != null) try zig_args.append("-femit-h"); + + try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); + + switch (compile.compress_debug_sections) { + .none => {}, + .zlib => try zig_args.append("--compress-debug-sections=zlib"), + .zstd => try zig_args.append("--compress-debug-sections=zstd"), + } + + if (compile.link_eh_frame_hdr) { + try zig_args.append("--eh-frame-hdr"); + } + if (compile.link_emit_relocs) { + try zig_args.append("--emit-relocs"); + } + if (compile.link_function_sections) { + try zig_args.append("-ffunction-sections"); + } + if (compile.link_data_sections) { + try zig_args.append("-fdata-sections"); + } + if (compile.link_gc_sections) |x| { + try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); + } + if (!compile.linker_dynamicbase) { + try zig_args.append("--no-dynamicbase"); + } + if (compile.linker_allow_shlib_undefined) |x| { + try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); + } + if (compile.link_z_notext) { + try zig_args.append("-z"); + try zig_args.append("notext"); + } + if (!compile.link_z_relro) { + try zig_args.append("-z"); + try zig_args.append("norelro"); + } + if (compile.link_z_lazy) { + try zig_args.append("-z"); + try zig_args.append("lazy"); + } + if (compile.link_z_common_page_size) |size| { + try zig_args.append("-z"); + try zig_args.append(b.fmt("common-page-size={d}", .{size})); + } + if (compile.link_z_max_page_size) |size| { + try zig_args.append("-z"); + try zig_args.append(b.fmt("max-page-size={d}", .{size})); + } + if (compile.link_z_defs) { + try zig_args.append("-z"); + try zig_args.append("defs"); + } + + if (compile.libc_file) |libc_file| { + try zig_args.append("--libc"); + try zig_args.append(libc_file.getPath2(b, step)); + } else if (b.libc_file) |libc_file| { + try zig_args.append("--libc"); + try zig_args.append(libc_file); + } + + try zig_args.append("--cache-dir"); + try zig_args.append(b.cache_root.path orelse "."); + + try zig_args.append("--global-cache-dir"); + try zig_args.append(b.graph.global_cache_root.path orelse "."); + + if (b.graph.debug_compiler_runtime_libs) |mode| + try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); + + try zig_args.append("--name"); + try zig_args.append(compile.name); + + if (compile.linkage) |some| switch (some) { + .dynamic => try zig_args.append("-dynamic"), + .static => try zig_args.append("-static"), + }; + if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { + if (compile.version) |version| { + try zig_args.append("--version"); + try zig_args.append(b.fmt("{f}", .{version})); + } + + if (compile.rootModuleTarget().os.tag.isDarwin()) { + const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ + compile.rootModuleTarget().libPrefix(), + compile.name, + compile.rootModuleTarget().dynamicLibSuffix(), + }); + try zig_args.append("-install_name"); + try zig_args.append(install_name); + } + } + + if (compile.entitlements) |entitlements| { + try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); + } + if (compile.pagezero_size) |pagezero_size| { + const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); + try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); + } + if (compile.headerpad_size) |headerpad_size| { + const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); + try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); + } + if (compile.headerpad_max_install_names) { + try zig_args.append("-headerpad_max_install_names"); + } + if (compile.dead_strip_dylibs) { + try zig_args.append("-dead_strip_dylibs"); + } + if (compile.force_load_objc) { + try zig_args.append("-ObjC"); + } + if (compile.discard_local_symbols) { + try zig_args.append("--discard-all"); + } + + try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); + try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); + try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); + if (compile.rdynamic) { + try zig_args.append("-rdynamic"); + } + if (compile.import_memory) { + try zig_args.append("--import-memory"); + } + if (compile.export_memory) { + try zig_args.append("--export-memory"); + } + if (compile.import_symbols) { + try zig_args.append("--import-symbols"); + } + if (compile.import_table) { + try zig_args.append("--import-table"); + } + if (compile.export_table) { + try zig_args.append("--export-table"); + } + if (compile.initial_memory) |initial_memory| { + try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); + } + if (compile.max_memory) |max_memory| { + try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); + } + if (compile.shared_memory) { + try zig_args.append("--shared-memory"); + } + if (compile.global_base) |global_base| { + try zig_args.append(b.fmt("--global-base={d}", .{global_base})); + } + + if (compile.wasi_exec_model) |model| { + try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); + } + if (compile.linker_script) |linker_script| { + try zig_args.append("--script"); + try zig_args.append(linker_script.getPath2(b, step)); + } + + if (compile.version_script) |version_script| { + try zig_args.append("--version-script"); + try zig_args.append(version_script.getPath2(b, step)); + } + if (compile.linker_allow_undefined_version) |x| { + try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); + } + + if (compile.linker_enable_new_dtags) |enabled| { + try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); + } + + if (compile.kind == .@"test") { + if (compile.exec_cmd_args) |exec_cmd_args| { + for (exec_cmd_args) |cmd_arg| { + if (cmd_arg) |arg| { + try zig_args.append("--test-cmd"); + try zig_args.append(arg); + } else { + try zig_args.append("--test-cmd-bin"); + } + } + } + } + + if (b.sysroot) |sysroot| { + try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); + } + + // -I and -L arguments that appear after the last --mod argument apply to all modules. + const cwd: Io.Dir = .cwd(); + const io = b.graph.io; + + for (b.search_prefixes.items) |search_prefix| { + var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { + return step.fail("unable to open prefix directory '{s}': {s}", .{ + search_prefix, @errorName(err), + }); + }; + defer prefix_dir.close(io); + + // Avoid passing -L and -I flags for nonexistent directories. + // This prevents a warning, that should probably be upgraded to an error in Zig's + // CLI parsing code, when the linker sees an -L directory that does not exist. + + if (prefix_dir.access(io, "lib", .{})) |_| { + try zig_args.appendSlice(&.{ + "-L", b.pathJoin(&.{ search_prefix, "lib" }), + }); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ + search_prefix, @errorName(e), + }), + } + + if (prefix_dir.access(io, "include", .{})) |_| { + try zig_args.appendSlice(&.{ + "-I", b.pathJoin(&.{ search_prefix, "include" }), + }); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ + search_prefix, @errorName(e), + }), + } + } + + if (compile.rc_includes != .any) { + try zig_args.append("-rcincludes"); + try zig_args.append(@tagName(compile.rc_includes)); + } + + try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); + + if (compile.build_id orelse b.build_id) |build_id| { + try zig_args.append(switch (build_id) { + .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), + .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), + }); + } + + const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| + dir.getPath2(b, step) + else if (b.graph.zig_lib_directory.path) |_| + b.fmt("{f}", .{b.graph.zig_lib_directory}) + else + null; + + if (opt_zig_lib_dir) |zig_lib_dir| { + try zig_args.append("--zig-lib-dir"); + try zig_args.append(zig_lib_dir); + } + + try addFlag(&zig_args, "PIE", compile.pie); + + if (compile.lto) |lto| { + try zig_args.append(switch (lto) { + .full => "-flto=full", + .thin => "-flto=thin", + .none => "-fno-lto", + }); + } + + try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); + + if (compile.subsystem) |subsystem| { + try zig_args.append("--subsystem"); + try zig_args.append(@tagName(subsystem)); + } + + if (compile.mingw_unicode_entry_point) { + try zig_args.append("-municode"); + } + + if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ + "--error-limit", b.fmt("{d}", .{err_limit}), + }); + + try addFlag(&zig_args, "incremental", b.graph.incremental); + + try zig_args.append("--listen=-"); + + // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux + // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and + // pass that to zig, e.g. via 'zig build-lib @args.rsp' + // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html + var args_length: usize = 0; + for (zig_args.items) |arg| { + args_length += arg.len + 1; // +1 to account for null terminator + } + if (args_length >= 30 * 1024) { + try b.cache_root.handle.createDirPath(io, "args"); + + const args_to_escape = zig_args.items[2..]; + var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); + arg_blk: for (args_to_escape) |arg| { + for (arg, 0..) |c, arg_idx| { + if (c == '\\' or c == '"') { + // Slow path for arguments that need to be escaped. We'll need to allocate and copy + var escaped: std.ArrayList(u8) = .empty; + try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1); + try escaped.appendSlice(arena, arg[0..arg_idx]); + for (arg[arg_idx..]) |to_escape| { + if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\'); + try escaped.append(arena, to_escape); + } + escaped_args.appendAssumeCapacity(escaped.items); + continue :arg_blk; + } + } + escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument + } + + // Write the args to zig-cache/args/ to avoid conflicts with + // other zig build commands running in parallel. + const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); + const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); + + var args_hash: [Sha256.digest_length]u8 = undefined; + Sha256.hash(args, &args_hash, .{}); + var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; + _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); + + const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; + if (b.cache_root.handle.access(io, args_file, .{})) |_| { + // The args file is already present from a previous run. + } else |err| switch (err) { + error.FileNotFound => { + var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ + .replace = false, + .make_path = true, + }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ + b.cache_root, args_file, e, + }); + defer af.deinit(io); + + af.file.writeStreamingAll(io, args) catch |e| { + return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ + b.cache_root, args_file, e, + }); + }; + // Note we can't clean up this file, not even after build + // success, because that might interfere with another build + // process that needs the same file. + af.link(io) catch |e| switch (e) { + error.PathAlreadyExists => { + // The args file was created by another concurrent build process. + }, + else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ + b.cache_root, args_file, other_err, + }), + }; + }, + else => |other_err| return other_err, + } + + const resolved_args_file = try mem.concat(arena, u8, &.{ + "@", + try b.cache_root.join(arena, &.{args_file}), + }); + + zig_args.shrinkRetainingCapacity(2); + try zig_args.append(resolved_args_file); + } + + return try zig_args.toOwnedSlice(); +} + +pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { + c.step.result_error_msgs.clearRetainingCapacity(); + c.step.result_stderr = ""; + + c.step.result_error_bundle.deinit(gpa); + c.step.result_error_bundle = std.zig.ErrorBundle.empty; + + if (c.step.result_failed_command) |cmd| { + gpa.free(cmd); + c.step.result_failed_command = null; + } + + const zig_args = try getZigArgs(c, true); + const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); + return maybe_output_bin_path.?; +} + +pub fn doAtomicSymLinks( + step: *Step, + output_path: []const u8, + filename_major_only: []const u8, + filename_name_only: []const u8, +) !void { + const b = step.owner; + const io = b.graph.io; + const out_dir = fs.path.dirname(output_path) orelse "."; + const out_basename = fs.path.basename(output_path); + // sym link for libfoo.so.1 to libfoo.so.1.2.3 + const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); + const cwd: Io.Dir = .cwd(); + cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { + return step.fail("unable to symlink {s} -> {s}: {s}", .{ + major_only_path, out_basename, @errorName(err), + }); + }; + // sym link for libfoo.so to libfoo.so.1 + const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); + cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { + return step.fail("Unable to symlink {s} -> {s}: {s}", .{ + name_only_path, filename_major_only, @errorName(err), + }); + }; +} + +fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { + const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); + var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); + errdefer list.deinit(); + var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); + while (line_it.next()) |line| { + if (mem.trim(u8, line, " \t").len == 0) continue; + var tok_it = mem.tokenizeAny(u8, line, " \t"); + try list.append(PkgConfigPkg{ + .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, + .desc = tok_it.rest(), + }); + } + return list.toOwnedSlice(); +} + +fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { + if (b.pkg_config_pkg_list) |res| { + return res; + } + var code: u8 = undefined; + if (execPkgConfigList(b, &code)) |list| { + b.pkg_config_pkg_list = list; + return list; + } else |err| { + const result = switch (err) { + error.ProcessTerminated => error.PkgConfigCrashed, + error.ExecNotSupported => error.PkgConfigFailed, + error.ExitCodeFailure => error.PkgConfigFailed, + error.FileNotFound => error.PkgConfigNotInstalled, + error.InvalidName => error.PkgConfigNotInstalled, + error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, + else => return err, + }; + b.pkg_config_pkg_list = result; + return result; + } +} + +fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { + const cond = opt orelse return; + try args.ensureUnusedCapacity(1); + if (cond) { + args.appendAssumeCapacity("-f" ++ name); + } else { + args.appendAssumeCapacity("-fno-" ++ name); + } +} + +const PkgConfigResult = struct { + cflags: []const []const u8, + libs: []const []const u8, +}; + +/// Run pkg-config for the given library name and parse the output, returning the arguments +/// that should be passed to zig to link the given library. +fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult { + const wl_rpath_prefix = "-Wl,-rpath,"; + + const b = compile.step.owner; + const arena = b.allocator; + const pkg_name = match: { + // First we have to map the library name to pkg config name. Unfortunately, + // there are several examples where this is not straightforward: + // -lSDL2 -> pkg-config sdl2 + // -lgdk-3 -> pkg-config gdk-3.0 + // -latk-1.0 -> pkg-config atk + // -lpulse -> pkg-config libpulse + const pkgs = try getPkgConfigList(b); + + // Exact match means instant winner. + for (pkgs) |pkg| { + if (mem.eql(u8, pkg.name, lib_name)) { + break :match pkg.name; + } + } + + // Next we'll try ignoring case. + for (pkgs) |pkg| { + if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { + break :match pkg.name; + } + } + + // Prefixed "lib" or suffixed ".0". + for (pkgs) |pkg| { + if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { + const prefix = pkg.name[0..pos]; + const suffix = pkg.name[pos + lib_name.len ..]; + if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; + if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; + break :match pkg.name; + } + } + + // Trimming "-1.0". + if (mem.endsWith(u8, lib_name, "-1.0")) { + const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; + for (pkgs) |pkg| { + if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { + break :match pkg.name; + } + } + } + + return error.PackageNotFound; + }; + + var code: u8 = undefined; + const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = if (b.runAllowFail(&[_][]const u8{ + pkg_config_exe, + pkg_name, + "--cflags", + "--libs", + }, &code, .ignore)) |stdout| stdout else |err| switch (err) { + error.ProcessTerminated => return error.PkgConfigCrashed, + error.ExecNotSupported => return error.PkgConfigFailed, + error.ExitCodeFailure => return error.PkgConfigFailed, + error.FileNotFound => return error.PkgConfigNotInstalled, + else => return err, + }; + + var zig_cflags: std.ArrayList([]const u8) = .empty; + defer zig_cflags.deinit(arena); + var zig_libs: std.ArrayList([]const u8) = .empty; + defer zig_libs.deinit(arena); + + var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); + while (arg_it.next()) |arg| { + if (mem.eql(u8, arg, "-I")) { + const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_cflags.appendSlice(arena, &.{ "-I", dir }); + } else if (mem.startsWith(u8, arg, "-I")) { + try zig_cflags.append(arena, arg); + } else if (mem.eql(u8, arg, "-L")) { + const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_libs.appendSlice(arena, &.{ "-L", dir }); + } else if (mem.startsWith(u8, arg, "-L")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-l")) { + const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_libs.appendSlice(arena, &.{ "-l", lib }); + } else if (mem.startsWith(u8, arg, "-l")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-D")) { + const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_cflags.appendSlice(arena, &.{ "-D", macro }); + } else if (mem.startsWith(u8, arg, "-D")) { + try zig_cflags.append(arena, arg); + } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { + try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); + } else if (b.debug_pkg_config) { + return compile.step.fail("unknown pkg-config flag '{s}'", .{arg}); + } + } + + try zig_cflags.shrinkToLen(arena); + try zig_libs.shrinkToLen(arena); + + return .{ + .cflags = zig_cflags.toOwnedSliceAssert(), + .libs = zig_libs.toOwnedSliceAssert(), + }; +} + + diff --git a/lib/compiler/maker/Step/InstallArtifact.zig b/lib/compiler/maker/Step/InstallArtifact.zig new file mode 100644 index 0000000000000000000000000000000000000000..ba3846c1a574f9934f4196f03529dac3abcbe275 --- /dev/null +++ b/lib/compiler/maker/Step/InstallArtifact.zig @@ -0,0 +1,96 @@ + +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; + const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); + const b = step.owner; + const io = b.graph.io; + + var all_cached = true; + + if (install_artifact.dest_dir) |dest_dir| { + const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); + const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); + all_cached = all_cached and p == .fresh; + + if (install_artifact.dylib_symlinks) |dls| { + try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); + } + + install_artifact.artifact.installed_path = full_dest_path; + } + + if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { + const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.implib_dir) |implib_dir| { + const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.pdb_dir) |pdb_dir| { + const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.h_dir) |h_dir| { + if (install_artifact.emitted_h) |emitted_h| { + const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); + const p = try step.installFile(emitted_h, full_h_path); + all_cached = all_cached and p == .fresh; + } + + for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { + .file => |file| { + const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); + const p = try step.installFile(file.source, full_h_path); + all_cached = all_cached and p == .fresh; + }, + .directory => |dir| { + const src_dir_path = dir.source.getPath3(b, step); + const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); + + var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{f}': {s}", .{ + src_dir_path, @errorName(err), + }); + }; + defer src_dir.close(io); + + var it = try src_dir.walk(b.allocator); + next_entry: while (try it.next(io)) |entry| { + for (dir.options.exclude_extensions) |ext| { + if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; + } + if (dir.options.include_extensions) |incs| { + for (incs) |inc| { + if (std.mem.endsWith(u8, entry.path, inc)) break; + } else { + continue :next_entry; + } + } + + const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); + switch (entry.kind) { + .directory => { + try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); + const p = try step.installDir(full_dest_path); + all_cached = all_cached and p == .existed; + }, + .file => { + const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); + all_cached = all_cached and p == .fresh; + }, + else => continue, + } + } + }, + }; + } + + step.result_cached = all_cached; +} diff --git a/lib/compiler/maker/Step/Run.zig b/lib/compiler/maker/Step/Run.zig new file mode 100644 index 0000000000000000000000000000000000000000..79b4d57b7e6fa772191b434bc90d7432b3442af3 --- /dev/null +++ b/lib/compiler/maker/Step/Run.zig @@ -0,0 +1,2127 @@ +const Run = @This(); + +const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; +const Dir = std.Io.Dir; +const mem = std.mem; +const process = std.process; +const EnvMap = std.process.Environ.Map; +const assert = std.debug.assert; +const Cache = std.Build.Cache; +const Path = std.Build.Cache.Path; + +const Step = @import("../Step.zig"); + +/// If this is a Zig unit test binary, this tracks the names of the unit +/// tests that are also fuzz tests. Indexes cannot be used as they may +/// change between reruns. +fuzz_tests: std.ArrayList([]const u8), +cached_test_metadata: ?CachedTestMetadata = null, + + +fn make(step: *Step, options: Step.MakeOptions) !void { + const b = step.owner; + const io = b.graph.io; + const arena = b.allocator; + const run: *Run = @fieldParentPtr("step", step); + const has_side_effects = run.hasSideEffects(); + + var argv_list = std.array_list.Managed([]const u8).init(arena); + var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); + + var man = b.graph.cache.obtain(); + defer man.deinit(); + + if (run.environ_map) |environ_map| { + for (environ_map.keys(), environ_map.values()) |key, value| { + man.hash.addBytes(key); + man.hash.addBytes(value); + } + } + + man.hash.add(run.color); + man.hash.add(run.disable_zig_progress); + + for (run.argv.items) |arg| { + switch (arg) { + .bytes => |bytes| { + try argv_list.append(bytes); + man.hash.addBytes(bytes); + }, + .lazy_path => |file| { + const file_path = file.lazy_path.getPath3(b, step); + try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); + man.hash.addBytes(file.prefix); + _ = try man.addFilePath(file_path, null); + }, + .decorated_directory => |dd| { + const file_path = dd.lazy_path.getPath3(b, step); + const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); + try argv_list.append(resolved_arg); + man.hash.addBytes(resolved_arg); + }, + .file_content => |file_plp| { + const file_path = file_plp.lazy_path.getPath3(b, step); + + var result: std.Io.Writer.Allocating = .init(arena); + errdefer result.deinit(); + result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + + const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { + return step.fail( + "unable to open input file '{f}': {t}", + .{ file_path, err }, + ); + }; + defer file.close(io); + + var buf: [1024]u8 = undefined; + var file_reader = file.reader(io, &buf); + _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { + error.ReadFailed => return step.fail( + "failed to read from '{f}': {t}", + .{ file_path, file_reader.err.? }, + ), + error.WriteFailed => return error.OutOfMemory, + }; + + try argv_list.append(result.written()); + man.hash.addBytes(file_plp.prefix); + _ = try man.addFilePath(file_path, null); + }, + .artifact => |pa| { + const artifact = pa.artifact; + + if (artifact.rootModuleTarget().os.tag == .windows) { + // On Windows we don't have rpaths so we have to add .dll search paths to PATH + run.addPathForDynLibs(artifact); + } + const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; + + try argv_list.append(b.fmt("{s}{s}", .{ + pa.prefix, + run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), + })); + + _ = try man.addFile(file_path, null); + }, + .output_file, .output_directory => |output| { + man.hash.addBytes(output.prefix); + man.hash.addBytes(output.basename); + // Add a placeholder into the argument list because we need the + // manifest hash to be updated with all arguments before the + // object directory is computed. + try output_placeholders.append(.{ + .index = argv_list.items.len, + .tag = arg, + .output = output, + }); + _ = try argv_list.addOne(); + }, + } + } + + switch (run.stdin) { + .bytes => |bytes| { + man.hash.addBytes(bytes); + }, + .lazy_path => |lazy_path| { + const file_path = lazy_path.getPath2(b, step); + _ = try man.addFile(file_path, null); + }, + .none => {}, + } + + if (run.captured_stdout) |captured| { + man.hash.addBytes(captured.output.basename); + man.hash.add(captured.trim_whitespace); + } + + if (run.captured_stderr) |captured| { + man.hash.addBytes(captured.output.basename); + man.hash.add(captured.trim_whitespace); + } + + hashStdIo(&man.hash, run.stdio); + + for (run.file_inputs.items) |lazy_path| { + _ = try man.addFile(lazy_path.getPath2(b, step), null); + } + + if (run.cwd) |cwd| { + const cwd_path = cwd.getPath3(b, step); + _ = man.hash.addBytes(try cwd_path.toString(arena)); + } + + if (!has_side_effects and try step.cacheHitAndWatch(&man)) { + // cache hit, skip running command + const digest = man.final(); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); + + step.result_cached = true; + return; + } + + const dep_output_file = run.dep_output_file orelse { + // We already know the final output paths, use them directly. + const digest = if (has_side_effects) + man.hash.final() + else + man.final(); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); + + const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; + for (output_placeholders.items) |placeholder| { + const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); + const output_sub_dir_path = switch (placeholder.tag) { + .output_file => Dir.path.dirname(output_sub_path).?, + .output_directory => output_sub_path, + else => unreachable, + }; + b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, output_sub_dir_path, @errorName(err), + }); + }; + const arg_output_path = run.convertPathArg(.{ + .root_dir = .cwd(), + .sub_path = placeholder.output.generated_file.getPath(), + }); + argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) + arg_output_path + else + b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); + } + + try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); + if (!has_side_effects) try step.writeManifestAndWatch(&man); + return; + }; + + // We do not know the final output paths yet, use temp paths to run the command. + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + + for (output_placeholders.items) |placeholder| { + const output_components = .{ tmp_dir_path, placeholder.output.basename }; + const output_sub_path = b.pathJoin(&output_components); + const output_sub_dir_path = switch (placeholder.tag) { + .output_file => Dir.path.dirname(output_sub_path).?, + .output_directory => output_sub_path, + else => unreachable, + }; + b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, output_sub_dir_path, @errorName(err), + }); + }; + const raw_output_path: Cache.Path = .{ + .root_dir = b.cache_root, + .sub_path = b.pathJoin(&output_components), + }; + placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); + argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ + placeholder.output.prefix, + run.convertPathArg(raw_output_path), + }); + } + + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); + + const dep_file_dir = Dir.cwd(); + const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); + if (has_side_effects) + try man.addDepFile(dep_file_dir, dep_file_basename) + else + try man.addDepFilePost(dep_file_dir, dep_file_basename); + + const digest = if (has_side_effects) + man.hash.final() + else + man.final(); + + const any_output = output_placeholders.items.len > 0 or + run.captured_stdout != null or run.captured_stderr != null; + + // Rename into place + if (any_output) { + const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; + + b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { + Dir.RenameError.DirNotEmpty => { + b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { + return step.fail("unable to remove dir '{f}'{s}: {t}", .{ + b.cache_root, tmp_dir_path, del_err, + }); + }; + b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { + return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ + b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, + }); + }; + }, + else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ + b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, + }), + }; + } + + if (!has_side_effects) try step.writeManifestAndWatch(&man); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); +} + +/// Reads stdout of a Zig test process until a termination condition is reached: +/// * A write fails, indicating the child unexpectedly closed stdin +/// * A test (or a response from the test runner) times out +/// * The wait fails, indicating the child closed stdout and stderr +fn waitZigTest( + run: *Run, + child: *process.Child, + options: Step.MakeOptions, + multi_reader: *Io.File.MultiReader, + opt_metadata: *?TestMetadata, + results: *Step.TestResults, +) !union(enum) { + write_failed: anyerror, + no_poll: struct { + active_test_index: ?u32, + ns_elapsed: u64, + }, + timeout: struct { + active_test_index: ?u32, + ns_elapsed: u64, + }, +} { + const gpa = run.step.owner.allocator; + const arena = run.step.owner.allocator; + const io = run.step.owner.graph.io; + + var sub_prog_node: ?std.Progress.Node = null; + defer if (sub_prog_node) |n| n.end(); + + if (opt_metadata.*) |*md| { + // Previous unit test process died or was killed; we're continuing where it left off + requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; + } else { + // Running unit tests normally + run.fuzz_tests.clearRetainingCapacity(); + sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err }; + } + + var active_test_index: ?u32 = null; + + var last_update: Io.Clock.Timestamp = .now(io, .awake); + + // This timeout is used when we're waiting on the test runner itself rather than a user-specified + // test. For instance, if the test runner leaves this much time between us requesting a test to + // start and it acknowledging the test starting, we terminate the child and raise an error. This + // *should* never happen, but could in theory be caused by some very unlucky IB in a test. + const response_timeout: Io.Clock.Duration = t: { + if (fuzz_context != null) break :t null; // don't timeout fuzz tests + const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); + break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; + }; + const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ + .clock = .awake, + .raw = .fromNanoseconds(ns), + } else null; + + const stdout = multi_reader.reader(0); + const stderr = multi_reader.reader(1); + const Header = std.zig.Server.Message.Header; + + while (true) { + const timeout: Io.Timeout = t: { + const opt_duration = if (active_test_index == null) response_timeout else test_timeout; + const duration = opt_duration orelse break :t .none; + break :t .{ .deadline = last_update.addDuration(duration) }; + }; + + // This block is exited when `stdout` contains enough bytes for a `Header`. + header_ready: { + if (stdout.buffered().len >= @sizeOf(Header)) { + // We already have one, no need to poll! + break :header_ready; + } + + multi_reader.fill(64, timeout) catch |err| switch (err) { + error.Timeout => return .{ .timeout = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + error.EndOfStream => return .{ .no_poll = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + else => |e| return e, + }; + + continue; + } + // There is definitely a header available now -- read it. + const header = stdout.takeStruct(Header, .little) catch unreachable; + + while (stdout.buffered().len < header.bytes_len) { + multi_reader.fill(64, timeout) catch |err| switch (err) { + error.Timeout => return .{ .timeout = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + error.EndOfStream => return .{ .no_poll = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + else => |e| return e, + }; + } + + const body = stdout.take(header.bytes_len) catch unreachable; + var body_r: std.Io.Reader = .fixed(body); + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + }, + .test_metadata => { + // `metadata` would only be populated if we'd already seen a `test_metadata`, but we + // only request it once (and importantly, we don't re-request it if we kill and + // restart the test runner). + assert(opt_metadata.* == null); + + const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable; + results.test_count = tm_hdr.tests_len; + + const names = try arena.alloc(u32, results.test_count); + for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; + + const expected_panic_msgs = try arena.alloc(u32, results.test_count); + for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; + + const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; + + options.progress_node.setEstimatedTotalItems(names.len); + opt_metadata.* = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .ns_per_test = try arena.alloc(u64, results.test_count), + .names = names, + .expected_panic_msgs = expected_panic_msgs, + .next_index = 0, + .prog_node = options.progress_node, + }; + @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); + + active_test_index = null; + last_update = .now(io, .awake); + + requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err }; + }, + .test_started => { + active_test_index = opt_metadata.*.?.next_index - 1; + last_update = .now(io, .awake); + }, + .test_results => { + const md = &opt_metadata.*.?; + + const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable; + assert(tr_hdr.index == active_test_index); + + switch (tr_hdr.flags.status) { + .pass => {}, + .skip => results.skip_count +|= 1, + .fail => results.fail_count +|= 1, + } + const leak_count = tr_hdr.flags.leak_count; + const log_err_count = tr_hdr.flags.log_err_count; + results.leak_count +|= leak_count; + results.log_err_count +|= log_err_count; + + if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index)); + + if (tr_hdr.flags.status == .fail) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + if (stderr_bytes.len == 0) { + try run.step.addError("'{s}' failed without output", .{name}); + } else { + try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); + } + } else if (leak_count > 0) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); + } else if (log_err_count > 0) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); + } + + active_test_index = null; + + const now: Io.Clock.Timestamp = .now(io, .awake); + md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds); + last_update = now; + + requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; + }, + else => {}, // ignore other messages + } + } +} + +const FuzzTestRunner = struct { + run: *Run, + ctx: FuzzContext, + coverage_id: ?u64, + + instances: []Instance, + /// The indexes of this are layed out such that it is effectively an array + /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr. + batch: Io.Batch, + /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter. + pending_broadcasts: std.ArrayList(u8), + broadcast: std.ArrayList(u8), + broadcast_undelivered: u32, + + const Instance = struct { + child: process.Child, + message: std.ArrayListAligned(u8, .@"4"), + broadcast_written: usize, + stderr: std.ArrayList(u8), + stdin_vec: [1][]u8, + stdout_vec: [1][]u8, + stderr_vec: [1][]u8, + progress_node: std.Progress.Node, + + fn messageHeader(instance: *Instance) InHeader { + assert(instance.message.items.len >= @sizeOf(InHeader)); + const header_ptr: *InHeader = @ptrCast(instance.message.items); + var header = header_ptr.*; + if (std.builtin.Endian.native != .little) { + std.mem.byteSwapAllFields(InHeader, &header); + } + return header; + } + }; + + const PendingBroadcastFooter = struct { + from_id: u32, + body_len: u32, + }; + + const InHeader = std.zig.Server.Message.Header; + const OutHeader = std.zig.Client.Message.Header; + + const stdin_i = 0; + const stdout_i = 1; + const stderr_i = 2; + + fn init( + run: *Run, + ctx: FuzzContext, + progress_node: std.Progress.Node, + spawn_options: process.SpawnOptions, + ) !FuzzTestRunner { + const step_owner = run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + + const n_instances = switch (ctx.fuzz.mode) { + .forever => step_owner.graph.max_jobs orelse @min( + std.Thread.getCpuCount() catch 1, + (std.math.maxInt(u32) - 2) / 3, + ), + .limit => 1, + }; + const instances = try gpa.alloc(Instance, n_instances); + errdefer gpa.free(instances); + const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3); + errdefer gpa.free(batch_storage); + + @memset(instances, .{ + .child = undefined, + .message = .empty, + .broadcast_written = undefined, + .stderr = .empty, + .stdin_vec = undefined, + .stdout_vec = undefined, + .stderr_vec = undefined, + .progress_node = undefined, + }); + for (0.., instances) |id, *instance| { + errdefer for (instances[0..id]) |*spawned| { + spawned.child.kill(io); + spawned.progress_node.end(); + }; + instance.child = try process.spawn(io, spawn_options); + instance.progress_node = progress_node.start("starting fuzzer", 0); + } + + return .{ + .run = run, + .ctx = ctx, + .coverage_id = null, + + .instances = instances, + .batch = .init(batch_storage), + .pending_broadcasts = .empty, + .broadcast = .empty, + .broadcast_undelivered = 0, + }; + } + + fn deinit(f: *FuzzTestRunner) void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + + f.batch.cancel(io); + gpa.free(f.batch.storage); + var total_rss: usize = 0; + for (f.instances) |*instance| { + instance.child.kill(io); + instance.message.deinit(gpa); + instance.stderr.deinit(gpa); + instance.progress_node.end(); + total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; + } + f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); + gpa.free(f.instances); + } + + fn startInstances(f: *FuzzTestRunner) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + + for (0.., f.instances) |id, *instance| { + const id32: u32 = @intCast(id); + (switch (f.ctx.fuzz.mode) { + .forever => sendRunFuzzTestMessage( + io, + instance.child.stdin.?, + f.run.fuzz_tests.items, + .forever, + id32, + ), + .limit => |limit| sendRunFuzzTestMessage( + io, + instance.child.stdin.?, + f.run.fuzz_tests.items, + .iterations, + limit.amount, + ), + }) catch |write_err| { + // The runner unexpectedly closed stdin, which means it crashed during initialization. + // Clean up everything and wait for the child to exit. + instance.child.stdin.?.close(io); + instance.child.stdin = null; + const term = try instance.child.wait(io); + return f.run.step.fail( + "unable to write stdin ({t}); test process unexpectedly {f}", + .{ write_err, fmtTerm(term) }, + ); + }; + + try f.addStdoutRead(id32, @sizeOf(InHeader)); + try f.addStderrRead(id32); + } + } + + fn listen(f: *FuzzTestRunner) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + + while (true) { + try f.batch.awaitConcurrent(io, .none); + while (f.batch.next()) |completion| { + const id = completion.index / 3; + const result = completion.result; + switch (completion.index % 3) { + 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) { + // Avoid calling `instanceEos` until EndOfStream is seen with stderr so + // that all stderr is collected. + error.BrokenPipe => continue, + else => |write_e| return write_e, + }), + 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) { + // Avoid calling `instanceEos` until EndOfStream is seen with stderr so + // that all stderr is collected. + error.EndOfStream => continue, + else => |read_e| return read_e, + }), + 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { + error.EndOfStream => return f.instanceEos(id), + else => |read_e| return read_e, + }), + else => unreachable, + } + } + } + } + + fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + const instance = &f.instances[id]; + + instance.message.items.len += n; + const total_read = instance.message.items.len; + if (total_read < @sizeOf(InHeader)) { + try f.addStdoutRead(id, @sizeOf(InHeader)); + return; + } + + const header = instance.messageHeader(); + const body = instance.message.items[@sizeOf(InHeader)..]; + if (body.len != header.bytes_len) { + try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len); + return; + } + + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + }, + .coverage_id => { + var body_r: Io.Reader = .fixed(body); + f.coverage_id = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable; + + const fuzz = f.ctx.fuzz; + fuzz.queue_mutex.lockUncancelable(io); + defer fuzz.queue_mutex.unlock(io); + try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ + .id = f.coverage_id.?, + .cumulative = .{ + .runs = cumulative_runs, + .unique = cumulative_unique, + .coverage = cumulative_coverage, + }, + .run = f.run, + } }); + fuzz.queue_cond.signal(io); + }, + .fuzz_start_addr => { + var body_r: Io.Reader = .fixed(body); + const fuzz = f.ctx.fuzz; + const addr = body_r.takeInt(u64, .little) catch unreachable; + + fuzz.queue_mutex.lockUncancelable(io); + defer fuzz.queue_mutex.unlock(io); + try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ + .addr = addr, + .coverage_id = f.coverage_id.?, + } }); + fuzz.queue_cond.signal(io); + }, + .fuzz_test_change => { + const test_i = std.mem.readInt(u32, body[0..4], .little); + instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); + }, + .broadcast_fuzz_input => { + if (f.instances.len == 1) { + // No other processes to broadcast to. + } else if (f.broadcast_undelivered == 0) { + try f.instanceBroadcast(id, body); + } else { + const footer: PendingBroadcastFooter = .{ + .from_id = id, + .body_len = @intCast(body.len), + }; + // There is another broadcast in progress so add this one to the queue. + const size = @sizeOf(PendingBroadcastFooter) + body.len; + try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); + f.pending_broadcasts.appendSliceAssumeCapacity(body); + f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); + } + }, + else => {}, // ignore other messages + } + + instance.message.clearRetainingCapacity(); + try f.addStdoutRead(id, @sizeOf(InHeader)); + } + + fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void { + const instance = &f.instances[id]; + instance.stderr.items.len += n; + try f.addStderrRead(id); + } + + fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void { + const instance = &f.instances[id]; + + instance.broadcast_written += n; + if (instance.broadcast_written == f.broadcast.items.len) { + f.broadcast_undelivered -= 1; + if (f.broadcast_undelivered == 0) { + try f.broadcastComplete(); + } + } else { + f.addStdinWrite(id); + } + } + + fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const instance = &f.instances[id]; + + try instance.message.ensureTotalCapacity(gpa, end); + const start = instance.message.items.len; + instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]}; + f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{ + .file = instance.child.stdout.?, + .data = &instance.stdout_vec, + } }); + } + + fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const instance = &f.instances[id]; + + try instance.stderr.ensureUnusedCapacity(gpa, 1); + instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()}; + f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{ + .file = instance.child.stderr.?, + .data = &instance.stderr_vec, + } }); + } + + fn addStdinWrite(f: *FuzzTestRunner, id: u32) void { + const instance = &f.instances[id]; + + assert(f.broadcast.items.len != instance.broadcast_written); + instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]}; + f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{ + .file = instance.child.stdin.?, + .data = &instance.stdin_vec, + } }); + } + + fn instanceEos(f: *FuzzTestRunner, id: u32) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + const instance = &f.instances[id]; + + instance.child.stdin.?.close(io); + instance.child.stdin = null; + const term = try instance.child.wait(io); + if (!termMatches(.{ .exited = 0 }, term)) { + f.run.step.result_stderr = try f.mergedStderr(); + try f.saveCrash(id, term); + return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); + } + } + + fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { + const step = &f.run.step; + const b = step.owner; + const io = b.graph.io; + + if (f.coverage_id == null) return; + + // Search for the input file corresponding to the instance + const InputHeader = Build.abi.fuzz.MmapInputHeader; + var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; + var in_r: Io.File.Reader = undefined; + var in_f: Io.File = undefined; + var in_name_buf: [12]u8 = undefined; + var in_name: []const u8 = undefined; + var i: u32 = 0; + const header: InputHeader = while (true) : ({ + if (i == std.math.maxInt(u32)) return; + i += 1; + }) { + const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; + in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; + in_f = b.cache_root.handle.openFile(io, in_name, .{ + .lock = .exclusive, + .lock_nonblocking = true, + }) catch |e| switch (e) { + error.FileNotFound => return, + error.WouldBlock => continue, // Can not be from + // the crashed instance since it is still locked. + else => return step.fail("failed to open file '{f}{s}': {t}", .{ + b.cache_root, in_name, e, + }), + }; + + in_r = in_f.readerStreaming(io, &in_r_buf); + const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { + in_f.close(io); + switch (e) { + error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ + b.cache_root, in_name, in_r.err.?, + }), + error.EndOfStream => continue, + } + }; + + if (header.pc_digest == f.coverage_id.? and + header.instance_id == id and + header.test_i < f.run.fuzz_tests.items.len) + { + break header; + } + + in_f.close(io); + }; + defer in_f.close(io); + + // Save it to a seperate file + const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; + const out = b.cache_root.handle.createFile(io, crash_name, .{ + .lock = .exclusive, // Multiple run steps could have found a crash at the same time + }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ + b.cache_root, crash_name, e, + }); + defer out.close(io); + + var out_w_buf: [512]u8 = undefined; + var out_w = out.writerStreaming(io, &out_w_buf); + _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { + error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ + b.cache_root, in_name, in_r.err.?, + }), + error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ + b.cache_root, crash_name, out_w.err.?, + }), + }; + + return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ + f.run.fuzz_tests.items[header.test_i], + fmtTerm(term), + b.cache_root, + crash_name, + }); + } + + fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void { + assert(f.instances.len > 1); + assert(f.broadcast_undelivered == 0); // no other broadcast is progress + assert(f.broadcast.items.len == 0); + assert(from_id < f.instances.len); + + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + + var out_header: OutHeader = .{ + .tag = .new_fuzz_input, + .bytes_len = @intCast(bytes.len), + }; + if (std.builtin.Endian.native != .little) { + std.mem.byteSwapAllFields(OutHeader, &out_header); + } + try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len); + f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header)); + f.broadcast.appendSliceAssumeCapacity(bytes); + + f.broadcast_undelivered = @intCast(f.instances.len - 1); + for (0.., f.instances) |to_id, *instance| { + if (to_id == from_id) continue; + instance.broadcast_written = 0; + f.addStdinWrite(@intCast(to_id)); + } + } + + fn broadcastComplete(f: *FuzzTestRunner) !void { + assert(f.instances.len > 1); + assert(f.broadcast_undelivered == 0); + f.broadcast.clearRetainingCapacity(); + + const pending = &f.pending_broadcasts; + if (pending.items.len != 0) { + // Another broadcast is pending; copy it over to `broadcast` + + const footer_len = @sizeOf(PendingBroadcastFooter); + const footer_bytes = pending.items[pending.items.len - footer_len ..]; + const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes); + pending.items.len -= footer_len; + + const body = pending.items[pending.items.len - footer.body_len ..]; + try f.instanceBroadcast(footer.from_id, body); + pending.items.len -= body.len; + } + } + + fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { + const step_owner = f.run.step.owner; + const arena = step_owner.allocator; + + // Collect any available stderr + while (f.batch.next()) |completion| { + if (completion.index % 3 != 2) continue; + const len = completion.result.file_read_streaming catch continue; + f.instances[completion.index / 3].stderr.items.len += len; + } + + var stderr_len: usize = 0; + for (f.instances) |*instance| stderr_len += instance.stderr.items.len; + const stderr = try arena.alloc(u8, stderr_len); + + stderr_len = 0; + for (f.instances) |*instance| { + @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items); + stderr_len += instance.stderr.items.len; + } + return stderr; + } +}; + +fn evalFuzzTest( + run: *Run, + spawn_options: process.SpawnOptions, + options: Step.MakeOptions, + fuzz_context: FuzzContext, +) !void { + var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); + defer f.deinit(); + try f.startInstances(); + try f.listen(); +} + +const StdioPollEnum = enum { stdout, stderr }; + +fn evalZigTest( + run: *Run, + spawn_options: process.SpawnOptions, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !void { + if (fuzz_context != null) { + try evalFuzzTest(run, spawn_options, options, fuzz_context.?); + return; + } + + const step_owner = run.step.owner; + const gpa = step_owner.allocator; + const arena = step_owner.allocator; + const io = step_owner.graph.io; + + // We will update this every time a child runs. + run.step.result_peak_rss = 0; + + var test_results: Step.TestResults = .{ + .test_count = 0, + .skip_count = 0, + .fail_count = 0, + .crash_count = 0, + .timeout_count = 0, + .leak_count = 0, + .log_err_count = 0, + }; + var test_metadata: ?TestMetadata = null; + + while (true) { + var child = try process.spawn(io, spawn_options); + var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; + var multi_reader: Io.File.MultiReader = undefined; + multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); + var child_killed = false; + defer if (!child_killed) { + child.kill(io); + multi_reader.deinit(); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + }; + + switch (try waitZigTest( + run, + &child, + options, + &multi_reader, + &test_metadata, + &test_results, + )) { + .write_failed => |err| { + // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured + // all available stderr to make our error output as useful as possible. + const stderr_fr = multi_reader.fileReader(1); + while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) { + error.ReadFailed => return stderr_fr.err.?, + error.EndOfStream => {}, + } + run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); + + // Clean up everything and wait for the child to exit. + child.stdin.?.close(io); + child.stdin = null; + multi_reader.deinit(); + child_killed = true; + const term = try child.wait(io); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + + // The individual unit test results are irrelevant: the test runner itself broke! + // Fail immediately without populating `s.test_results`. + return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); + }, + .no_poll => |no_poll| { + // This might be a success (we requested exit and the child dutifully closed stdout) or + // a crash of some kind. Either way, the child will terminate by itself -- wait for it. + const stderr_reader = multi_reader.reader(1); + const stderr_owned = try arena.dupe(u8, stderr_reader.buffered()); + + // Clean up everything and wait for the child to exit. + child.stdin.?.close(io); + child.stdin = null; + multi_reader.deinit(); + child_killed = true; + const term = try child.wait(io); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + + if (no_poll.active_test_index) |test_index| { + // A test was running, so this is definitely a crash. Report it against that + // test, and continue to the next test. + test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; + test_results.crash_count += 1; + try run.step.addError("'{s}' {f}{s}{s}", .{ + test_metadata.?.testName(test_index), + fmtTerm(term), + if (stderr_owned.len != 0) " with stderr:\n" else "", + std.mem.trim(u8, stderr_owned, "\n"), + }); + continue; + } + + // Report an error if the child terminated uncleanly or if we were still trying to run more tests. + run.step.result_stderr = stderr_owned; + const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); + if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { + // The individual unit test results are irrelevant: the test runner itself broke! + // Fail immediately without populating `s.test_results`. + return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); + } + + // We're done with all of the tests! Commit the test results and return. + run.step.test_results = test_results; + if (test_metadata) |tm| { + run.cached_test_metadata = tm.toCachedTestMetadata(); + if (options.web_server) |ws| { + if (run.step.owner.graph.time_report) { + ws.updateTimeReportRunTest( + run, + &run.cached_test_metadata.?, + tm.ns_per_test, + ); + } + } + } + return; + }, + .timeout => |timeout| { + const stderr_reader = multi_reader.reader(1); + const stderr = stderr_reader.buffered(); + stderr_reader.tossBuffered(); + if (timeout.active_test_index) |test_index| { + // A test was running. Report the timeout against that test, and continue on to + // the next test. + test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; + test_results.timeout_count += 1; + try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ + test_metadata.?.testName(test_index), + Io.Duration{ .nanoseconds = timeout.ns_elapsed }, + if (stderr.len != 0) " with stderr:\n" else "", + std.mem.trim(u8, stderr, "\n"), + }); + continue; + } + // Just log an error and let the child be killed. + run.step.result_stderr = try arena.dupe(u8, stderr); + // The individual unit test results in `results` are irrelevant: the test runner + // is broken! Fail immediately without populating `s.test_results`. + return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); + }, + } + comptime unreachable; + } +} + +const TestMetadata = struct { + names: []const u32, + ns_per_test: []u64, + expected_panic_msgs: []const u32, + string_bytes: []const u8, + next_index: u32, + prog_node: std.Progress.Node, + + fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { + return .{ + .names = tm.names, + .string_bytes = tm.string_bytes, + }; + } + + fn testName(tm: TestMetadata, index: u32) []const u8 { + return tm.toCachedTestMetadata().testName(index); + } +}; + +pub const CachedTestMetadata = struct { + names: []const u32, + string_bytes: []const u8, + + pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { + return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); + } +}; + +fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { + while (metadata.next_index < metadata.names.len) { + const i = metadata.next_index; + metadata.next_index += 1; + + if (metadata.expected_panic_msgs[i] != 0) continue; + + const name = metadata.testName(i); + if (sub_prog_node.*) |n| n.end(); + sub_prog_node.* = metadata.prog_node.start(name, 0); + + try sendRunTestMessage(io, in, .run_test, i); + return; + } else { + metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done + try sendMessage(io, in, .exit); + } +} + +fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 4, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u32, index, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +fn sendRunFuzzTestMessage( + io: Io, + file: Io.File, + test_names: []const []const u8, + kind: std.Build.abi.fuzz.LimitKind, + amount_or_instance: u64, +) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = .start_fuzzing, + .bytes_len = 1 + 8 + 4 + count: { + var c: u32 = @intCast(test_names.len * 4); + for (test_names) |name| { + c += @intCast(name.len); + } + break :count c; + }, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + for (test_names) |test_name| { + w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeAll(test_name) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + } +} + +fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { + const b = run.step.owner; + const io = b.graph.io; + const arena = b.allocator; + const gpa = b.allocator; + + var child = try process.spawn(io, spawn_options); + defer child.kill(io); + + switch (run.stdin) { + .bytes => |bytes| { + child.stdin.?.writeStreamingAll(io, bytes) catch |err| { + return run.step.fail("unable to write stdin: {t}", .{err}); + }; + child.stdin.?.close(io); + child.stdin = null; + }, + .lazy_path => |lazy_path| { + const path = lazy_path.getPath3(b, &run.step); + const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { + return run.step.fail("unable to open stdin file: {t}", .{err}); + }; + defer file.close(io); + // TODO https://github.com/ziglang/zig/issues/23955 + var read_buffer: [1024]u8 = undefined; + var file_reader = file.reader(io, &read_buffer); + var write_buffer: [1024]u8 = undefined; + var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); + _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { + error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ + path, file_reader.err.?, + }), + error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ + stdin_writer.err.?, + }), + }; + stdin_writer.interface.flush() catch |err| switch (err) { + error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ + stdin_writer.err.?, + }), + }; + child.stdin.?.close(io); + child.stdin = null; + }, + .none => {}, + } + + var stdout_bytes: ?[]const u8 = null; + var stderr_bytes: ?[]const u8 = null; + + if (child.stdout) |stdout| { + if (child.stderr) |stderr| { + var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; + var multi_reader: Io.File.MultiReader = undefined; + multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); + defer multi_reader.deinit(); + + const stdout_reader = multi_reader.reader(0); + const stderr_reader = multi_reader.reader(1); + + while (multi_reader.fill(64, .none)) |_| { + if (run.stdio_limit.toInt()) |limit| { + if (stdout_reader.buffered().len > limit) + return error.StdoutStreamTooLong; + if (stderr_reader.buffered().len > limit) + return error.StderrStreamTooLong; + } + } else |err| switch (err) { + error.Timeout => unreachable, + error.EndOfStream => {}, + else => |e| return e, + } + + try multi_reader.checkAnyError(); + + // TODO: this string can leak since alloc below can return error. + stdout_bytes = try multi_reader.toOwnedSlice(0); + // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. + stderr_bytes = try multi_reader.toOwnedSlice(1); + } else { + var stdout_reader = stdout.readerStreaming(io, &.{}); + stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return stdout_reader.err.?, + error.StreamTooLong => return error.StdoutStreamTooLong, + }; + } + } else if (child.stderr) |stderr| { + var stderr_reader = stderr.readerStreaming(io, &.{}); + stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return stderr_reader.err.?, + error.StreamTooLong => return error.StderrStreamTooLong, + }; + } + + if (stderr_bytes) |bytes| if (bytes.len > 0) { + // Treat stderr as an error message. + const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { + .check => |checks| !checksContainStderr(checks.items), + else => true, + }; + if (stderr_is_diagnostic) { + run.step.result_stderr = bytes; + } + }; + + run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; + + return .{ + .term = try child.wait(io), + .stdout = stdout_bytes, + .stderr = stderr_bytes, + }; +} + +const IndexedOutput = struct { + index: usize, + tag: @typeInfo(Arg).@"union".tag_type.?, + output: *Output, +}; + +pub fn rerunInFuzzMode( + run: *Run, + fuzz: *std.Build.Fuzz, + prog_node: std.Progress.Node, +) !void { + const step = &run.step; + const b = step.owner; + const io = b.graph.io; + const arena = b.allocator; + var argv_list: std.ArrayList([]const u8) = .empty; + for (run.argv.items) |arg| { + switch (arg) { + .bytes => |bytes| { + try argv_list.append(arena, bytes); + }, + .lazy_path => |file| { + const file_path = file.lazy_path.getPath3(b, step); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); + }, + .decorated_directory => |dd| { + const file_path = dd.lazy_path.getPath3(b, step); + try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); + }, + .file_content => |file_plp| { + const file_path = file_plp.lazy_path.getPath3(b, step); + + var result: std.Io.Writer.Allocating = .init(arena); + errdefer result.deinit(); + result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + + const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); + defer file.close(io); + + var buf: [1024]u8 = undefined; + var file_reader = file.reader(io, &buf); + _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { + error.ReadFailed => return file_reader.err.?, + error.WriteFailed => return error.OutOfMemory, + }; + + try argv_list.append(arena, result.written()); + }, + .artifact => |pa| { + const artifact = pa.artifact; + const file_path: []const u8 = p: { + if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); + break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; + }; + try argv_list.append(arena, b.fmt("{s}{s}", .{ + pa.prefix, + run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), + })); + }, + .output_file, .output_directory => unreachable, + } + } + + if (run.step.result_failed_command) |cmd| { + fuzz.gpa.free(cmd); + run.step.result_failed_command = null; + } + + const has_side_effects = false; + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ + .progress_node = prog_node, + .watch = undefined, // not used by `runCommand` + .web_server = null, // only needed for time reports + .unit_test_timeout_ns = null, // don't time out fuzz tests for now + .gpa = fuzz.gpa, + }, .{ + .fuzz = fuzz, + }); +} + +fn populateGeneratedPaths( + arena: std.mem.Allocator, + output_placeholders: []const IndexedOutput, + captured_stdout: ?*CapturedStdIo, + captured_stderr: ?*CapturedStdIo, + cache_root: Cache.Directory, + digest: *const Cache.HexDigest, +) !void { + for (output_placeholders) |placeholder| { + placeholder.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, placeholder.output.basename, + }); + } + + if (captured_stdout) |captured| { + captured.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, captured.output.basename, + }); + } + + if (captured_stderr) |captured| { + captured.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, captured.output.basename, + }); + } +} + +fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void { + if (term) |t| switch (t) { + .exited => |code| try w.print("exited with code {d}", .{code}), + .signal => |sig| try w.print("terminated with signal {t}", .{sig}), + .stopped => |sig| try w.print("stopped with signal {t}", .{sig}), + .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}), + } else { + try w.writeAll("exited with any code"); + } +} +fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) { + return .{ .data = term }; +} + +const FuzzContext = struct { + fuzz: *std.Build.Fuzz, +}; + +fn runCommand( + run: *Run, + argv: []const []const u8, + has_side_effects: bool, + output_dir_path: []const u8, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !void { + const step = &run.step; + const b = step.owner; + const arena = b.allocator; + const gpa = options.gpa; + const io = b.graph.io; + + const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; + + try step.handleChildProcUnsupported(); + try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); + + const allow_skip = switch (run.stdio) { + .check, .zig_test => run.skip_foreign_checks, + else => false, + }; + + var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); + defer interp_argv.deinit(); + + var environ_map: EnvMap = env: { + const orig = run.environ_map orelse &b.graph.environ_map; + break :env try orig.clone(gpa); + }; + defer environ_map.deinit(); + + const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { + // InvalidExe: cpu arch mismatch + // FileNotFound: can happen with a wrong dynamic linker path + if (err == error.InvalidExe or err == error.FileNotFound) interpret: { + // TODO: learn the target from the binary directly rather than from + // relying on it being a Compile step. This will make this logic + // work even for the edge case that the binary was produced by a + // third party. + const exe = switch (run.argv.items[0]) { + .artifact => |exe| exe.artifact, + else => break :interpret, + }; + switch (exe.kind) { + .exe, .@"test" => {}, + else => break :interpret, + } + + const root_target = exe.rootModuleTarget(); + const need_cross_libc = exe.is_linking_libc and + (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); + const other_target = exe.root_module.resolved_target.?.result; + switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ + .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, + .link_libc = exe.is_linking_libc, + })) { + .native, .rosetta => { + if (allow_skip) return error.MakeSkipped; + break :interpret; + }, + .wine => |bin_name| { + if (b.enable_wine) { + try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); + + // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but + // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. + if (environ_map.get("WINEDEBUG") == null) { + try environ_map.put("WINEDEBUG", "-all"); + } + } else { + return failForeign(run, "-fwine", argv[0], exe); + } + }, + .qemu => |bin_name| { + if (b.enable_qemu) { + try interp_argv.append(bin_name); + + if (need_cross_libc) { + if (b.libc_runtimes_dir) |dir| { + try interp_argv.append("-L"); + try interp_argv.append(b.pathJoin(&.{ + dir, + try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( + b.allocator, + root_target.cpu.arch, + root_target.os.tag, + root_target.abi, + ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( + b.allocator, + root_target.cpu.arch, + root_target.abi, + ) else unreachable, + })); + } else return failForeign(run, "--libc-runtimes", argv[0], exe); + } + + try interp_argv.appendSlice(argv); + } else return failForeign(run, "-fqemu", argv[0], exe); + }, + .darling => |bin_name| { + if (b.enable_darling) { + try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); + } else { + return failForeign(run, "-fdarling", argv[0], exe); + } + }, + .wasmtime => |bin_name| { + if (b.enable_wasmtime) { + try interp_argv.append(bin_name); + try interp_argv.append("--dir=."); + // Wasmtime doeesn't inherit environment variables from the parent process + // by default. '-S inherit-env' was added in Wasmtime version 20. + try interp_argv.append("-Sinherit-env"); + try interp_argv.append(argv[0]); + try interp_argv.appendSlice(argv[1..]); + } else { + return failForeign(run, "-fwasmtime", argv[0], exe); + } + }, + .bad_dl => |foreign_dl| { + if (allow_skip) return error.MakeSkipped; + + const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; + + return step.fail( + \\the host system is unable to execute binaries from the target + \\ because the host dynamic linker is '{s}', + \\ while the target dynamic linker is '{s}'. + \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step + , .{ host_dl, foreign_dl }); + }, + .bad_os_or_cpu => { + if (allow_skip) return error.MakeSkipped; + + const host_name = try b.graph.host.result.zigTriple(b.allocator); + const foreign_name = try root_target.zigTriple(b.allocator); + + return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ + host_name, foreign_name, + }); + }, + } + + if (root_target.os.tag == .windows) { + // On Windows we don't have rpaths so we have to add .dll search paths to PATH + run.addPathForDynLibs(exe); + } + + gpa.free(step.result_failed_command.?); + step.result_failed_command = null; + try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); + + break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { + if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; + if (e == error.MakeFailed) return error.MakeFailed; // error already reported + return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); + }; + } + if (err == error.MakeFailed) return error.MakeFailed; // error already reported + + return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); + }; + + const generic_result = opt_generic_result orelse { + assert(run.stdio == .zig_test); + // Specific errors have already been reported, and test results are populated. All we need + // to do is report step failure if any test failed. + if (!step.test_results.isSuccess()) return error.MakeFailed; + return; + }; + + assert(fuzz_context == null); + assert(run.stdio != .zig_test); + + // Capture stdout and stderr to GeneratedFile objects. + const Stream = struct { + captured: ?*CapturedStdIo, + bytes: ?[]const u8, + }; + for ([_]Stream{ + .{ + .captured = run.captured_stdout, + .bytes = generic_result.stdout, + }, + .{ + .captured = run.captured_stderr, + .bytes = generic_result.stderr, + }, + }) |stream| { + if (stream.captured) |captured| { + const output_components = .{ output_dir_path, captured.output.basename }; + const output_path = try b.cache_root.join(arena, &output_components); + captured.output.generated_file.path = output_path; + + const sub_path = b.pathJoin(&output_components); + const sub_path_dirname = Dir.path.dirname(sub_path).?; + b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, sub_path_dirname, @errorName(err), + }); + }; + const data = switch (captured.trim_whitespace) { + .none => stream.bytes.?, + .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), + .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), + .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), + }; + b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { + return step.fail("unable to write file '{f}{s}': {s}", .{ + b.cache_root, sub_path, @errorName(err), + }); + }; + } + } + + switch (run.stdio) { + .zig_test => unreachable, + .check => |checks| for (checks.items) |check| switch (check) { + .expect_stderr_exact => |expected_bytes| { + if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { + return step.fail( + \\========= expected this stderr: ========= + \\{s} + \\========= but found: ==================== + \\{s} + , .{ + expected_bytes, + generic_result.stderr.?, + }); + } + }, + .expect_stderr_match => |match| { + if (mem.find(u8, generic_result.stderr.?, match) == null) { + return step.fail( + \\========= expected to find in stderr: ========= + \\{s} + \\========= but stderr does not contain it: ===== + \\{s} + , .{ + match, + generic_result.stderr.?, + }); + } + }, + .expect_stdout_exact => |expected_bytes| { + if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { + return step.fail( + \\========= expected this stdout: ========= + \\{s} + \\========= but found: ==================== + \\{s} + , .{ + expected_bytes, + generic_result.stdout.?, + }); + } + }, + .expect_stdout_match => |match| { + if (mem.find(u8, generic_result.stdout.?, match) == null) { + return step.fail( + \\========= expected to find in stdout: ========= + \\{s} + \\========= but stdout does not contain it: ===== + \\{s} + , .{ + match, + generic_result.stdout.?, + }); + } + }, + .expect_term => |expected_term| { + if (!termMatches(expected_term, generic_result.term)) { + return step.fail("process {f} (expected {f})", .{ + fmtTerm(generic_result.term), + fmtTerm(expected_term), + }); + } + }, + }, + else => { + // On failure, report captured stderr like normal standard error output. + const bad_exit = switch (generic_result.term) { + .exited => |code| code != 0, + .signal, .stopped, .unknown => true, + }; + if (bad_exit) { + if (generic_result.stderr) |bytes| { + run.step.result_stderr = bytes; + } + } + + try step.handleChildProcessTerm(generic_result.term); + }, + } +} + +const EvalGenericResult = struct { + term: process.Child.Term, + stdout: ?[]const u8, + stderr: ?[]const u8, +}; + +fn spawnChildAndCollect( + run: *Run, + argv: []const []const u8, + environ_map: *EnvMap, + has_side_effects: bool, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !?EvalGenericResult { + const b = run.step.owner; + const graph = b.graph; + const io = graph.io; + + if (fuzz_context != null) { + assert(!has_side_effects); + assert(run.stdio == .zig_test); + } + + const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; + + // If an error occurs, it's caused by this command: + assert(run.step.result_failed_command == null); + run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ + .child = environ_map, + .parent = &graph.environ_map, + }, argv); + + var spawn_options: process.SpawnOptions = .{ + .argv = argv, + .cwd = child_cwd, + .environ_map = environ_map, + .request_resource_usage_statistics = true, + .stdin = if (run.stdin != .none) s: { + assert(run.stdio != .inherit); + break :s .pipe; + } else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .ignore, + .inherit => .inherit, + .check => .ignore, + .zig_test => .pipe, + }, + .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .ignore, + .inherit => .inherit, + .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, + .zig_test => .pipe, + }, + .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .pipe, + .inherit => .inherit, + .check => .pipe, + .zig_test => .pipe, + }, + }; + + if (run.stdio == .zig_test) { + const started: Io.Clock.Timestamp = .now(io, .awake); + const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| e, + }; + run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + try result; + return null; + } else { + const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; + if (!run.disable_zig_progress and !inherit) { + spawn_options.progress_node = options.progress_node; + } + const terminal_mode: Io.Terminal.Mode = if (inherit) m: { + const stderr = try io.lockStderr(&.{}, graph.stderr_mode); + break :m stderr.terminal_mode; + } else .no_color; + defer if (inherit) io.unlockStderr(); + try setColorEnvironmentVariables(run, environ_map, terminal_mode); + + const started: Io.Clock.Timestamp = .now(io, .awake); + const result = evalGeneric(run, spawn_options) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| e, + }; + run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + return try result; + } +} + +fn hashStdIo(hh: *Cache.HashHelper, stdio: StdIo) void { + switch (stdio) { + .infer_from_args, .inherit, .zig_test => {}, + .check => |checks| for (checks.items) |check| { + hh.add(@as(std.meta.Tag(StdIo.Check), check)); + switch (check) { + .expect_stderr_exact, + .expect_stderr_match, + .expect_stdout_exact, + .expect_stdout_match, + => |s| hh.addBytes(s), + + .expect_term => |term| { + hh.add(@as(std.meta.Tag(process.Child.Term), term)); + switch (term) { + inline .exited, .signal, .stopped => |x| hh.add(x), + .unknown => |x| hh.add(x), + } + }, + } + }, + } +} +fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { + return if (expected) |e| switch (e) { + .exited => |expected_code| switch (actual) { + .exited => |actual_code| expected_code == actual_code, + else => false, + }, + .signal => |expected_sig| switch (actual) { + .signal => |actual_sig| expected_sig == actual_sig, + else => false, + }, + .stopped => |expected_sig| switch (actual) { + .stopped => |actual_sig| expected_sig == actual_sig, + else => false, + }, + .unknown => |expected_code| switch (actual) { + .unknown => |actual_code| expected_code == actual_code, + else => false, + }, + } else switch (actual) { + .exited => true, + else => false, + }; +} + +fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { + color: switch (run.color) { + .manual => {}, + .enable => { + try environ_map.put("CLICOLOR_FORCE", "1"); + _ = environ_map.swapRemove("NO_COLOR"); + }, + .disable => { + try environ_map.put("NO_COLOR", "1"); + _ = environ_map.swapRemove("CLICOLOR_FORCE"); + }, + .inherit => switch (terminal_mode) { + .no_color, .windows_api => continue :color .disable, + .escape_codes => continue :color .enable, + }, + .auto => { + const capture_stderr = run.captured_stderr != null or switch (run.stdio) { + .check => |checks| checksContainStderr(checks.items), + .infer_from_args, .inherit, .zig_test => false, + }; + if (capture_stderr) { + continue :color .disable; + } else { + continue :color .inherit; + } + }, + } +} + +fn checksContainStdout(checks: []const StdIo.Check) bool { + for (checks) |check| switch (check) { + .expect_stderr_exact, + .expect_stderr_match, + .expect_term, + => continue, + + .expect_stdout_exact, + .expect_stdout_match, + => return true, + }; + return false; +} + +fn checksContainStderr(checks: []const StdIo.Check) bool { + for (checks) |check| switch (check) { + .expect_stdout_exact, + .expect_stdout_match, + .expect_term, + => continue, + + .expect_stderr_exact, + .expect_stderr_match, + => return true, + }; + return false; +} + +/// Returns whether the Run step has side effects *other than* updating the output arguments. +fn hasSideEffects(run: Run) bool { + if (run.has_side_effects) return true; + return switch (run.stdio) { + .infer_from_args => !run.hasAnyOutputArgs(), + .inherit => true, + .check => false, + .zig_test => false, + }; +} + +fn hasAnyOutputArgs(run: Run) bool { + if (run.captured_stdout != null) return true; + if (run.captured_stderr != null) return true; + for (run.argv.items) |arg| switch (arg) { + .output_file, .output_directory => return true, + else => continue, + }; + return false; +} + +/// If `path` is cwd-relative, make it relative to the cwd of the child instead. +/// +/// Whenever a path is included in the argv of a child, it should be put through this function first +/// to make sure the child doesn't see paths relative to a cwd other than its own. +fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { + const b = run.step.owner; + const graph = b.graph; + const arena = graph.arena; + + const path_str = path.toString(arena) catch @panic("OOM"); + if (Dir.path.isAbsolute(path_str)) { + // Absolute paths don't need changing. + return path_str; + } + const child_cwd_rel: []const u8 = rel: { + const child_lazy_cwd = run.cwd orelse break :rel path_str; + const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); + // Convert it from relative to *our* cwd, to relative to the *child's* cwd. + break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); + }; + // Not every path can be made relative, e.g. if the path and the child cwd are on different + // disk designators on Windows. In that case, `relative` will return an absolute path which we can + // just return. + if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel; + + // We're not done yet. In some cases this path must be prefixed with './': + // * On POSIX, the executable name cannot be a single component like 'foo' + // * Some executables might treat a leading '-' like a flag, which we must avoid + // There's no harm in it, so just *always* apply this prefix. + return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); +} + +fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { + const b = run.step.owner; + const compiles = artifact.getCompileDependencies(true); + for (compiles) |compile| { + if (compile.root_module.resolved_target.?.result.os.tag == .windows and + compile.isDynamicLibrary()) + { + addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); + } + } +} + +fn failForeign( + run: *Run, + suggested_flag: []const u8, + argv0: []const u8, + exe: *Step.Compile, +) error{ MakeFailed, MakeSkipped, OutOfMemory } { + switch (run.stdio) { + .check, .zig_test => { + if (run.skip_foreign_checks) + return error.MakeSkipped; + + const b = run.step.owner; + const host_name = try b.graph.host.result.zigTriple(b.allocator); + const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); + + return run.step.fail( + \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) + \\ consider using {s} or enabling skip_foreign_checks in the Run step + , .{ argv0, foreign_name, host_name, suggested_flag }); + }, + else => { + return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); + }, + } +} diff --git a/lib/compiler/maker/Step/WriteFile.zig b/lib/compiler/maker/Step/WriteFile.zig new file mode 100644 index 0000000000000000000000000000000000000000..d594f8983fe5ead949a5efdd0ba106c176777d34 --- /dev/null +++ b/lib/compiler/maker/Step/WriteFile.zig @@ -0,0 +1,206 @@ + +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; + const b = step.owner; + const graph = b.graph; + const io = graph.io; + const arena = b.allocator; + const gpa = graph.cache.gpa; + const write_file: *WriteFile = @fieldParentPtr("step", step); + + const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); + var open_dirs_count: usize = 0; + defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); + + switch (write_file.mode) { + .whole_cached => { + step.clearWatchInputs(); + + // The cache is used here not really as a way to speed things up - because writing + // the data to a file would probably be very fast - but as a way to find a canonical + // location to put build artifacts. + + // If, for example, a hard-coded path was used as the location to put WriteFile + // files, then two WriteFiles executing in parallel might clobber each other. + + var man = b.graph.cache.obtain(); + defer man.deinit(); + + for (write_file.files.items) |file| { + man.hash.addBytes(file.sub_path); + + switch (file.contents) { + .bytes => |bytes| { + man.hash.addBytes(bytes); + }, + .copy => |lazy_path| { + const path = lazy_path.getPath3(b, step); + _ = try man.addFilePath(path, null); + try step.addWatchInput(lazy_path); + }, + } + } + + for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { + man.hash.addBytes(dir.sub_path); + for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); + if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); + + const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); + const src_dir_path = dir.source.getPath3(b, step); + + var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{f}': {s}", .{ + src_dir_path, @errorName(err), + }); + }; + open_dir_cache_elem.* = src_dir; + open_dirs_count += 1; + + var it = try src_dir.walk(gpa); + defer it.deinit(); + while (try it.next(io)) |entry| { + if (!dir.options.pathIncluded(entry.path)) continue; + + switch (entry.kind) { + .directory => { + if (need_derived_inputs) { + const entry_path = try src_dir_path.join(arena, entry.path); + try step.addDirectoryWatchInputFromPath(entry_path); + } + }, + .file => { + const entry_path = try src_dir_path.join(arena, entry.path); + _ = try man.addFilePath(entry_path, null); + }, + else => continue, + } + } + } + + if (try step.cacheHit(&man)) { + const digest = man.final(); + write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); + assert(step.result_cached); + return; + } + + const digest = man.final(); + const cache_path = "o" ++ Dir.path.sep_str ++ digest; + + write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); + + try operate(write_file, open_dir_cache, .{ + .root_dir = b.cache_root, + .sub_path = cache_path, + }); + + try step.writeManifest(&man); + }, + .tmp => { + step.result_cached = false; + + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + + write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); + + try operate(write_file, open_dir_cache, .{ + .root_dir = b.cache_root, + .sub_path = tmp_dir_sub_path, + }); + }, + .mutate => |lp| { + step.result_cached = false; + const root_path = try lp.getPath4(b, step); + write_file.generated_directory.path = try root_path.toString(arena); + try operate(write_file, open_dir_cache, root_path); + }, + } +} + +fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { + const step = &write_file.step; + const b = step.owner; + const io = b.graph.io; + const gpa = b.graph.cache.gpa; + const arena = b.allocator; + + var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| + return step.fail("unable to make path {f}: {t}", .{ root_path, err }); + defer cache_dir.close(io); + + for (write_file.files.items) |file| { + if (Dir.path.dirname(file.sub_path)) |dirname| { + cache_dir.createDirPath(io, dirname) catch |err| { + return step.fail("unable to make path '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, dirname, err, + }); + }; + } + switch (file.contents) { + .bytes => |bytes| { + cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { + return step.fail("unable to write file '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, file.sub_path, err, + }); + }; + }, + .copy => |file_source| { + const source_path = file_source.getPath2(b, step); + const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { + return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ + source_path, root_path, Dir.path.sep, file.sub_path, err, + }); + }; + // At this point we already will mark the step as a cache miss. + // But this is kind of a partial cache hit since individual + // file copies may be avoided. Oh well, this information is + // discarded. + _ = prev_status; + }, + } + } + + for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { + const src_dir_path = dir.source.getPath3(b, step); + const dest_dirname = dir.sub_path; + + if (dest_dirname.len != 0) { + cache_dir.createDirPath(io, dest_dirname) catch |err| { + return step.fail("unable to make path '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, dest_dirname, err, + }); + }; + } + + var it = try already_open_dir.walk(gpa); + defer it.deinit(); + while (try it.next(io)) |entry| { + if (!dir.options.pathIncluded(entry.path)) continue; + + const src_entry_path = try src_dir_path.join(arena, entry.path); + const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); + switch (entry.kind) { + .directory => try cache_dir.createDirPath(io, dest_path), + .file => { + const prev_status = Io.Dir.updateFile( + src_entry_path.root_dir.handle, + io, + src_entry_path.sub_path, + cache_dir, + dest_path, + .{}, + ) catch |err| { + return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ + src_entry_path, root_path, Dir.path.sep, dest_path, err, + }); + }; + _ = prev_status; + }, + else => continue, + } + } + } +} diff --git a/lib/std/Build/Watch.zig b/lib/compiler/maker/Watch.zig similarity index 100% rename from lib/std/Build/Watch.zig rename to lib/compiler/maker/Watch.zig diff --git a/lib/std/Build/Watch/FsEvents.zig b/lib/compiler/maker/Watch/FsEvents.zig similarity index 100% rename from lib/std/Build/Watch/FsEvents.zig rename to lib/compiler/maker/Watch/FsEvents.zig diff --git a/lib/std/Build/WebServer.zig b/lib/compiler/maker/WebServer.zig similarity index 100% rename from lib/std/Build/WebServer.zig rename to lib/compiler/maker/WebServer.zig diff --git a/lib/std/Build.zig b/lib/std/Build.zig index f0eae560d432684103f13b4d43bcf64676c57155..f0edfb7b817eb2dbc0d03dff4a0267c2da519ae6 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -19,15 +19,14 @@ const ArrayList = std.ArrayList; pub const Cache = @import("Build/Cache.zig"); pub const Step = @import("Build/Step.zig"); pub const Module = @import("Build/Module.zig"); -pub const Watch = @import("Build/Watch.zig"); -pub const Fuzz = @import("Build/Fuzz.zig"); -pub const WebServer = @import("Build/WebServer.zig"); pub const abi = @import("Build/abi.zig"); +/// The serialized output of configure phase ingested by make phase. +pub const Configuration = @import("zig/Configuration.zig"); /// Shared state among all Build instances. graph: *Graph, -install_tls: TopLevelStep, -uninstall_tls: TopLevelStep, +install_tls: Step.TopLevel, +uninstall_tls: Step.TopLevel, allocator: Allocator, user_input_options: UserInputOptionsMap, available_options_map: AvailableOptionsMap, @@ -39,28 +38,17 @@ verbose_air: bool, verbose_llvm_ir: ?[]const u8, verbose_llvm_bc: ?[]const u8, verbose_llvm_cpu_features: bool, -reference_trace: ?u32 = null, invalid_user_input: bool, default_step: *Step, -top_level_steps: std.StringArrayHashMapUnmanaged(*TopLevelStep), +top_level_steps: std.StringArrayHashMapUnmanaged(*Step.TopLevel), install_prefix: []const u8, -dest_dir: ?[]const u8, -lib_dir: []const u8, -exe_dir: []const u8, -h_dir: []const u8, -install_path: []const u8, -sysroot: ?[]const u8 = null, -search_prefixes: ArrayList([]const u8), -libc_file: ?[]const u8 = null, /// Path to the directory containing build.zig. build_root: Cache.Directory, cache_root: Cache.Directory, pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, -args: ?[]const []const u8 = null, debug_log_scopes: []const []const u8 = &.{}, debug_compile_errors: bool = false, debug_incremental: bool = false, -debug_pkg_config: bool = false, /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, /// in particular at `Step` creation. /// Set to 0 to disable stack collection. @@ -76,12 +64,6 @@ enable_rosetta: bool = false, enable_wasmtime: bool = false, /// Use system Wine installation to run cross compiled Windows build artifacts. enable_wine: bool = false, -/// After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, -/// this will be the directory $glibc-build-dir/install/glibcs -/// Given the example of the aarch64 target, this is the directory -/// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. -/// Also works for dynamic musl. -libc_runtimes_dir: ?[]const u8 = null, dep_prefix: []const u8 = "", @@ -94,8 +76,6 @@ pkg_hash: []const u8, /// A mapping from dependency names to package hashes. available_deps: AvailableDeps, -release_mode: ReleaseMode, - build_id: ?std.zig.BuildId = null, pub const ReleaseMode = enum { @@ -116,14 +96,13 @@ pub const Graph = struct { system_package_mode: bool = false, debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, cache: Cache, - zig_exe: [:0]const u8, + zig_exe: []const u8, environ_map: process.Environ.Map, global_cache_root: Cache.Directory, zig_lib_directory: Cache.Directory, needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .empty, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, - incremental: ?bool = null, random_seed: u32 = 0, dependency_cache: InitializedDepMap = .empty, allow_so_scripts: ?bool = null, @@ -134,11 +113,12 @@ pub const Graph = struct { /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also /// respects the '--color' flag. stderr_mode: ?Io.Terminal.Mode = null, + release_mode: ReleaseMode = .off, }; const AvailableDeps = []const struct { []const u8, []const u8 }; -const SystemLibraryMode = enum { +pub const SystemLibraryMode = enum { /// User asked for the library to be disabled. /// The build runner has not confirmed whether the setting is recognized yet. user_disabled, @@ -245,19 +225,6 @@ const TypeId = enum { lazy_path_list, }; -const TopLevelStep = struct { - pub const base_id: Step.Id = .top_level; - - step: Step, - description: []const u8, -}; - -pub const DirList = struct { - lib_dir: ?[]const u8 = null, - exe_dir: ?[]const u8 = null, - include_dir: ?[]const u8 = null, -}; - pub fn create( graph: *Graph, build_root: Cache.Directory, @@ -285,15 +252,10 @@ pub fn create( .available_options_list = std.array_list.Managed(AvailableOption).init(arena), .top_level_steps = .{}, .default_step = undefined, - .search_prefixes = .empty, .install_prefix = undefined, - .lib_dir = undefined, - .exe_dir = undefined, - .h_dir = undefined, - .dest_dir = graph.environ_map.get("DESTDIR"), .install_tls = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = "install", .owner = b, }), @@ -301,21 +263,17 @@ pub fn create( }, .uninstall_tls = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = "uninstall", .owner = b, - .makeFn = makeUninstall, }), .description = "Remove build artifacts from prefix path", }, - .install_path = undefined, - .args = null, .modules = .empty, .named_writefiles = .empty, .named_lazy_paths = .empty, .pkg_hash = "", .available_deps = available_deps, - .release_mode = .off, }; try b.top_level_steps.put(arena, b.install_tls.step.name, &b.install_tls); try b.top_level_steps.put(arena, b.uninstall_tls.step.name, &b.uninstall_tls); @@ -330,19 +288,6 @@ fn createChild( pkg_hash: []const u8, pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap, -) error{OutOfMemory}!*Build { - const child = try createChildOnly(parent, dep_name, build_root, pkg_hash, pkg_deps, user_input_options); - try determineAndApplyInstallPrefix(child); - return child; -} - -fn createChildOnly( - parent: *Build, - dep_name: []const u8, - build_root: Cache.Directory, - pkg_hash: []const u8, - pkg_deps: AvailableDeps, - user_input_options: UserInputOptionsMap, ) error{OutOfMemory}!*Build { const allocator = parent.allocator; const child = try allocator.create(Build); @@ -351,7 +296,7 @@ fn createChildOnly( .allocator = allocator, .install_tls = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = "install", .owner = child, }), @@ -359,10 +304,9 @@ fn createChildOnly( }, .uninstall_tls = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = "uninstall", .owner = child, - .makeFn = makeUninstall, }), .description = "Remove build artifacts from prefix path", }, @@ -376,38 +320,31 @@ fn createChildOnly( .verbose_llvm_ir = parent.verbose_llvm_ir, .verbose_llvm_bc = parent.verbose_llvm_bc, .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, - .reference_trace = parent.reference_trace, .invalid_user_input = false, .default_step = undefined, .top_level_steps = .{}, .install_prefix = undefined, - .dest_dir = parent.dest_dir, .lib_dir = parent.lib_dir, .exe_dir = parent.exe_dir, .h_dir = parent.h_dir, .install_path = parent.install_path, .sysroot = parent.sysroot, - .search_prefixes = parent.search_prefixes, - .libc_file = parent.libc_file, .build_root = build_root, .cache_root = parent.cache_root, .debug_log_scopes = parent.debug_log_scopes, .debug_compile_errors = parent.debug_compile_errors, .debug_incremental = parent.debug_incremental, - .debug_pkg_config = parent.debug_pkg_config, .enable_darling = parent.enable_darling, .enable_qemu = parent.enable_qemu, .enable_rosetta = parent.enable_rosetta, .enable_wasmtime = parent.enable_wasmtime, .enable_wine = parent.enable_wine, - .libc_runtimes_dir = parent.libc_runtimes_dir, .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }), .modules = .empty, .named_writefiles = .empty, .named_lazy_paths = .empty, .pkg_hash = pkg_hash, .available_deps = pkg_deps, - .release_mode = parent.release_mode, }; try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls); try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls); @@ -702,59 +639,6 @@ fn hashUserInputOptionsMap(allocator: Allocator, user_input_options: UserInputOp user_option.hash(hasher); } -fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { - // Create an installation directory local to this package. This will be used when - // dependant packages require a standard prefix, such as include directories for C headers. - var hash = b.graph.cache.hash; - // Random bytes to make unique. Refresh this with new random bytes when - // implementation is modified in a non-backwards-compatible way. - hash.add(@as(u32, 0xd8cb0055)); - hash.addBytes(b.dep_prefix); - - var wyhash = std.hash.Wyhash.init(0); - hashUserInputOptionsMap(b.allocator, b.user_input_options, &wyhash); - hash.add(wyhash.final()); - - const digest = hash.final(); - const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); - b.resolveInstallPrefix(install_prefix, .{}); -} - -/// This function is intended to be called by lib/build_runner.zig, not a build.zig file. -pub fn resolveInstallPrefix(b: *Build, install_prefix: ?[]const u8, dir_list: DirList) void { - if (b.dest_dir) |dest_dir| { - b.install_prefix = install_prefix orelse "/usr"; - b.install_path = b.pathJoin(&.{ dest_dir, b.install_prefix }); - } else { - b.install_prefix = install_prefix orelse - (b.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); - b.install_path = b.install_prefix; - } - - var lib_list = [_][]const u8{ b.install_path, "lib" }; - var exe_list = [_][]const u8{ b.install_path, "bin" }; - var h_list = [_][]const u8{ b.install_path, "include" }; - - if (dir_list.lib_dir) |dir| { - if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; - lib_list[1] = dir; - } - - if (dir_list.exe_dir) |dir| { - if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; - exe_list[1] = dir; - } - - if (dir_list.include_dir) |dir| { - if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; - h_list[1] = dir; - } - - b.lib_dir = b.pathJoin(&lib_list); - b.exe_dir = b.pathJoin(&exe_list); - b.h_dir = b.pathJoin(&h_list); -} - /// Create a set of key-value pairs that can be converted into a Zig source /// file and then inserted into a Zig compilation's module table for importing. /// In other words, this provides a way to expose build.zig values to Zig @@ -1121,15 +1005,6 @@ pub fn getUninstallStep(b: *Build) *Step { return &b.uninstall_tls.step; } -fn makeUninstall(uninstall_step: *Step, options: Step.MakeOptions) anyerror!void { - _ = options; - const uninstall_tls: *TopLevelStep = @fieldParentPtr("step", uninstall_step); - const b: *Build = @fieldParentPtr("uninstall_tls", uninstall_tls); - - _ = b; - @panic("TODO implement https://github.com/ziglang/zig/issues/14943"); -} - /// Creates a configuration option to be passed to the build.zig script. /// When a user directly runs `zig build`, they can set these options with `-D` arguments. /// When a project depends on a Zig package as a dependency, it programmatically sets @@ -1350,10 +1225,10 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw } pub fn step(b: *Build, name: []const u8, description: []const u8) *Step { - const step_info = b.allocator.create(TopLevelStep) catch @panic("OOM"); + const step_info = b.allocator.create(Step.TopLevel) catch @panic("OOM"); step_info.* = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = name, .owner = b, }), @@ -1373,8 +1248,10 @@ pub const StandardOptimizeOptionOptions = struct { }; pub fn standardOptimizeOption(b: *Build, options: StandardOptimizeOptionOptions) std.builtin.OptimizeMode { + const graph = b.graph; + if (options.preferred_optimize_mode) |mode| { - if (b.option(bool, "release", "optimize for end users") orelse (b.release_mode != .off)) { + if (b.option(bool, "release", "optimize for end users") orelse (graph.release_mode != .off)) { return mode; } else { return .Debug; @@ -1389,7 +1266,7 @@ pub fn standardOptimizeOption(b: *Build, options: StandardOptimizeOptionOptions) return mode; } - return switch (b.release_mode) { + return switch (graph.release_mode) { .off => .Debug, .any => { std.debug.print("the project does not declare a preferred optimization mode. choose: --release=fast, --release=safe, or --release=small\n", .{}); @@ -1824,36 +1701,11 @@ fn tryFindProgram(b: *Build, full_path: []const u8) ?[]const u8 { return null; } -pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) error{FileNotFound}![]const u8 { - // TODO report error for ambiguous situations - for (b.search_prefixes.items) |search_prefix| { - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - return tryFindProgram(b, b.pathJoin(&.{ search_prefix, "bin", name })) orelse continue; - } - } - if (b.graph.environ_map.get("PATH")) |PATH| { - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - var it = mem.tokenizeScalar(u8, PATH, fs.path.delimiter); - while (it.next()) |p| { - return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; - } - } - } - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - for (paths) |p| { - return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; - } - } - return error.FileNotFound; +pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) LazyPath { + _ = b; + _ = names; + _ = paths; + @panic("TODO rework findProgram to be based on LazyPath"); } pub fn runAllowFail( @@ -1918,10 +1770,6 @@ pub fn run(b: *Build, argv: []const []const u8) []u8 { ); } -pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { - b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); -} - pub fn getInstallPath(b: *Build, dir: InstallDir, dest_rel_path: []const u8) []const u8 { assert(!fs.path.isAbsolute(dest_rel_path)); // Install paths must be relative to the prefix const base_dir = switch (dir) { diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 51c1514f8597790dd8c02f53eae2e23dd9a26fd7..7b8a04d66e923953d5fec8fec03e938e6e9f22ac 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -10,24 +10,11 @@ const Cache = Build.Cache; const Path = Cache.Path; const ArrayList = std.ArrayList; -id: Id, +tag: std.Build.Configuration.Step.Tag, name: []const u8, owner: *Build, -makeFn: MakeFn, -dependencies: std.array_list.Managed(*Step), -/// This field is empty during execution of the user's build script, and -/// then populated during dependency loop checking in the build runner. -dependants: ArrayList(*Step), -/// Collects the set of files that retrigger this step to run. -/// -/// This is used by the build system's implementation of `--watch` but it can -/// also be potentially useful for IDEs to know what effects editing a -/// particular file has. -/// -/// Populated within `make`. Implementation may choose to clear and repopulate, -/// retain previous value, or update. -inputs: Inputs, +dependencies: ArrayList(*Step), /// Set this field to declare an upper bound on the amount of bytes of memory it will /// take to run the step. Zero means no limit. @@ -51,77 +38,11 @@ inputs: Inputs, max_rss: usize, state: State, -pending_deps: u32, - -result_error_msgs: ArrayList([]const u8), -result_error_bundle: std.zig.ErrorBundle, -result_stderr: []const u8, -result_cached: bool, -result_duration_ns: ?u64, -/// 0 means unavailable or not reported. -result_peak_rss: usize, -/// If the step is failed and this field is populated, this is the command which failed. -/// This field may be populated even if the step succeeded. -result_failed_command: ?[]const u8, -test_results: TestResults, /// The return address associated with creation of this step that can be useful /// to print along with debugging messages. debug_stack_trace: std.debug.StackTrace, -pub const TestResults = struct { - /// The total number of tests in the step. Every test has a "status" from the following: - /// * passed - /// * skipped - /// * failed cleanly - /// * crashed - /// * timed out - test_count: u32 = 0, - - /// The number of tests which were skipped (`error.SkipZigTest`). - skip_count: u32 = 0, - /// The number of tests which failed cleanly. - fail_count: u32 = 0, - /// The number of tests which terminated unexpectedly, i.e. crashed. - crash_count: u32 = 0, - /// The number of tests which timed out. - timeout_count: u32 = 0, - - /// The number of detected memory leaks. The associated test may still have passed; indeed, *all* - /// individual tests may have passed. However, the step as a whole fails if any test has leaks. - leak_count: u32 = 0, - /// The number of detected error logs. The associated test may still have passed; indeed, *all* - /// individual tests may have passed. However, the step as a whole fails if any test logs errors. - log_err_count: u32 = 0, - - pub fn isSuccess(tr: TestResults) bool { - // all steps are success or skip - return tr.fail_count == 0 and - tr.crash_count == 0 and - tr.timeout_count == 0 and - // no (otherwise successful) step leaked memory or logged errors - tr.leak_count == 0 and - tr.log_err_count == 0; - } - - /// Computes the number of tests which passed from the other values. - pub fn passCount(tr: TestResults) u32 { - return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count; - } -}; - -pub const MakeOptions = struct { - progress_node: std.Progress.Node, - watch: bool, - web_server: ?*Build.WebServer, - /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. - unit_test_timeout_ns: ?u64, - /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. - gpa: Allocator, -}; - -pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; - pub const State = enum { precheck_unstarted, precheck_started, @@ -132,57 +53,29 @@ pub const State = enum { /// be re-evaluated. precheck_done, dependency_failure, - success, - failure, - /// This state indicates that the step did not complete, however, it also did not fail, - /// and it is safe to continue executing its dependencies. - skipped, - /// This step was skipped because it specified a max_rss that exceeded the runner's maximum. - /// It is not safe to run its dependencies. - skipped_oom, }; -pub const Id = enum { - top_level, - compile, - install_artifact, - install_file, - install_dir, - remove_dir, - fail, - fmt, - translate_c, - write_file, - update_source_files, - run, - check_file, - check_object, - config_header, - objcopy, - options, - custom, +pub const Tag = std.Build.Configuration.Step.Tag; - pub fn Type(comptime id: Id) type { - return switch (id) { - .top_level => Build.TopLevelStep, - .compile => Compile, - .install_artifact => InstallArtifact, - .install_file => InstallFile, - .install_dir => InstallDir, - .fail => Fail, - .fmt => Fmt, - .translate_c => TranslateC, - .write_file => WriteFile, - .update_source_files => UpdateSourceFiles, - .run => Run, - .check_file => CheckFile, - .config_header => ConfigHeader, - .objcopy => ObjCopy, - .options => Options, - .custom => @compileError("no type available for custom step"), - }; - } -}; +pub fn Type(comptime tag: Tag) type { + return switch (tag) { + .top_level => Build.TopLevelStep, + .compile => Compile, + .install_artifact => InstallArtifact, + .install_file => InstallFile, + .install_dir => InstallDir, + .fail => Fail, + .fmt => Fmt, + .translate_c => TranslateC, + .write_file => WriteFile, + .update_source_files => UpdateSourceFiles, + .run => Run, + .check_file => CheckFile, + .config_header => ConfigHeader, + .objcopy => ObjCopy, + .options => Options, + }; +} pub const CheckFile = @import("Step/CheckFile.zig"); pub const ConfigHeader = @import("Step/ConfigHeader.zig"); @@ -199,32 +92,17 @@ pub const TranslateC = @import("Step/TranslateC.zig"); pub const WriteFile = @import("Step/WriteFile.zig"); pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); -pub const Inputs = struct { - table: Table, +pub const TopLevel = struct { + pub const base_tag: Step.Tag = .top_level; - pub const init: Inputs = .{ - .table = .{}, - }; - - pub const Table = std.ArrayHashMapUnmanaged(Build.Cache.Path, Files, Build.Cache.Path.TableAdapter, false); - /// The special file name "." means any changes inside the directory. - pub const Files = ArrayList([]const u8); - - pub fn populated(inputs: *Inputs) bool { - return inputs.table.count() != 0; - } - - pub fn clear(inputs: *Inputs, gpa: Allocator) void { - for (inputs.table.values()) |*files| files.deinit(gpa); - inputs.table.clearRetainingCapacity(); - } + step: Step, + description: []const u8, }; pub const StepOptions = struct { - id: Id, + tag: Tag, name: []const u8, owner: *Build, - makeFn: MakeFn = makeNoOp, first_ret_addr: ?usize = null, max_rss: usize = 0, }; @@ -233,90 +111,27 @@ pub fn init(options: StepOptions) Step { const arena = options.owner.allocator; return .{ - .id = options.id, + .tag = options.tag, .name = arena.dupe(u8, options.name) catch @panic("OOM"), .owner = options.owner, - .makeFn = options.makeFn, - .dependencies = std.array_list.Managed(*Step).init(arena), - .dependants = .empty, - .inputs = Inputs.init, + .dependencies = .empty, .state = .precheck_unstarted, - .pending_deps = undefined, // initialized by build runner .max_rss = options.max_rss, .debug_stack_trace = blk: { const addr_buf = arena.alloc(usize, options.owner.debug_stack_frames_count) catch @panic("OOM"); const first_ret_addr = options.first_ret_addr orelse @returnAddress(); break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf); }, - .result_error_msgs = .empty, - .result_error_bundle = std.zig.ErrorBundle.empty, - .result_stderr = "", - .result_cached = false, - .result_duration_ns = null, - .result_peak_rss = 0, - .result_failed_command = null, - .test_results = .{}, }; } -/// If the Step's `make` function reports `error.MakeFailed`, it indicates they -/// have already reported the error. Otherwise, we add a simple error report -/// here. -pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { - const arena = s.owner.allocator; - const graph = s.owner.graph; - const io = graph.io; - - var start_ts: ?Io.Timestamp = t: { - if (!graph.time_report) break :t null; - if (s.id == .compile) break :t null; - if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; - break :t Io.Clock.awake.now(io); - }; - const make_result = s.makeFn(s, options); - if (start_ts) |*ts| { - const duration = ts.untilNow(io, .awake); - options.web_server.?.updateTimeReportGeneric(s, duration); - } - - make_result catch |err| switch (err) { - error.MakeFailed, error.MakeSkipped => |e| return e, - else => { - s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); - return error.MakeFailed; - }, - }; - - if (!s.test_results.isSuccess()) { - return error.MakeFailed; - } - - if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { - const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ - s.result_peak_rss, s.max_rss, - }) catch @panic("OOM"); - s.result_error_msgs.append(arena, msg) catch @panic("OOM"); - } -} - pub fn dependOn(step: *Step, other: *Step) void { - step.dependencies.append(other) catch @panic("OOM"); -} - -fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { - _ = options; - - var all_cached = true; - - for (step.dependencies.items) |dep| { - all_cached = all_cached and dep.result_cached; - } - - step.result_cached = all_cached; + const arena = step.owner.allocator; + step.dependencies.append(arena, other) catch @panic("OOM"); } pub fn cast(step: *Step, comptime T: type) ?*T { - if (step.id == T.base_id) { + if (step.tag == T.base_tag) { return @fieldParentPtr("step", step); } return null; @@ -337,670 +152,6 @@ pub fn dump(step: *Step, t: Io.Terminal) void { } } -/// Populates `s.result_failed_command`. -pub fn captureChildProcess( - s: *Step, - gpa: Allocator, - progress_node: std.Progress.Node, - argv: []const []const u8, -) !std.process.RunResult { - const graph = s.owner.graph; - const arena = graph.arena; - const io = graph.io; - - // If an error occurs, it's happened in this command: - assert(s.result_failed_command == null); - s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); - - try handleChildProcUnsupported(s); - try handleVerbose(s.owner, .inherit, argv); - - const result = std.process.run(arena, io, .{ - .argv = argv, - .environ_map = &graph.environ_map, - .progress_node = progress_node, - }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); - - if (result.stderr.len > 0) { - try s.result_error_msgs.append(arena, result.stderr); - } - - return result; -} - -pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { - try step.addError(fmt, args); - return error.MakeFailed; -} - -pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { - const arena = step.owner.allocator; - const msg = try std.fmt.allocPrint(arena, fmt, args); - try step.result_error_msgs.append(arena, msg); -} - -pub const ZigProcess = struct { - child: std.process.Child, - multi_reader_buffer: Io.File.MultiReader.Buffer(2), - multi_reader: Io.File.MultiReader, - progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn, - - pub const StreamEnum = enum { stdout, stderr }; - - pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void { - zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null; - } - - pub fn deinit(zp: *ZigProcess, io: Io) void { - zp.child.kill(io); - zp.multi_reader.deinit(); - zp.* = undefined; - } -}; - -/// Assumes that argv contains `--listen=-` and that the process being spawned -/// is the zig compiler - the same version that compiled the build runner. -/// Populates `s.result_failed_command`. -pub fn evalZigProcess( - s: *Step, - argv: []const []const u8, - prog_node: std.Progress.Node, - watch: bool, - web_server: ?*Build.WebServer, - gpa: Allocator, -) !?Path { - const b = s.owner; - const io = b.graph.io; - - // If an error occurs, it's happened in this command: - assert(s.result_failed_command == null); - s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); - - if (s.getZigProcess()) |zp| update: { - assert(watch); - if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index); - zp.progress_ipc_index = null; - var exited = false; - defer if (exited) { - s.cast(Compile).?.zig_process = null; - zp.deinit(io); - gpa.destroy(zp); - } else zp.saveState(prog_node); - const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { - error.BrokenPipe, error.EndOfStream => |reason| { - std.log.info("{s} restart required: {t}", .{ argv[0], reason }); - // Process restart required. - const term = zp.child.wait(io) catch |e| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); - }; - _ = term; - exited = true; - break :update; - }, - else => |e| return e, - }; - - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); - } - - if (s.result_error_msgs.items.len > 0 and result == null) { - // Crash detected. - const term = zp.child.wait(io) catch |e| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); - }; - s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; - exited = true; - try handleChildProcessTerm(s, term); - return error.MakeFailed; - } - - return result; - } - assert(argv.len != 0); - - try handleChildProcUnsupported(s); - try handleVerbose(s.owner, .inherit, argv); - - const zp = try gpa.create(ZigProcess); - defer if (!watch) gpa.destroy(zp); - - zp.child = std.process.spawn(io, .{ - .argv = argv, - .environ_map = &b.graph.environ_map, - .stdin = .pipe, - .stdout = .pipe, - .stderr = .pipe, - .request_resource_usage_statistics = true, - .progress_node = prog_node, - }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); - - zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ - zp.child.stdout.?, zp.child.stderr.?, - }); - if (watch) s.cast(Compile).?.zig_process = zp; - defer if (!watch) zp.deinit(io); - - const result = result: { - defer if (watch) zp.saveState(prog_node); - break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); - }; - - if (!watch) { - // Send EOF to stdin. - zp.child.stdin.?.close(io); - zp.child.stdin = null; - - const term = zp.child.wait(io) catch |err| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); - }; - s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; - - // Special handling for Compile step that is expecting compile errors. - if (s.cast(Compile)) |compile| switch (term) { - .exited => { - // Note that the exit code may be 0 in this case due to the - // compiler server protocol. - if (compile.expect_errors != null) { - return error.NeedCompileErrorCheck; - } - }, - else => {}, - }; - - try handleChildProcessTerm(s, term); - } - - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); - } - - return result; -} - -/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. -pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { - const b = s.owner; - const io = b.graph.io; - const src_path = src_lazy_path.getPath3(b, s); - try handleVerbose(b, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); - return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| - return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); -} - -/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. -pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { - const b = s.owner; - const io = b.graph.io; - try handleVerbose(b, .inherit, &.{ "install", "-d", dest_path }); - return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| - return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); -} - -fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build.WebServer, gpa: Allocator) !?Path { - const b = s.owner; - const arena = b.allocator; - const io = b.graph.io; - - const start_ts = Io.Clock.awake.now(io); - - try sendMessage(io, zp.child.stdin.?, .update); - if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); - - var result: ?Path = null; - var eos_err: error{EndOfStream}!void = {}; - - const stdout = zp.multi_reader.fileReader(0); - - while (true) { - const Header = std.zig.Server.Message.Header; - const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) { - error.EndOfStream => break, - error.ReadFailed => return stdout.err.?, - }; - const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) { - error.EndOfStream => |e| { - // Better to report the crash with stderr below, but we set - // this in case the child exits successfully while violating - // this protocol. - eos_err = e; - break; - }, - error.ReadFailed => return stdout.err.?, - }; - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) { - return s.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - } - }, - .error_bundle => { - s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); - // This message indicates the end of the update. - if (watch) break; - }, - .emit_digest => { - const EmitDigest = std.zig.Server.Message.EmitDigest; - const emit_digest: *align(1) const EmitDigest = @ptrCast(body); - s.result_cached = emit_digest.flags.cache_hit; - const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; - result = .{ - .root_dir = b.cache_root, - .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), - }; - }, - .file_system_inputs => { - s.clearWatchInputs(); - var it = std.mem.splitScalar(u8, body, 0); - while (it.next()) |prefixed_path| { - const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); - const sub_path = try arena.dupe(u8, prefixed_path[1..]); - const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; - switch (prefix_index) { - .cwd => { - const path: Build.Cache.Path = .{ - .root_dir = Build.Cache.Directory.cwd(), - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .zig_lib => zl: { - if (s.cast(Step.Compile)) |compile| { - if (compile.zig_lib_dir) |zig_lib_dir| { - const lp = try zig_lib_dir.join(arena, sub_path); - try addWatchInput(s, lp); - break :zl; - } - } - const path: Build.Cache.Path = .{ - .root_dir = s.owner.graph.zig_lib_directory, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .local_cache => { - const path: Build.Cache.Path = .{ - .root_dir = b.cache_root, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .global_cache => { - const path: Build.Cache.Path = .{ - .root_dir = s.owner.graph.global_cache_root, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - } - } - }, - .time_report => if (web_server) |ws| { - const TimeReport = std.zig.Server.Message.TimeReport; - const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); - ws.updateTimeReportCompile(.{ - .compile = s.cast(Step.Compile).?, - .use_llvm = tr.flags.use_llvm, - .stats = tr.stats, - .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), - .llvm_pass_timings_len = tr.llvm_pass_timings_len, - .files_len = tr.files_len, - .decls_len = tr.decls_len, - .trailing = body[@sizeOf(TimeReport)..], - }); - }, - else => {}, // ignore other messages - } - } - - s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()); - - const stderr_contents = zp.multi_reader.reader(1).buffered(); - if (stderr_contents.len > 0) { - try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); - } - - try eos_err; - - return result; -} - -pub fn getZigProcess(s: *Step) ?*ZigProcess { - return switch (s.id) { - .compile => s.cast(Compile).?.zig_process, - else => null, - }; -} - -fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 0, - }; - var w = file.writer(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -pub fn handleVerbose( - b: *Build, - cwd: std.process.Child.Cwd, - argv: []const []const u8, -) error{OutOfMemory}!void { - return handleVerbose2(b, cwd, null, argv); -} - -pub fn handleVerbose2( - b: *Build, - cwd: std.process.Child.Cwd, - opt_env: ?*const std.process.Environ.Map, - argv: []const []const u8, -) error{OutOfMemory}!void { - if (b.verbose) { - const graph = b.graph; - // Intention of verbose is to print all sub-process command lines to - // stderr before spawning them. - const text = try allocPrintCmd(b.allocator, cwd, if (opt_env) |env| .{ - .child = env, - .parent = &graph.environ_map, - } else null, argv); - std.debug.print("{s}\n", .{text}); - } -} - -/// Asserts that the caller has already populated `s.result_failed_command`. -pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { - if (!std.process.can_spawn) { - return s.fail("unable to spawn process: host cannot spawn child processes", .{}); - } -} - -/// Asserts that the caller has already populated `s.result_failed_command`. -pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { - assert(s.result_failed_command != null); - return switch (term) { - .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), - .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), - .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), - .unknown => s.fail("process terminated unexpectedly", .{}), - }; -} - -pub fn allocPrintCmd( - gpa: Allocator, - cwd: std.process.Child.Cwd, - opt_env: ?struct { - child: *const std.process.Environ.Map, - parent: *const std.process.Environ.Map, - }, - argv: []const []const u8, -) Allocator.Error![]u8 { - const shell = struct { - fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { - for (string) |c| { - if (switch (c) { - else => true, - '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, - '=' => is_argv0, - }) break; - } else return writer.writeAll(string); - - try writer.writeByte('"'); - for (string) |c| { - if (switch (c) { - std.ascii.control_code.nul => break, - '!', '"', '$', '\\', '`' => true, - else => !std.ascii.isPrint(c), - }) try writer.writeByte('\\'); - switch (c) { - std.ascii.control_code.nul => unreachable, - std.ascii.control_code.bel => try writer.writeByte('a'), - std.ascii.control_code.bs => try writer.writeByte('b'), - std.ascii.control_code.ht => try writer.writeByte('t'), - std.ascii.control_code.lf => try writer.writeByte('n'), - std.ascii.control_code.vt => try writer.writeByte('v'), - std.ascii.control_code.ff => try writer.writeByte('f'), - std.ascii.control_code.cr => try writer.writeByte('r'), - std.ascii.control_code.esc => try writer.writeByte('E'), - ' '...'~' => try writer.writeByte(c), - else => try writer.print("{o:0>3}", .{c}), - } - } - try writer.writeByte('"'); - } - }; - - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - const writer = &aw.writer; - switch (cwd) { - .inherit => {}, - .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, - .dir => @panic("TODO"), - } - if (opt_env) |env| { - var it = env.child.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; - if (env.parent.get(key)) |process_value| { - if (std.mem.eql(u8, value, process_value)) continue; - } - writer.print("{s}=", .{key}) catch return error.OutOfMemory; - shell.escape(writer, value, false) catch return error.OutOfMemory; - writer.writeByte(' ') catch return error.OutOfMemory; - } - } - shell.escape(writer, argv[0], true) catch return error.OutOfMemory; - for (argv[1..]) |arg| { - writer.writeByte(' ') catch return error.OutOfMemory; - shell.escape(writer, arg, false) catch return error.OutOfMemory; - } - return aw.toOwnedSlice(); -} - -/// Prefer `cacheHitAndWatch` unless you already added watch inputs -/// separately from using the cache system. -pub fn cacheHit(s: *Step, man: *Build.Cache.Manifest) !bool { - s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); - return s.result_cached; -} - -/// Clears previous watch inputs, if any, and then populates watch inputs from -/// the full set of files picked up by the cache manifest. -/// -/// Must be accompanied with `writeManifestAndWatch`. -pub fn cacheHitAndWatch(s: *Step, man: *Build.Cache.Manifest) !bool { - const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); - s.result_cached = is_hit; - // The above call to hit() populates the manifest with files, so in case of - // a hit, we need to populate watch inputs. - if (is_hit) try setWatchInputsFromManifest(s, man); - return is_hit; -} - -fn failWithCacheError( - s: *Step, - man: *const Build.Cache.Manifest, - err: Build.Cache.Manifest.HitError, -) error{ OutOfMemory, Canceled, MakeFailed } { - switch (err) { - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ - man.diagnostic, e, - }), - .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ - prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, - }); - }, - }, - error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), - } -} - -/// Prefer `writeManifestAndWatch` unless you already added watch inputs -/// separately from using the cache system. -pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void { - if (s.test_results.isSuccess()) { - man.writeManifest() catch |err| { - try s.addError("unable to write cache manifest: {t}", .{err}); - }; - } -} - -/// Clears previous watch inputs, if any, and then populates watch inputs from -/// the full set of files picked up by the cache manifest. -/// -/// Must be accompanied with `cacheHitAndWatch`. -pub fn writeManifestAndWatch(s: *Step, man: *Build.Cache.Manifest) !void { - try writeManifest(s, man); - try setWatchInputsFromManifest(s, man); -} - -fn setWatchInputsFromManifest(s: *Step, man: *Build.Cache.Manifest) !void { - const arena = s.owner.allocator; - const prefixes = man.cache.prefixes(); - clearWatchInputs(s); - for (man.files.keys()) |file| { - // The file path data is freed when the cache manifest is cleaned up at the end of `make`. - const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); - try addWatchInputFromPath(s, .{ - .root_dir = prefixes[file.prefixed_path.prefix], - .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, std.fs.path.basename(sub_path)); - } -} - -/// For steps that have a single input that never changes when re-running `make`. -pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { - if (!step.inputs.populated()) try step.addWatchInput(lazy_path); -} - -pub fn clearWatchInputs(step: *Step) void { - const gpa = step.owner.allocator; - step.inputs.clear(gpa); -} - -/// Places a *file* dependency on the path. -pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!void { - switch (lazy_file) { - .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), - .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), - .cwd_relative => |path_string| { - try addWatchInputFromPath(step, .{ - .root_dir = .{ - .path = null, - .handle = Io.Dir.cwd(), - }, - .sub_path = std.fs.path.dirname(path_string) orelse "", - }, std.fs.path.basename(path_string)); - }, - // Nothing to watch because this dependency edge is modeled instead via `dependants`. - .generated => {}, - } -} - -/// Any changes inside the directory will trigger invalidation. -/// -/// See also `addDirectoryWatchInputFromPath` which takes a `Build.Cache.Path` instead. -/// -/// Paths derived from this directory should also be manually added via -/// `addDirectoryWatchInputFromPath` if and only if this function returns -/// `true`. -pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool { - switch (lazy_directory) { - .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), - .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), - .cwd_relative => |path_string| { - try addDirectoryWatchInputFromPath(step, .{ - .root_dir = .{ - .path = null, - .handle = Io.Dir.cwd(), - }, - .sub_path = path_string, - }); - }, - // Nothing to watch because this dependency edge is modeled instead via `dependants`. - .generated => return false, - } - return true; -} - -/// Any changes inside the directory will trigger invalidation. -/// -/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. -/// -/// This function should only be called when it has been verified that the -/// dependency on `path` is not already accounted for by a `Step` dependency. -/// In other words, before calling this function, first check that the -/// `Build.LazyPath` which this `path` is derived from is not `generated`. -pub fn addDirectoryWatchInputFromPath(step: *Step, path: Build.Cache.Path) !void { - return addWatchInputFromPath(step, path, "."); -} - -fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { - return addWatchInputFromPath(step, .{ - .root_dir = builder.build_root, - .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, std.fs.path.basename(sub_path)); -} - -fn addDirectoryWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { - return addDirectoryWatchInputFromPath(step, .{ - .root_dir = builder.build_root, - .sub_path = sub_path, - }); -} - -fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const u8) !void { - const gpa = step.owner.allocator; - const gop = try step.inputs.table.getOrPut(gpa, path); - if (!gop.found_existing) gop.value_ptr.* = .empty; - try gop.value_ptr.append(gpa, basename); -} - -/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. -pub fn reset(step: *Step, gpa: Allocator) void { - assert(step.state == .precheck_done); - - if (step.result_failed_command) |cmd| gpa.free(cmd); - - step.result_error_msgs.clearRetainingCapacity(); - step.result_stderr = ""; - step.result_cached = false; - step.result_duration_ns = null; - step.result_peak_rss = 0; - step.result_failed_command = null; - step.test_results = .{}; - step.clearWatchInputs(); - - step.result_error_bundle.deinit(gpa); - step.result_error_bundle = std.zig.ErrorBundle.empty; -} - -/// Implementation detail of file watching. Prepares the step for being re-evaluated. -/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. -pub fn invalidateResult(step: *Step, gpa: Allocator) bool { - if (step.state == .precheck_done) return false; - assert(step.pending_deps == 0); - step.state = .precheck_done; - step.reset(gpa); - for (step.dependants.items) |dependant| { - _ = dependant.invalidateResult(gpa); - dependant.pending_deps += 1; - } - return true; -} - test { _ = CheckFile; _ = Fail; diff --git a/lib/std/Build/Step/CheckFile.zig b/lib/std/Build/Step/CheckFile.zig index 1c3813ca824bbea35e6e15b19567dd1b47c212cc..4cb968ff96ef2343153d1f522406ac35db088d7d 100644 --- a/lib/std/Build/Step/CheckFile.zig +++ b/lib/std/Build/Step/CheckFile.zig @@ -16,7 +16,7 @@ expected_exact: ?[]const u8, source: std.Build.LazyPath, max_bytes: usize = 20 * 1024 * 1024, -pub const base_id: Step.Id = .check_file; +pub const base_tag: Step.Tag = .check_file; pub const Options = struct { expected_matches: []const []const u8 = &.{}, @@ -31,7 +31,7 @@ pub fn create( const check_file = owner.allocator.create(CheckFile) catch @panic("OOM"); check_file.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "CheckFile", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 5ce7bcb9d6b49d4baa6676bff610c46ff1975704..2897fddb747f199a72381070dfbfd28787e41734 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -20,7 +20,7 @@ const InstallDir = std.Build.InstallDir; const GeneratedFile = std.Build.GeneratedFile; const Path = std.Build.Cache.Path; -pub const base_id: Step.Id = .compile; +pub const base_tag: Step.Tag = .compile; step: Step, root_module: *Module, @@ -235,10 +235,6 @@ is_linking_libc: bool = false, /// Computed during make(). is_linking_libcpp: bool = false, -/// Populated during the make phase when there is a long-lived compiler process. -/// Managed by the build runner, not user build script. -zig_process: ?*Step.ZigProcess, - /// Enables coverage instrumentation that is only useful if you are using third /// party fuzzers that depend on it. Otherwise, slows down the instrumented /// binary with unnecessary function calls. @@ -418,10 +414,9 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .kind = options.kind, .name = name, .step = .init(.{ - .id = base_id, + .tag = base_tag, .name = step_name, .owner = owner, - .makeFn = make, .max_rss = options.max_rss, }), .version = options.version, @@ -452,8 +447,6 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .use_new_linker = null, - - .zig_process = null, }; if (options.zig_lib_dir) |lp| { @@ -701,122 +694,6 @@ pub fn producesImplib(compile: *Compile) bool { return compile.isDll(); } -const PkgConfigResult = struct { - cflags: []const []const u8, - libs: []const []const u8, -}; - -/// Run pkg-config for the given library name and parse the output, returning the arguments -/// that should be passed to zig to link the given library. -pub fn runPkgConfig(step: *Step, lib_name: []const u8) !PkgConfigResult { - const wl_rpath_prefix = "-Wl,-rpath,"; - - const b = step.owner; - const pkg_name = match: { - // First we have to map the library name to pkg config name. Unfortunately, - // there are several examples where this is not straightforward: - // -lSDL2 -> pkg-config sdl2 - // -lgdk-3 -> pkg-config gdk-3.0 - // -latk-1.0 -> pkg-config atk - // -lpulse -> pkg-config libpulse - const pkgs = try getPkgConfigList(b); - - // Exact match means instant winner. - for (pkgs) |pkg| { - if (mem.eql(u8, pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Next we'll try ignoring case. - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Prefixed "lib" or suffixed ".0". - for (pkgs) |pkg| { - if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { - const prefix = pkg.name[0..pos]; - const suffix = pkg.name[pos + lib_name.len ..]; - if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; - if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; - break :match pkg.name; - } - } - - // Trimming "-1.0". - if (mem.endsWith(u8, lib_name, "-1.0")) { - const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { - break :match pkg.name; - } - } - } - - return error.PackageNotFound; - }; - - var code: u8 = undefined; - const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = if (b.runAllowFail(&[_][]const u8{ - pkg_config_exe, - pkg_name, - "--cflags", - "--libs", - }, &code, .ignore)) |stdout| stdout else |err| switch (err) { - error.ProcessTerminated => return error.PkgConfigCrashed, - error.ExecNotSupported => return error.PkgConfigFailed, - error.ExitCodeFailure => return error.PkgConfigFailed, - error.FileNotFound => return error.PkgConfigNotInstalled, - else => return err, - }; - - var zig_cflags: std.ArrayList([]const u8) = .empty; - defer zig_cflags.deinit(b.allocator); - var zig_libs: std.ArrayList([]const u8) = .empty; - defer zig_libs.deinit(b.allocator); - - var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); - while (arg_it.next()) |arg| { - if (mem.eql(u8, arg, "-I")) { - const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_cflags.appendSlice(b.allocator, &.{ "-I", dir }); - } else if (mem.startsWith(u8, arg, "-I")) { - try zig_cflags.append(b.allocator, arg); - } else if (mem.eql(u8, arg, "-L")) { - const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_libs.appendSlice(b.allocator, &.{ "-L", dir }); - } else if (mem.startsWith(u8, arg, "-L")) { - try zig_libs.append(b.allocator, arg); - } else if (mem.eql(u8, arg, "-l")) { - const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_libs.appendSlice(b.allocator, &.{ "-l", lib }); - } else if (mem.startsWith(u8, arg, "-l")) { - try zig_libs.append(b.allocator, arg); - } else if (mem.eql(u8, arg, "-D")) { - const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_cflags.appendSlice(b.allocator, &.{ "-D", macro }); - } else if (mem.startsWith(u8, arg, "-D")) { - try zig_cflags.append(b.allocator, arg); - } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { - try zig_cflags.appendSlice(b.allocator, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); - } else if (b.debug_pkg_config) { - return step.fail("unknown pkg-config flag '{s}'", .{arg}); - } - } - - try zig_cflags.shrinkToLen(b.allocator); - try zig_libs.shrinkToLen(b.allocator); - - return .{ - .cflags = zig_cflags.toOwnedSliceAssert(), - .libs = zig_libs.toOwnedSliceAssert(), - }; -} - pub fn setVerboseLink(compile: *Compile, value: bool) void { compile.verbose_link = value; } @@ -974,863 +851,6 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking return path; } -fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { - const step = &compile.step; - const b = step.owner; - const arena = b.allocator; - - var zig_args = std.array_list.Managed([]const u8).init(arena); - defer zig_args.deinit(); - - try zig_args.append(b.graph.zig_exe); - - const cmd = switch (compile.kind) { - .lib => "build-lib", - .exe => "build-exe", - .obj => "build-obj", - .@"test" => "test", - .test_obj => "test-obj", - }; - try zig_args.append(cmd); - - if (b.reference_trace) |some| { - try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); - } - try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); - - try addFlag(&zig_args, "llvm", compile.use_llvm); - try addFlag(&zig_args, "lld", compile.use_lld); - try addFlag(&zig_args, "new-linker", compile.use_new_linker); - - if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { - try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); - } - - switch (compile.entry) { - .default => {}, - .disabled => try zig_args.append("-fno-entry"), - .enabled => try zig_args.append("-fentry"), - .symbol_name => |entry_name| { - try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); - }, - } - - { - var symbol_it = compile.force_undefined_symbols.keyIterator(); - while (symbol_it.next()) |symbol_name| { - try zig_args.append("--force_undefined"); - try zig_args.append(symbol_name.*); - } - } - - if (compile.stack_size) |stack_size| { - try zig_args.append("--stack"); - try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); - } - - if (fuzz) { - try zig_args.append("-ffuzz"); - } - - { - // Stores system libraries that have already been seen for at least one - // module, along with any arguments that need to be passed to the - // compiler for each module individually. - var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; - var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; - - var prev_has_cflags = false; - var prev_has_rcflags = false; - var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; - var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; - // Track the number of positional arguments so that a nice error can be - // emitted if there is nothing to link. - var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); - - // Fully recursive iteration including dynamic libraries to detect - // libc and libc++ linkage. - for (compile.getCompileDependencies(true)) |some_compile| { - for (some_compile.root_module.getGraph().modules) |mod| { - if (mod.link_libc == true) compile.is_linking_libc = true; - if (mod.link_libcpp == true) compile.is_linking_libcpp = true; - } - } - - var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); - - // For this loop, don't chase dynamic libraries because their link - // objects are already linked. - for (compile.getCompileDependencies(false)) |dep_compile| { - for (dep_compile.root_module.getGraph().modules) |mod| { - // While walking transitive dependencies, if a given link object is - // already included in a library, it should not redundantly be - // placed on the linker line of the dependee. - const my_responsibility = dep_compile == compile; - const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); - - // Inherit dependencies on darwin frameworks. - if (!already_linked) { - for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { - try frameworks.put(arena, name, info); - } - } - - // Inherit dependencies on system libraries and static libraries. - for (mod.link_objects.items) |link_object| { - switch (link_object) { - .static_path => |static_path| { - if (my_responsibility) { - try zig_args.append(static_path.getPath2(mod.owner, step)); - total_linker_objects += 1; - } - }, - .system_lib => |system_lib| { - const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); - if (system_lib_gop.found_existing) { - try zig_args.appendSlice(system_lib_gop.value_ptr.*); - continue; - } else { - system_lib_gop.value_ptr.* = &.{}; - } - - if (already_linked) - continue; - - if ((system_lib.search_strategy != prev_search_strategy or - system_lib.preferred_link_mode != prev_preferred_link_mode) and - compile.linkage != .static) - { - switch (system_lib.search_strategy) { - .no_fallback => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_dylibs_only"), - .static => try zig_args.append("-search_static_only"), - }, - .paths_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_paths_first"), - .static => try zig_args.append("-search_paths_first_static"), - }, - .mode_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_dylibs_first"), - .static => try zig_args.append("-search_static_first"), - }, - } - prev_search_strategy = system_lib.search_strategy; - prev_preferred_link_mode = system_lib.preferred_link_mode; - } - - const prefix: []const u8 = prefix: { - if (system_lib.needed) break :prefix "-needed-l"; - if (system_lib.weak) break :prefix "-weak-l"; - break :prefix "-l"; - }; - switch (system_lib.use_pkg_config) { - .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), - .yes, .force => { - if (runPkgConfig(&compile.step, system_lib.name)) |result| { - try zig_args.appendSlice(result.cflags); - try zig_args.appendSlice(result.libs); - try seen_system_libs.put(arena, system_lib.name, result.cflags); - } else |err| switch (err) { - error.PkgConfigInvalidOutput, - error.PkgConfigCrashed, - error.PkgConfigFailed, - error.PkgConfigNotInstalled, - error.PackageNotFound, - => switch (system_lib.use_pkg_config) { - .yes => { - // pkg-config failed, so fall back to linking the library - // by name directly. - try zig_args.append(b.fmt("{s}{s}", .{ - prefix, - system_lib.name, - })); - }, - .force => { - panic("pkg-config failed for library {s}", .{system_lib.name}); - }, - .no => unreachable, - }, - - else => |e| return e, - } - }, - } - }, - .other_step => |other| { - switch (other.kind) { - .exe => return step.fail("cannot link with an executable build artifact", .{}), - .@"test" => return step.fail("cannot link with a test", .{}), - .obj, .test_obj => { - const included_in_lib_or_obj = !my_responsibility and - (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); - if (!already_linked and !included_in_lib_or_obj) { - try zig_args.append(other.getEmittedBin().getPath2(b, step)); - total_linker_objects += 1; - } - }, - .lib => l: { - const other_produces_implib = other.producesImplib(); - const other_is_static = other_produces_implib or other.isStaticLibrary(); - - if (compile.isStaticLibrary() and other_is_static) { - // Avoid putting a static library inside a static library. - break :l; - } - - // For DLLs, we must link against the implib. - // For everything else, we directly link - // against the library file. - const full_path_lib = if (other_produces_implib) - try other.getGeneratedFilePath("generated_implib", &compile.step) - else - try other.getGeneratedFilePath("generated_bin", &compile.step); - - try zig_args.append(full_path_lib); - total_linker_objects += 1; - - if (other.linkage == .dynamic and - compile.rootModuleTarget().os.tag != .windows) - { - if (fs.path.dirname(full_path_lib)) |dirname| { - try zig_args.append("-rpath"); - try zig_args.append(dirname); - } - } - }, - } - }, - .assembly_file => |asm_file| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags) { - try zig_args.append("-cflags"); - try zig_args.append("--"); - prev_has_cflags = false; - } - try zig_args.append(asm_file.getPath2(mod.owner, step)); - total_linker_objects += 1; - }, - - .c_source_file => |c_source_file| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags or c_source_file.flags.len != 0) { - try zig_args.append("-cflags"); - for (c_source_file.flags) |arg| { - try zig_args.append(arg); - } - try zig_args.append("--"); - } - prev_has_cflags = (c_source_file.flags.len != 0); - - if (c_source_file.language) |lang| { - try zig_args.append("-x"); - try zig_args.append(lang.internalIdentifier()); - } - - try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); - - if (c_source_file.language != null) { - try zig_args.append("-x"); - try zig_args.append("none"); - } - total_linker_objects += 1; - }, - - .c_source_files => |c_source_files| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags or c_source_files.flags.len != 0) { - try zig_args.append("-cflags"); - for (c_source_files.flags) |arg| { - try zig_args.append(arg); - } - try zig_args.append("--"); - } - prev_has_cflags = (c_source_files.flags.len != 0); - - if (c_source_files.language) |lang| { - try zig_args.append("-x"); - try zig_args.append(lang.internalIdentifier()); - } - - const root_path = c_source_files.root.getPath2(mod.owner, step); - for (c_source_files.files) |file| { - try zig_args.append(b.pathJoin(&.{ root_path, file })); - } - - if (c_source_files.language != null) { - try zig_args.append("-x"); - try zig_args.append("none"); - } - - total_linker_objects += c_source_files.files.len; - }, - - .win32_resource_file => |rc_source_file| l: { - if (!my_responsibility) break :l; - - if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { - if (prev_has_rcflags) { - try zig_args.append("-rcflags"); - try zig_args.append("--"); - prev_has_rcflags = false; - } - } else { - try zig_args.append("-rcflags"); - for (rc_source_file.flags) |arg| { - try zig_args.append(arg); - } - for (rc_source_file.include_paths) |include_path| { - try zig_args.append("/I"); - try zig_args.append(include_path.getPath2(mod.owner, step)); - } - try zig_args.append("--"); - prev_has_rcflags = true; - } - try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); - total_linker_objects += 1; - }, - } - } - - // We need to emit the --mod argument here so that the above link objects - // have the correct parent module, but only if the module is part of - // this compilation. - if (!my_responsibility) continue; - if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { - const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; - try mod.appendZigProcessFlags(&zig_args, step); - - // --dep arguments - try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); - for (mod.import_table.keys(), mod.import_table.values()) |name, import| { - const import_index = cli_named_modules.modules.getIndex(import).?; - const import_cli_name = cli_named_modules.names.keys()[import_index]; - zig_args.appendAssumeCapacity("--dep"); - if (std.mem.eql(u8, import_cli_name, name)) { - zig_args.appendAssumeCapacity(import_cli_name); - } else { - zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); - } - } - - // When the CLI sees a -M argument, it determines whether it - // implies the existence of a Zig compilation unit based on - // whether there is a root source file. If there is no root - // source file, then this is not a zig compilation unit - it is - // perhaps a set of linker objects, or C source files instead. - // Linker objects are added to the CLI globally, while C source - // files must have a module parent. - if (mod.root_source_file) |lp| { - const src = lp.getPath2(mod.owner, step); - try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); - } else if (moduleNeedsCliArg(mod)) { - try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); - } - } - } - } - - if (total_linker_objects == 0) { - return step.fail("the linker needs one or more objects to link", .{}); - } - - for (frameworks.keys(), frameworks.values()) |name, info| { - if (info.needed) { - try zig_args.append("-needed_framework"); - } else if (info.weak) { - try zig_args.append("-weak_framework"); - } else { - try zig_args.append("-framework"); - } - try zig_args.append(name); - } - - if (compile.is_linking_libcpp) { - try zig_args.append("-lc++"); - } - - if (compile.is_linking_libc) { - try zig_args.append("-lc"); - } - } - - if (compile.win32_manifest) |manifest_file| { - try zig_args.append(manifest_file.getPath2(b, step)); - } - - if (compile.win32_module_definition) |module_file| { - try zig_args.append(module_file.getPath2(b, step)); - } - - if (compile.image_base) |image_base| { - try zig_args.append("--image-base"); - try zig_args.append(b.fmt("0x{x}", .{image_base})); - } - - for (compile.filters) |filter| { - try zig_args.append("--test-filter"); - try zig_args.append(filter); - } - - if (compile.test_runner) |test_runner| { - try zig_args.append("--test-runner"); - try zig_args.append(test_runner.path.getPath2(b, step)); - } - - for (b.debug_log_scopes) |log_scope| { - try zig_args.append("--debug-log"); - try zig_args.append(log_scope); - } - - if (b.debug_compile_errors) { - try zig_args.append("--debug-compile-errors"); - } - - if (b.debug_incremental) { - try zig_args.append("--debug-incremental"); - } - - if (b.verbose_air) try zig_args.append("--verbose-air"); - if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); - if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); - if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); - if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); - if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); - if (b.graph.time_report) try zig_args.append("--time-report"); - - if (compile.generated_asm != null) try zig_args.append("-femit-asm"); - if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); - if (compile.generated_docs != null) try zig_args.append("-femit-docs"); - if (compile.generated_implib != null) try zig_args.append("-femit-implib"); - if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); - if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); - if (compile.generated_h != null) try zig_args.append("-femit-h"); - - try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); - - switch (compile.compress_debug_sections) { - .none => {}, - .zlib => try zig_args.append("--compress-debug-sections=zlib"), - .zstd => try zig_args.append("--compress-debug-sections=zstd"), - } - - if (compile.link_eh_frame_hdr) { - try zig_args.append("--eh-frame-hdr"); - } - if (compile.link_emit_relocs) { - try zig_args.append("--emit-relocs"); - } - if (compile.link_function_sections) { - try zig_args.append("-ffunction-sections"); - } - if (compile.link_data_sections) { - try zig_args.append("-fdata-sections"); - } - if (compile.link_gc_sections) |x| { - try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); - } - if (!compile.linker_dynamicbase) { - try zig_args.append("--no-dynamicbase"); - } - if (compile.linker_allow_shlib_undefined) |x| { - try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); - } - if (compile.link_z_notext) { - try zig_args.append("-z"); - try zig_args.append("notext"); - } - if (!compile.link_z_relro) { - try zig_args.append("-z"); - try zig_args.append("norelro"); - } - if (compile.link_z_lazy) { - try zig_args.append("-z"); - try zig_args.append("lazy"); - } - if (compile.link_z_common_page_size) |size| { - try zig_args.append("-z"); - try zig_args.append(b.fmt("common-page-size={d}", .{size})); - } - if (compile.link_z_max_page_size) |size| { - try zig_args.append("-z"); - try zig_args.append(b.fmt("max-page-size={d}", .{size})); - } - if (compile.link_z_defs) { - try zig_args.append("-z"); - try zig_args.append("defs"); - } - - if (compile.libc_file) |libc_file| { - try zig_args.append("--libc"); - try zig_args.append(libc_file.getPath2(b, step)); - } else if (b.libc_file) |libc_file| { - try zig_args.append("--libc"); - try zig_args.append(libc_file); - } - - try zig_args.append("--cache-dir"); - try zig_args.append(b.cache_root.path orelse "."); - - try zig_args.append("--global-cache-dir"); - try zig_args.append(b.graph.global_cache_root.path orelse "."); - - if (b.graph.debug_compiler_runtime_libs) |mode| - try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); - - try zig_args.append("--name"); - try zig_args.append(compile.name); - - if (compile.linkage) |some| switch (some) { - .dynamic => try zig_args.append("-dynamic"), - .static => try zig_args.append("-static"), - }; - if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { - if (compile.version) |version| { - try zig_args.append("--version"); - try zig_args.append(b.fmt("{f}", .{version})); - } - - if (compile.rootModuleTarget().os.tag.isDarwin()) { - const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ - compile.rootModuleTarget().libPrefix(), - compile.name, - compile.rootModuleTarget().dynamicLibSuffix(), - }); - try zig_args.append("-install_name"); - try zig_args.append(install_name); - } - } - - if (compile.entitlements) |entitlements| { - try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); - } - if (compile.pagezero_size) |pagezero_size| { - const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); - try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); - } - if (compile.headerpad_size) |headerpad_size| { - const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); - try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); - } - if (compile.headerpad_max_install_names) { - try zig_args.append("-headerpad_max_install_names"); - } - if (compile.dead_strip_dylibs) { - try zig_args.append("-dead_strip_dylibs"); - } - if (compile.force_load_objc) { - try zig_args.append("-ObjC"); - } - if (compile.discard_local_symbols) { - try zig_args.append("--discard-all"); - } - - try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); - try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); - try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); - if (compile.rdynamic) { - try zig_args.append("-rdynamic"); - } - if (compile.import_memory) { - try zig_args.append("--import-memory"); - } - if (compile.export_memory) { - try zig_args.append("--export-memory"); - } - if (compile.import_symbols) { - try zig_args.append("--import-symbols"); - } - if (compile.import_table) { - try zig_args.append("--import-table"); - } - if (compile.export_table) { - try zig_args.append("--export-table"); - } - if (compile.initial_memory) |initial_memory| { - try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); - } - if (compile.max_memory) |max_memory| { - try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); - } - if (compile.shared_memory) { - try zig_args.append("--shared-memory"); - } - if (compile.global_base) |global_base| { - try zig_args.append(b.fmt("--global-base={d}", .{global_base})); - } - - if (compile.wasi_exec_model) |model| { - try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); - } - if (compile.linker_script) |linker_script| { - try zig_args.append("--script"); - try zig_args.append(linker_script.getPath2(b, step)); - } - - if (compile.version_script) |version_script| { - try zig_args.append("--version-script"); - try zig_args.append(version_script.getPath2(b, step)); - } - if (compile.linker_allow_undefined_version) |x| { - try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); - } - - if (compile.linker_enable_new_dtags) |enabled| { - try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); - } - - if (compile.kind == .@"test") { - if (compile.exec_cmd_args) |exec_cmd_args| { - for (exec_cmd_args) |cmd_arg| { - if (cmd_arg) |arg| { - try zig_args.append("--test-cmd"); - try zig_args.append(arg); - } else { - try zig_args.append("--test-cmd-bin"); - } - } - } - } - - if (b.sysroot) |sysroot| { - try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); - } - - // -I and -L arguments that appear after the last --mod argument apply to all modules. - const cwd: Io.Dir = .cwd(); - const io = b.graph.io; - - for (b.search_prefixes.items) |search_prefix| { - var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { - return step.fail("unable to open prefix directory '{s}': {s}", .{ - search_prefix, @errorName(err), - }); - }; - defer prefix_dir.close(io); - - // Avoid passing -L and -I flags for nonexistent directories. - // This prevents a warning, that should probably be upgraded to an error in Zig's - // CLI parsing code, when the linker sees an -L directory that does not exist. - - if (prefix_dir.access(io, "lib", .{})) |_| { - try zig_args.appendSlice(&.{ - "-L", b.pathJoin(&.{ search_prefix, "lib" }), - }); - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ - search_prefix, @errorName(e), - }), - } - - if (prefix_dir.access(io, "include", .{})) |_| { - try zig_args.appendSlice(&.{ - "-I", b.pathJoin(&.{ search_prefix, "include" }), - }); - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ - search_prefix, @errorName(e), - }), - } - } - - if (compile.rc_includes != .any) { - try zig_args.append("-rcincludes"); - try zig_args.append(@tagName(compile.rc_includes)); - } - - try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); - - if (compile.build_id orelse b.build_id) |build_id| { - try zig_args.append(switch (build_id) { - .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), - .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), - }); - } - - const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| - dir.getPath2(b, step) - else if (b.graph.zig_lib_directory.path) |_| - b.fmt("{f}", .{b.graph.zig_lib_directory}) - else - null; - - if (opt_zig_lib_dir) |zig_lib_dir| { - try zig_args.append("--zig-lib-dir"); - try zig_args.append(zig_lib_dir); - } - - try addFlag(&zig_args, "PIE", compile.pie); - - if (compile.lto) |lto| { - try zig_args.append(switch (lto) { - .full => "-flto=full", - .thin => "-flto=thin", - .none => "-fno-lto", - }); - } - - try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); - - if (compile.subsystem) |subsystem| { - try zig_args.append("--subsystem"); - try zig_args.append(@tagName(subsystem)); - } - - if (compile.mingw_unicode_entry_point) { - try zig_args.append("-municode"); - } - - if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ - "--error-limit", b.fmt("{d}", .{err_limit}), - }); - - try addFlag(&zig_args, "incremental", b.graph.incremental); - - try zig_args.append("--listen=-"); - - // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux - // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and - // pass that to zig, e.g. via 'zig build-lib @args.rsp' - // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html - var args_length: usize = 0; - for (zig_args.items) |arg| { - args_length += arg.len + 1; // +1 to account for null terminator - } - if (args_length >= 30 * 1024) { - try b.cache_root.handle.createDirPath(io, "args"); - - const args_to_escape = zig_args.items[2..]; - var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); - arg_blk: for (args_to_escape) |arg| { - for (arg, 0..) |c, arg_idx| { - if (c == '\\' or c == '"') { - // Slow path for arguments that need to be escaped. We'll need to allocate and copy - var escaped: std.ArrayList(u8) = .empty; - try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1); - try escaped.appendSlice(arena, arg[0..arg_idx]); - for (arg[arg_idx..]) |to_escape| { - if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\'); - try escaped.append(arena, to_escape); - } - escaped_args.appendAssumeCapacity(escaped.items); - continue :arg_blk; - } - } - escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument - } - - // Write the args to zig-cache/args/ to avoid conflicts with - // other zig build commands running in parallel. - const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); - const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); - - var args_hash: [Sha256.digest_length]u8 = undefined; - Sha256.hash(args, &args_hash, .{}); - var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; - _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); - - const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; - if (b.cache_root.handle.access(io, args_file, .{})) |_| { - // The args file is already present from a previous run. - } else |err| switch (err) { - error.FileNotFound => { - var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ - .replace = false, - .make_path = true, - }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ - b.cache_root, args_file, e, - }); - defer af.deinit(io); - - af.file.writeStreamingAll(io, args) catch |e| { - return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ - b.cache_root, args_file, e, - }); - }; - // Note we can't clean up this file, not even after build - // success, because that might interfere with another build - // process that needs the same file. - af.link(io) catch |e| switch (e) { - error.PathAlreadyExists => { - // The args file was created by another concurrent build process. - }, - else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ - b.cache_root, args_file, other_err, - }), - }; - }, - else => |other_err| return other_err, - } - - const resolved_args_file = try mem.concat(arena, u8, &.{ - "@", - try b.cache_root.join(arena, &.{args_file}), - }); - - zig_args.shrinkRetainingCapacity(2); - try zig_args.append(resolved_args_file); - } - - return try zig_args.toOwnedSlice(); -} - -fn make(step: *Step, options: Step.MakeOptions) !void { - const b = step.owner; - const compile: *Compile = @fieldParentPtr("step", step); - - const zig_args = try getZigArgs(compile, false); - - const maybe_output_dir = step.evalZigProcess( - zig_args, - options.progress_node, - (b.graph.incremental == true) and (options.watch or options.web_server != null), - options.web_server, - options.gpa, - ) catch |err| switch (err) { - error.NeedCompileErrorCheck => { - assert(compile.expect_errors != null); - try checkCompileErrors(compile); - return; - }, - else => |e| return e, - }; - - // Update generated files - if (maybe_output_dir) |output_dir| { - if (compile.emit_directory) |lp| { - lp.path = b.fmt("{f}", .{output_dir}); - } - - // zig fmt: off - if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); - if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); - // hack for stage2_x86_64 + coff - if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); - if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); - if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); - if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); - if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); - if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); - if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); - // zig fmt: on - } - - if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and - compile.version != null and compile.generated_bin != null and - std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) - { - try doAtomicSymLinks( - step, - compile.getEmittedBin().getPath2(b, step), - compile.major_only_filename.?, - compile.name_only_filename.?, - ); - } -} fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifact) []const u8 { const arena = c.step.owner.graph.arena; const name = ea.cacheName(arena, .{ @@ -1847,100 +867,6 @@ fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifa return out_dir.joinString(arena, name) catch @panic("OOM"); } -pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { - c.step.result_error_msgs.clearRetainingCapacity(); - c.step.result_stderr = ""; - - c.step.result_error_bundle.deinit(gpa); - c.step.result_error_bundle = std.zig.ErrorBundle.empty; - - if (c.step.result_failed_command) |cmd| { - gpa.free(cmd); - c.step.result_failed_command = null; - } - - const zig_args = try getZigArgs(c, true); - const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); - return maybe_output_bin_path.?; -} - -pub fn doAtomicSymLinks( - step: *Step, - output_path: []const u8, - filename_major_only: []const u8, - filename_name_only: []const u8, -) !void { - const b = step.owner; - const io = b.graph.io; - const out_dir = fs.path.dirname(output_path) orelse "."; - const out_basename = fs.path.basename(output_path); - // sym link for libfoo.so.1 to libfoo.so.1.2.3 - const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); - const cwd: Io.Dir = .cwd(); - cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { - return step.fail("unable to symlink {s} -> {s}: {s}", .{ - major_only_path, out_basename, @errorName(err), - }); - }; - // sym link for libfoo.so to libfoo.so.1 - const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); - cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { - return step.fail("Unable to symlink {s} -> {s}: {s}", .{ - name_only_path, filename_major_only, @errorName(err), - }); - }; -} - -fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { - const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); - var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); - errdefer list.deinit(); - var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); - while (line_it.next()) |line| { - if (mem.trim(u8, line, " \t").len == 0) continue; - var tok_it = mem.tokenizeAny(u8, line, " \t"); - try list.append(PkgConfigPkg{ - .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, - .desc = tok_it.rest(), - }); - } - return list.toOwnedSlice(); -} - -fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { - if (b.pkg_config_pkg_list) |res| { - return res; - } - var code: u8 = undefined; - if (execPkgConfigList(b, &code)) |list| { - b.pkg_config_pkg_list = list; - return list; - } else |err| { - const result = switch (err) { - error.ProcessTerminated => error.PkgConfigCrashed, - error.ExecNotSupported => error.PkgConfigFailed, - error.ExitCodeFailure => error.PkgConfigFailed, - error.FileNotFound => error.PkgConfigNotInstalled, - error.InvalidName => error.PkgConfigNotInstalled, - error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, - else => return err, - }; - b.pkg_config_pkg_list = result; - return result; - } -} - -fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { - const cond = opt orelse return; - try args.ensureUnusedCapacity(1); - if (cond) { - args.appendAssumeCapacity("-f" ++ name); - } else { - args.appendAssumeCapacity("-fno-" ++ name); - } -} - fn checkCompileErrors(compile: *Compile) !void { // Clear this field so that it does not get printed by the build runner. const actual_eb = compile.step.result_error_bundle; diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index 23c977296600c7eef619f49d56a12839159db141..2406250b4764d59fed65b8186e820f32b7078ecb 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -47,7 +47,7 @@ max_bytes: usize, include_path: []const u8, include_guard_override: ?[]const u8, -pub const base_id: Step.Id = .config_header; +pub const base_tag: Step.Tag = .config_header; pub const Options = struct { style: Style = .blank, @@ -88,7 +88,7 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { config_header.* = .{ .step = .init(.{ - .id = base_id, + .tag = base_tag, .name = name, .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Fail.zig b/lib/std/Build/Step/Fail.zig index 9236c2ac7b6176ab9a7fc15f0afee83fa2107c42..50c7ed2789fb2b81434716cd4687c38eb0cb5a3c 100644 --- a/lib/std/Build/Step/Fail.zig +++ b/lib/std/Build/Step/Fail.zig @@ -6,14 +6,14 @@ const Fail = @This(); step: Step, error_msg: []const u8, -pub const base_id: Step.Id = .fail; +pub const base_tag: Step.Tag = .fail; pub fn create(owner: *std.Build, error_msg: []const u8) *Fail { const fail = owner.allocator.create(Fail) catch @panic("OOM"); fail.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "fail", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Fmt.zig b/lib/std/Build/Step/Fmt.zig index 2da07b63bb422176c8674ff6ccf71a0e00749c4e..bca5385a541ca992e4429e2e64b945ddcf8dfc4e 100644 --- a/lib/std/Build/Step/Fmt.zig +++ b/lib/std/Build/Step/Fmt.zig @@ -10,7 +10,7 @@ paths: []const []const u8, exclude_paths: []const []const u8, check: bool, -pub const base_id: Step.Id = .fmt; +pub const base_tag: Step.Tag = .fmt; pub const Options = struct { paths: []const []const u8 = &.{}, @@ -24,7 +24,7 @@ pub fn create(owner: *std.Build, options: Options) *Fmt { const name = if (options.check) "zig fmt --check" else "zig fmt"; fmt.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = name, .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig index aafd18f01c6354d7d87632c529c49a908d4152c0..f4e9b1d8185f665a08e90983514593b43357bdb8 100644 --- a/lib/std/Build/Step/InstallArtifact.zig +++ b/lib/std/Build/Step/InstallArtifact.zig @@ -33,7 +33,7 @@ const DylibSymlinkInfo = struct { name_only_filename: []const u8, }; -pub const base_id: Step.Id = .install_artifact; +pub const base_tag: Step.Tag = .install_artifact; pub const Options = struct { /// Which installation directory to put the main output file into. @@ -69,10 +69,9 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins }; install_artifact.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = owner.fmt("install {s}", .{artifact.name}), .owner = owner, - .makeFn = make, }), .dest_dir = dest_dir, .pdb_dir = switch (options.pdb_dir) { @@ -126,99 +125,3 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins return install_artifact; } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); - const b = step.owner; - const io = b.graph.io; - - var all_cached = true; - - if (install_artifact.dest_dir) |dest_dir| { - const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); - const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); - all_cached = all_cached and p == .fresh; - - if (install_artifact.dylib_symlinks) |dls| { - try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); - } - - install_artifact.artifact.installed_path = full_dest_path; - } - - if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { - const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.implib_dir) |implib_dir| { - const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.pdb_dir) |pdb_dir| { - const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.h_dir) |h_dir| { - if (install_artifact.emitted_h) |emitted_h| { - const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); - const p = try step.installFile(emitted_h, full_h_path); - all_cached = all_cached and p == .fresh; - } - - for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { - .file => |file| { - const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); - const p = try step.installFile(file.source, full_h_path); - all_cached = all_cached and p == .fresh; - }, - .directory => |dir| { - const src_dir_path = dir.source.getPath3(b, step); - const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); - - var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {s}", .{ - src_dir_path, @errorName(err), - }); - }; - defer src_dir.close(io); - - var it = try src_dir.walk(b.allocator); - next_entry: while (try it.next(io)) |entry| { - for (dir.options.exclude_extensions) |ext| { - if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; - } - if (dir.options.include_extensions) |incs| { - for (incs) |inc| { - if (std.mem.endsWith(u8, entry.path, inc)) break; - } else { - continue :next_entry; - } - } - - const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); - switch (entry.kind) { - .directory => { - try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); - const p = try step.installDir(full_dest_path); - all_cached = all_cached and p == .existed; - }, - .file => { - const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); - all_cached = all_cached and p == .fresh; - }, - else => continue, - } - } - }, - }; - } - - step.result_cached = all_cached; -} diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index d03e72ca75f45a314fc911affc150b1488623f85..f755a28662b24f60a08157bef23a2a2b3b40c5e1 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -8,7 +8,7 @@ const InstallDir = @This(); step: Step, options: Options, -pub const base_id: Step.Id = .install_dir; +pub const base_tag: Step.Tag = .install_dir; pub const Options = struct { source_dir: LazyPath, @@ -44,7 +44,7 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir { const install_dir = owner.allocator.create(InstallDir) catch @panic("OOM"); install_dir.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = owner.fmt("install {s}/", .{options.source_dir.getDisplayName()}), .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index 10adb4754db39cf6b8e93267257c07bdfc760815..a73f126d16087893b9787bff6c372ba7f15cdedc 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -5,7 +5,7 @@ const InstallDir = std.Build.InstallDir; const InstallFile = @This(); const assert = std.debug.assert; -pub const base_id: Step.Id = .install_file; +pub const base_tag: Step.Tag = .install_file; step: Step, source: LazyPath, @@ -22,7 +22,7 @@ pub fn create( const install_file = owner.allocator.create(InstallFile) catch @panic("OOM"); install_file.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = owner.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index ea0714adf9b5517dfd206c9001749636d4e9a438..5ab21c3bcc835254226d27e66c5049be1b972ad3 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -10,7 +10,7 @@ const elf = std.elf; const fs = std.fs; const sort = std.sort; -pub const base_id: Step.Id = .objcopy; +pub const base_tag: Step.Tag = .objcopy; pub const RawFormat = enum { bin, @@ -111,7 +111,7 @@ pub fn create( const objcopy = owner.allocator.create(ObjCopy) catch @panic("OOM"); objcopy.* = ObjCopy{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = owner.fmt("objcopy {s}", .{input_file.getDisplayName()}), .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 34073264e888553f1ff59b74959ede7a675b0241..21df380b18ed05dd0ad181116370b4c38b533c84 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -8,7 +8,7 @@ const Step = std.Build.Step; const GeneratedFile = std.Build.GeneratedFile; const LazyPath = std.Build.LazyPath; -pub const base_id: Step.Id = .options; +pub const base_tag: Step.Tag = .options; step: Step, generated_file: GeneratedFile, @@ -21,7 +21,7 @@ pub fn create(owner: *std.Build) *Options { const options = owner.allocator.create(Options) catch @panic("OOM"); options.* = .{ .step = .init(.{ - .id = base_id, + .tag = base_tag, .name = "options", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 221a8b686027300fb0b6f645ca033787f479aa03..b7dd256d8ed34189e9f59fbcf1b41b1bdcb4eb15 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -12,7 +12,7 @@ const EnvMap = std.process.Environ.Map; const assert = std.debug.assert; const Path = std.Build.Cache.Path; -pub const base_id: Step.Id = .run; +pub const base_tag: Step.Tag = .run; step: Step, @@ -88,12 +88,6 @@ dep_output_file: ?*Output, has_side_effects: bool, -/// If this is a Zig unit test binary, this tracks the names of the unit -/// tests that are also fuzz tests. Indexes cannot be used as they may -/// change between reruns. -fuzz_tests: std.ArrayList([]const u8), -cached_test_metadata: ?CachedTestMetadata = null, - /// Populated during the fuzz phase if this run step corresponds to a unit test /// executable that contains fuzz tests. rebuilt_executable: ?Path, @@ -209,10 +203,9 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { const run = owner.allocator.create(Run) catch @panic("OOM"); run.* = .{ .step = .init(.{ - .id = base_id, + .tag = base_tag, .name = name, .owner = owner, - .makeFn = make, }), .argv = .empty, .cwd = null, @@ -229,7 +222,6 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .captured_stderr = null, .dep_output_file = null, .has_side_effects = false, - .fuzz_tests = .empty, .rebuilt_executable = null, .producer = null, }; @@ -702,2107 +694,3 @@ pub fn addFileInput(self: *Run, file_input: std.Build.LazyPath) void { file_input.addStepDependencies(&self.step); self.file_inputs.append(self.step.owner.allocator, file_input.dupe(self.step.owner)) catch @panic("OOM"); } - -/// Returns whether the Run step has side effects *other than* updating the output arguments. -fn hasSideEffects(run: Run) bool { - if (run.has_side_effects) return true; - return switch (run.stdio) { - .infer_from_args => !run.hasAnyOutputArgs(), - .inherit => true, - .check => false, - .zig_test => false, - }; -} - -fn hasAnyOutputArgs(run: Run) bool { - if (run.captured_stdout != null) return true; - if (run.captured_stderr != null) return true; - for (run.argv.items) |arg| switch (arg) { - .output_file, .output_directory => return true, - else => continue, - }; - return false; -} - -fn checksContainStdout(checks: []const StdIo.Check) bool { - for (checks) |check| switch (check) { - .expect_stderr_exact, - .expect_stderr_match, - .expect_term, - => continue, - - .expect_stdout_exact, - .expect_stdout_match, - => return true, - }; - return false; -} - -fn checksContainStderr(checks: []const StdIo.Check) bool { - for (checks) |check| switch (check) { - .expect_stdout_exact, - .expect_stdout_match, - .expect_term, - => continue, - - .expect_stderr_exact, - .expect_stderr_match, - => return true, - }; - return false; -} - -/// If `path` is cwd-relative, make it relative to the cwd of the child instead. -/// -/// Whenever a path is included in the argv of a child, it should be put through this function first -/// to make sure the child doesn't see paths relative to a cwd other than its own. -fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { - const b = run.step.owner; - const graph = b.graph; - const arena = graph.arena; - - const path_str = path.toString(arena) catch @panic("OOM"); - if (Dir.path.isAbsolute(path_str)) { - // Absolute paths don't need changing. - return path_str; - } - const child_cwd_rel: []const u8 = rel: { - const child_lazy_cwd = run.cwd orelse break :rel path_str; - const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); - // Convert it from relative to *our* cwd, to relative to the *child's* cwd. - break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); - }; - // Not every path can be made relative, e.g. if the path and the child cwd are on different - // disk designators on Windows. In that case, `relative` will return an absolute path which we can - // just return. - if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel; - - // We're not done yet. In some cases this path must be prefixed with './': - // * On POSIX, the executable name cannot be a single component like 'foo' - // * Some executables might treat a leading '-' like a flag, which we must avoid - // There's no harm in it, so just *always* apply this prefix. - return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); -} - -const IndexedOutput = struct { - index: usize, - tag: @typeInfo(Arg).@"union".tag_type.?, - output: *Output, -}; -fn make(step: *Step, options: Step.MakeOptions) !void { - const b = step.owner; - const io = b.graph.io; - const arena = b.allocator; - const run: *Run = @fieldParentPtr("step", step); - const has_side_effects = run.hasSideEffects(); - - var argv_list = std.array_list.Managed([]const u8).init(arena); - var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); - - var man = b.graph.cache.obtain(); - defer man.deinit(); - - if (run.environ_map) |environ_map| { - for (environ_map.keys(), environ_map.values()) |key, value| { - man.hash.addBytes(key); - man.hash.addBytes(value); - } - } - - man.hash.add(run.color); - man.hash.add(run.disable_zig_progress); - - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(bytes); - man.hash.addBytes(bytes); - }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); - man.hash.addBytes(file.prefix); - _ = try man.addFilePath(file_path, null); - }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); - try argv_list.append(resolved_arg); - man.hash.addBytes(resolved_arg); - }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); - - var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; - - const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { - return step.fail( - "unable to open input file '{f}': {t}", - .{ file_path, err }, - ); - }; - defer file.close(io); - - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); - _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return step.fail( - "failed to read from '{f}': {t}", - .{ file_path, file_reader.err.? }, - ), - error.WriteFailed => return error.OutOfMemory, - }; - - try argv_list.append(result.written()); - man.hash.addBytes(file_plp.prefix); - _ = try man.addFilePath(file_path, null); - }, - .artifact => |pa| { - const artifact = pa.artifact; - - if (artifact.rootModuleTarget().os.tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - run.addPathForDynLibs(artifact); - } - const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; - - try argv_list.append(b.fmt("{s}{s}", .{ - pa.prefix, - run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), - })); - - _ = try man.addFile(file_path, null); - }, - .output_file, .output_directory => |output| { - man.hash.addBytes(output.prefix); - man.hash.addBytes(output.basename); - // Add a placeholder into the argument list because we need the - // manifest hash to be updated with all arguments before the - // object directory is computed. - try output_placeholders.append(.{ - .index = argv_list.items.len, - .tag = arg, - .output = output, - }); - _ = try argv_list.addOne(); - }, - } - } - - switch (run.stdin) { - .bytes => |bytes| { - man.hash.addBytes(bytes); - }, - .lazy_path => |lazy_path| { - const file_path = lazy_path.getPath2(b, step); - _ = try man.addFile(file_path, null); - }, - .none => {}, - } - - if (run.captured_stdout) |captured| { - man.hash.addBytes(captured.output.basename); - man.hash.add(captured.trim_whitespace); - } - - if (run.captured_stderr) |captured| { - man.hash.addBytes(captured.output.basename); - man.hash.add(captured.trim_whitespace); - } - - hashStdIo(&man.hash, run.stdio); - - for (run.file_inputs.items) |lazy_path| { - _ = try man.addFile(lazy_path.getPath2(b, step), null); - } - - if (run.cwd) |cwd| { - const cwd_path = cwd.getPath3(b, step); - _ = man.hash.addBytes(try cwd_path.toString(arena)); - } - - if (!has_side_effects and try step.cacheHitAndWatch(&man)) { - // cache hit, skip running command - const digest = man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); - - step.result_cached = true; - return; - } - - const dep_output_file = run.dep_output_file orelse { - // We already know the final output paths, use them directly. - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); - - const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; - for (output_placeholders.items) |placeholder| { - const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, output_sub_dir_path, @errorName(err), - }); - }; - const arg_output_path = run.convertPathArg(.{ - .root_dir = .cwd(), - .sub_path = placeholder.output.generated_file.getPath(), - }); - argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) - arg_output_path - else - b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); - } - - try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); - if (!has_side_effects) try step.writeManifestAndWatch(&man); - return; - }; - - // We do not know the final output paths yet, use temp paths to run the command. - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - - for (output_placeholders.items) |placeholder| { - const output_components = .{ tmp_dir_path, placeholder.output.basename }; - const output_sub_path = b.pathJoin(&output_components); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, output_sub_dir_path, @errorName(err), - }); - }; - const raw_output_path: Build.Cache.Path = .{ - .root_dir = b.cache_root, - .sub_path = b.pathJoin(&output_components), - }; - placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); - argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ - placeholder.output.prefix, - run.convertPathArg(raw_output_path), - }); - } - - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); - - const dep_file_dir = Dir.cwd(); - const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); - if (has_side_effects) - try man.addDepFile(dep_file_dir, dep_file_basename) - else - try man.addDepFilePost(dep_file_dir, dep_file_basename); - - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - const any_output = output_placeholders.items.len > 0 or - run.captured_stdout != null or run.captured_stderr != null; - - // Rename into place - if (any_output) { - const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; - - b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { - Dir.RenameError.DirNotEmpty => { - b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { - return step.fail("unable to remove dir '{f}'{s}: {t}", .{ - b.cache_root, tmp_dir_path, del_err, - }); - }; - b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { - return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, - }); - }; - }, - else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, - }), - }; - } - - if (!has_side_effects) try step.writeManifestAndWatch(&man); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); -} - -pub fn rerunInFuzzMode( - run: *Run, - fuzz: *std.Build.Fuzz, - prog_node: std.Progress.Node, -) !void { - const step = &run.step; - const b = step.owner; - const io = b.graph.io; - const arena = b.allocator; - var argv_list: std.ArrayList([]const u8) = .empty; - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(arena, bytes); - }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); - }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); - }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); - - var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; - - const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); - defer file.close(io); - - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); - _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - error.WriteFailed => return error.OutOfMemory, - }; - - try argv_list.append(arena, result.written()); - }, - .artifact => |pa| { - const artifact = pa.artifact; - const file_path: []const u8 = p: { - if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); - break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; - }; - try argv_list.append(arena, b.fmt("{s}{s}", .{ - pa.prefix, - run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), - })); - }, - .output_file, .output_directory => unreachable, - } - } - - if (run.step.result_failed_command) |cmd| { - fuzz.gpa.free(cmd); - run.step.result_failed_command = null; - } - - const has_side_effects = false; - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ - .progress_node = prog_node, - .watch = undefined, // not used by `runCommand` - .web_server = null, // only needed for time reports - .unit_test_timeout_ns = null, // don't time out fuzz tests for now - .gpa = fuzz.gpa, - }, .{ - .fuzz = fuzz, - }); -} - -fn populateGeneratedPaths( - arena: std.mem.Allocator, - output_placeholders: []const IndexedOutput, - captured_stdout: ?*CapturedStdIo, - captured_stderr: ?*CapturedStdIo, - cache_root: Build.Cache.Directory, - digest: *const Build.Cache.HexDigest, -) !void { - for (output_placeholders) |placeholder| { - placeholder.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, placeholder.output.basename, - }); - } - - if (captured_stdout) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); - } - - if (captured_stderr) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); - } -} - -fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void { - if (term) |t| switch (t) { - .exited => |code| try w.print("exited with code {d}", .{code}), - .signal => |sig| try w.print("terminated with signal {t}", .{sig}), - .stopped => |sig| try w.print("stopped with signal {t}", .{sig}), - .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}), - } else { - try w.writeAll("exited with any code"); - } -} -fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) { - return .{ .data = term }; -} - -fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { - return if (expected) |e| switch (e) { - .exited => |expected_code| switch (actual) { - .exited => |actual_code| expected_code == actual_code, - else => false, - }, - .signal => |expected_sig| switch (actual) { - .signal => |actual_sig| expected_sig == actual_sig, - else => false, - }, - .stopped => |expected_sig| switch (actual) { - .stopped => |actual_sig| expected_sig == actual_sig, - else => false, - }, - .unknown => |expected_code| switch (actual) { - .unknown => |actual_code| expected_code == actual_code, - else => false, - }, - } else switch (actual) { - .exited => true, - else => false, - }; -} - -const FuzzContext = struct { - fuzz: *std.Build.Fuzz, -}; - -fn runCommand( - run: *Run, - argv: []const []const u8, - has_side_effects: bool, - output_dir_path: []const u8, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !void { - const step = &run.step; - const b = step.owner; - const arena = b.allocator; - const gpa = options.gpa; - const io = b.graph.io; - - const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; - - try step.handleChildProcUnsupported(); - try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); - - const allow_skip = switch (run.stdio) { - .check, .zig_test => run.skip_foreign_checks, - else => false, - }; - - var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); - defer interp_argv.deinit(); - - var environ_map: EnvMap = env: { - const orig = run.environ_map orelse &b.graph.environ_map; - break :env try orig.clone(gpa); - }; - defer environ_map.deinit(); - - const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { - // InvalidExe: cpu arch mismatch - // FileNotFound: can happen with a wrong dynamic linker path - if (err == error.InvalidExe or err == error.FileNotFound) interpret: { - // TODO: learn the target from the binary directly rather than from - // relying on it being a Compile step. This will make this logic - // work even for the edge case that the binary was produced by a - // third party. - const exe = switch (run.argv.items[0]) { - .artifact => |exe| exe.artifact, - else => break :interpret, - }; - switch (exe.kind) { - .exe, .@"test" => {}, - else => break :interpret, - } - - const root_target = exe.rootModuleTarget(); - const need_cross_libc = exe.is_linking_libc and - (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); - const other_target = exe.root_module.resolved_target.?.result; - switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ - .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, - .link_libc = exe.is_linking_libc, - })) { - .native, .rosetta => { - if (allow_skip) return error.MakeSkipped; - break :interpret; - }, - .wine => |bin_name| { - if (b.enable_wine) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); - - // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but - // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. - if (environ_map.get("WINEDEBUG") == null) { - try environ_map.put("WINEDEBUG", "-all"); - } - } else { - return failForeign(run, "-fwine", argv[0], exe); - } - }, - .qemu => |bin_name| { - if (b.enable_qemu) { - try interp_argv.append(bin_name); - - if (need_cross_libc) { - if (b.libc_runtimes_dir) |dir| { - try interp_argv.append("-L"); - try interp_argv.append(b.pathJoin(&.{ - dir, - try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( - b.allocator, - root_target.cpu.arch, - root_target.os.tag, - root_target.abi, - ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( - b.allocator, - root_target.cpu.arch, - root_target.abi, - ) else unreachable, - })); - } else return failForeign(run, "--libc-runtimes", argv[0], exe); - } - - try interp_argv.appendSlice(argv); - } else return failForeign(run, "-fqemu", argv[0], exe); - }, - .darling => |bin_name| { - if (b.enable_darling) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); - } else { - return failForeign(run, "-fdarling", argv[0], exe); - } - }, - .wasmtime => |bin_name| { - if (b.enable_wasmtime) { - try interp_argv.append(bin_name); - try interp_argv.append("--dir=."); - // Wasmtime doeesn't inherit environment variables from the parent process - // by default. '-S inherit-env' was added in Wasmtime version 20. - try interp_argv.append("-Sinherit-env"); - try interp_argv.append(argv[0]); - try interp_argv.appendSlice(argv[1..]); - } else { - return failForeign(run, "-fwasmtime", argv[0], exe); - } - }, - .bad_dl => |foreign_dl| { - if (allow_skip) return error.MakeSkipped; - - const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; - - return step.fail( - \\the host system is unable to execute binaries from the target - \\ because the host dynamic linker is '{s}', - \\ while the target dynamic linker is '{s}'. - \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step - , .{ host_dl, foreign_dl }); - }, - .bad_os_or_cpu => { - if (allow_skip) return error.MakeSkipped; - - const host_name = try b.graph.host.result.zigTriple(b.allocator); - const foreign_name = try root_target.zigTriple(b.allocator); - - return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ - host_name, foreign_name, - }); - }, - } - - if (root_target.os.tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - run.addPathForDynLibs(exe); - } - - gpa.free(step.result_failed_command.?); - step.result_failed_command = null; - try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); - - break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { - if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; - if (e == error.MakeFailed) return error.MakeFailed; // error already reported - return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); - }; - } - if (err == error.MakeFailed) return error.MakeFailed; // error already reported - - return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); - }; - - const generic_result = opt_generic_result orelse { - assert(run.stdio == .zig_test); - // Specific errors have already been reported, and test results are populated. All we need - // to do is report step failure if any test failed. - if (!step.test_results.isSuccess()) return error.MakeFailed; - return; - }; - - assert(fuzz_context == null); - assert(run.stdio != .zig_test); - - // Capture stdout and stderr to GeneratedFile objects. - const Stream = struct { - captured: ?*CapturedStdIo, - bytes: ?[]const u8, - }; - for ([_]Stream{ - .{ - .captured = run.captured_stdout, - .bytes = generic_result.stdout, - }, - .{ - .captured = run.captured_stderr, - .bytes = generic_result.stderr, - }, - }) |stream| { - if (stream.captured) |captured| { - const output_components = .{ output_dir_path, captured.output.basename }; - const output_path = try b.cache_root.join(arena, &output_components); - captured.output.generated_file.path = output_path; - - const sub_path = b.pathJoin(&output_components); - const sub_path_dirname = Dir.path.dirname(sub_path).?; - b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, sub_path_dirname, @errorName(err), - }); - }; - const data = switch (captured.trim_whitespace) { - .none => stream.bytes.?, - .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), - .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), - .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), - }; - b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { - return step.fail("unable to write file '{f}{s}': {s}", .{ - b.cache_root, sub_path, @errorName(err), - }); - }; - } - } - - switch (run.stdio) { - .zig_test => unreachable, - .check => |checks| for (checks.items) |check| switch (check) { - .expect_stderr_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { - return step.fail( - \\========= expected this stderr: ========= - \\{s} - \\========= but found: ==================== - \\{s} - , .{ - expected_bytes, - generic_result.stderr.?, - }); - } - }, - .expect_stderr_match => |match| { - if (mem.find(u8, generic_result.stderr.?, match) == null) { - return step.fail( - \\========= expected to find in stderr: ========= - \\{s} - \\========= but stderr does not contain it: ===== - \\{s} - , .{ - match, - generic_result.stderr.?, - }); - } - }, - .expect_stdout_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { - return step.fail( - \\========= expected this stdout: ========= - \\{s} - \\========= but found: ==================== - \\{s} - , .{ - expected_bytes, - generic_result.stdout.?, - }); - } - }, - .expect_stdout_match => |match| { - if (mem.find(u8, generic_result.stdout.?, match) == null) { - return step.fail( - \\========= expected to find in stdout: ========= - \\{s} - \\========= but stdout does not contain it: ===== - \\{s} - , .{ - match, - generic_result.stdout.?, - }); - } - }, - .expect_term => |expected_term| { - if (!termMatches(expected_term, generic_result.term)) { - return step.fail("process {f} (expected {f})", .{ - fmtTerm(generic_result.term), - fmtTerm(expected_term), - }); - } - }, - }, - else => { - // On failure, report captured stderr like normal standard error output. - const bad_exit = switch (generic_result.term) { - .exited => |code| code != 0, - .signal, .stopped, .unknown => true, - }; - if (bad_exit) { - if (generic_result.stderr) |bytes| { - run.step.result_stderr = bytes; - } - } - - try step.handleChildProcessTerm(generic_result.term); - }, - } -} - -const EvalGenericResult = struct { - term: process.Child.Term, - stdout: ?[]const u8, - stderr: ?[]const u8, -}; - -fn spawnChildAndCollect( - run: *Run, - argv: []const []const u8, - environ_map: *EnvMap, - has_side_effects: bool, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !?EvalGenericResult { - const b = run.step.owner; - const graph = b.graph; - const io = graph.io; - - if (fuzz_context != null) { - assert(!has_side_effects); - assert(run.stdio == .zig_test); - } - - const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; - - // If an error occurs, it's caused by this command: - assert(run.step.result_failed_command == null); - run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ - .child = environ_map, - .parent = &graph.environ_map, - }, argv); - - var spawn_options: process.SpawnOptions = .{ - .argv = argv, - .cwd = child_cwd, - .environ_map = environ_map, - .request_resource_usage_statistics = true, - .stdin = if (run.stdin != .none) s: { - assert(run.stdio != .inherit); - break :s .pipe; - } else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .ignore, - .inherit => .inherit, - .check => .ignore, - .zig_test => .pipe, - }, - .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .ignore, - .inherit => .inherit, - .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, - .zig_test => .pipe, - }, - .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .pipe, - .inherit => .inherit, - .check => .pipe, - .zig_test => .pipe, - }, - }; - - if (run.stdio == .zig_test) { - const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| e, - }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); - try result; - return null; - } else { - const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; - if (!run.disable_zig_progress and !inherit) { - spawn_options.progress_node = options.progress_node; - } - const terminal_mode: Io.Terminal.Mode = if (inherit) m: { - const stderr = try io.lockStderr(&.{}, graph.stderr_mode); - break :m stderr.terminal_mode; - } else .no_color; - defer if (inherit) io.unlockStderr(); - try setColorEnvironmentVariables(run, environ_map, terminal_mode); - - const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalGeneric(run, spawn_options) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| e, - }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); - return try result; - } -} - -fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { - color: switch (run.color) { - .manual => {}, - .enable => { - try environ_map.put("CLICOLOR_FORCE", "1"); - _ = environ_map.swapRemove("NO_COLOR"); - }, - .disable => { - try environ_map.put("NO_COLOR", "1"); - _ = environ_map.swapRemove("CLICOLOR_FORCE"); - }, - .inherit => switch (terminal_mode) { - .no_color, .windows_api => continue :color .disable, - .escape_codes => continue :color .enable, - }, - .auto => { - const capture_stderr = run.captured_stderr != null or switch (run.stdio) { - .check => |checks| checksContainStderr(checks.items), - .infer_from_args, .inherit, .zig_test => false, - }; - if (capture_stderr) { - continue :color .disable; - } else { - continue :color .inherit; - } - }, - } -} - -const StdioPollEnum = enum { stdout, stderr }; - -fn evalZigTest( - run: *Run, - spawn_options: process.SpawnOptions, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !void { - if (fuzz_context != null) { - try evalFuzzTest(run, spawn_options, options, fuzz_context.?); - return; - } - - const step_owner = run.step.owner; - const gpa = step_owner.allocator; - const arena = step_owner.allocator; - const io = step_owner.graph.io; - - // We will update this every time a child runs. - run.step.result_peak_rss = 0; - - var test_results: Step.TestResults = .{ - .test_count = 0, - .skip_count = 0, - .fail_count = 0, - .crash_count = 0, - .timeout_count = 0, - .leak_count = 0, - .log_err_count = 0, - }; - var test_metadata: ?TestMetadata = null; - - while (true) { - var child = try process.spawn(io, spawn_options); - var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; - var multi_reader: Io.File.MultiReader = undefined; - multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); - var child_killed = false; - defer if (!child_killed) { - child.kill(io); - multi_reader.deinit(); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - }; - - switch (try waitZigTest( - run, - &child, - options, - &multi_reader, - &test_metadata, - &test_results, - )) { - .write_failed => |err| { - // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured - // all available stderr to make our error output as useful as possible. - const stderr_fr = multi_reader.fileReader(1); - while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) { - error.ReadFailed => return stderr_fr.err.?, - error.EndOfStream => {}, - } - run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); - - // Clean up everything and wait for the child to exit. - child.stdin.?.close(io); - child.stdin = null; - multi_reader.deinit(); - child_killed = true; - const term = try child.wait(io); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - - // The individual unit test results are irrelevant: the test runner itself broke! - // Fail immediately without populating `s.test_results`. - return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); - }, - .no_poll => |no_poll| { - // This might be a success (we requested exit and the child dutifully closed stdout) or - // a crash of some kind. Either way, the child will terminate by itself -- wait for it. - const stderr_reader = multi_reader.reader(1); - const stderr_owned = try arena.dupe(u8, stderr_reader.buffered()); - - // Clean up everything and wait for the child to exit. - child.stdin.?.close(io); - child.stdin = null; - multi_reader.deinit(); - child_killed = true; - const term = try child.wait(io); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - - if (no_poll.active_test_index) |test_index| { - // A test was running, so this is definitely a crash. Report it against that - // test, and continue to the next test. - test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; - test_results.crash_count += 1; - try run.step.addError("'{s}' {f}{s}{s}", .{ - test_metadata.?.testName(test_index), - fmtTerm(term), - if (stderr_owned.len != 0) " with stderr:\n" else "", - std.mem.trim(u8, stderr_owned, "\n"), - }); - continue; - } - - // Report an error if the child terminated uncleanly or if we were still trying to run more tests. - run.step.result_stderr = stderr_owned; - const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); - if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { - // The individual unit test results are irrelevant: the test runner itself broke! - // Fail immediately without populating `s.test_results`. - return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); - } - - // We're done with all of the tests! Commit the test results and return. - run.step.test_results = test_results; - if (test_metadata) |tm| { - run.cached_test_metadata = tm.toCachedTestMetadata(); - if (options.web_server) |ws| { - if (run.step.owner.graph.time_report) { - ws.updateTimeReportRunTest( - run, - &run.cached_test_metadata.?, - tm.ns_per_test, - ); - } - } - } - return; - }, - .timeout => |timeout| { - const stderr_reader = multi_reader.reader(1); - const stderr = stderr_reader.buffered(); - stderr_reader.tossBuffered(); - if (timeout.active_test_index) |test_index| { - // A test was running. Report the timeout against that test, and continue on to - // the next test. - test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; - test_results.timeout_count += 1; - try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ - test_metadata.?.testName(test_index), - Io.Duration{ .nanoseconds = timeout.ns_elapsed }, - if (stderr.len != 0) " with stderr:\n" else "", - std.mem.trim(u8, stderr, "\n"), - }); - continue; - } - // Just log an error and let the child be killed. - run.step.result_stderr = try arena.dupe(u8, stderr); - // The individual unit test results in `results` are irrelevant: the test runner - // is broken! Fail immediately without populating `s.test_results`. - return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); - }, - } - comptime unreachable; - } -} - -/// Reads stdout of a Zig test process until a termination condition is reached: -/// * A write fails, indicating the child unexpectedly closed stdin -/// * A test (or a response from the test runner) times out -/// * The wait fails, indicating the child closed stdout and stderr -fn waitZigTest( - run: *Run, - child: *process.Child, - options: Step.MakeOptions, - multi_reader: *Io.File.MultiReader, - opt_metadata: *?TestMetadata, - results: *Step.TestResults, -) !union(enum) { - write_failed: anyerror, - no_poll: struct { - active_test_index: ?u32, - ns_elapsed: u64, - }, - timeout: struct { - active_test_index: ?u32, - ns_elapsed: u64, - }, -} { - const gpa = run.step.owner.allocator; - const arena = run.step.owner.allocator; - const io = run.step.owner.graph.io; - - var sub_prog_node: ?std.Progress.Node = null; - defer if (sub_prog_node) |n| n.end(); - - if (opt_metadata.*) |*md| { - // Previous unit test process died or was killed; we're continuing where it left off - requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; - } else { - // Running unit tests normally - run.fuzz_tests.clearRetainingCapacity(); - sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err }; - } - - var active_test_index: ?u32 = null; - - var last_update: Io.Clock.Timestamp = .now(io, .awake); - - // This timeout is used when we're waiting on the test runner itself rather than a user-specified - // test. For instance, if the test runner leaves this much time between us requesting a test to - // start and it acknowledging the test starting, we terminate the child and raise an error. This - // *should* never happen, but could in theory be caused by some very unlucky IB in a test. - const response_timeout: Io.Clock.Duration = t: { - const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); - break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; - }; - const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ - .clock = .awake, - .raw = .fromNanoseconds(ns), - } else null; - - const stdout = multi_reader.reader(0); - const stderr = multi_reader.reader(1); - const Header = std.zig.Server.Message.Header; - - while (true) { - const timeout: Io.Timeout = t: { - const opt_duration = if (active_test_index == null) response_timeout else test_timeout; - const duration = opt_duration orelse break :t .none; - break :t .{ .deadline = last_update.addDuration(duration) }; - }; - - // This block is exited when `stdout` contains enough bytes for a `Header`. - header_ready: { - if (stdout.buffered().len >= @sizeOf(Header)) { - // We already have one, no need to poll! - break :header_ready; - } - - multi_reader.fill(64, timeout) catch |err| switch (err) { - error.Timeout => return .{ .timeout = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - error.EndOfStream => return .{ .no_poll = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - else => |e| return e, - }; - - continue; - } - // There is definitely a header available now -- read it. - const header = stdout.takeStruct(Header, .little) catch unreachable; - - while (stdout.buffered().len < header.bytes_len) { - multi_reader.fill(64, timeout) catch |err| switch (err) { - error.Timeout => return .{ .timeout = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - error.EndOfStream => return .{ .no_poll = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - else => |e| return e, - }; - } - - const body = stdout.take(header.bytes_len) catch unreachable; - var body_r: std.Io.Reader = .fixed(body); - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - }, - .test_metadata => { - // `metadata` would only be populated if we'd already seen a `test_metadata`, but we - // only request it once (and importantly, we don't re-request it if we kill and - // restart the test runner). - assert(opt_metadata.* == null); - - const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable; - results.test_count = tm_hdr.tests_len; - - const names = try arena.alloc(u32, results.test_count); - for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; - - const expected_panic_msgs = try arena.alloc(u32, results.test_count); - for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; - - const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; - - options.progress_node.setEstimatedTotalItems(names.len); - opt_metadata.* = .{ - .string_bytes = try arena.dupe(u8, string_bytes), - .ns_per_test = try arena.alloc(u64, results.test_count), - .names = names, - .expected_panic_msgs = expected_panic_msgs, - .next_index = 0, - .prog_node = options.progress_node, - }; - @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); - - active_test_index = null; - last_update = .now(io, .awake); - - requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err }; - }, - .test_started => { - active_test_index = opt_metadata.*.?.next_index - 1; - last_update = .now(io, .awake); - }, - .test_results => { - const md = &opt_metadata.*.?; - - const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable; - assert(tr_hdr.index == active_test_index); - - switch (tr_hdr.flags.status) { - .pass => {}, - .skip => results.skip_count +|= 1, - .fail => results.fail_count +|= 1, - } - const leak_count = tr_hdr.flags.leak_count; - const log_err_count = tr_hdr.flags.log_err_count; - results.leak_count +|= leak_count; - results.log_err_count +|= log_err_count; - - if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index)); - - if (tr_hdr.flags.status == .fail) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - if (stderr_bytes.len == 0) { - try run.step.addError("'{s}' failed without output", .{name}); - } else { - try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); - } - } else if (leak_count > 0) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); - } else if (log_err_count > 0) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); - } - - active_test_index = null; - - const now: Io.Clock.Timestamp = .now(io, .awake); - md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds); - last_update = now; - - requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; - }, - else => {}, // ignore other messages - } - } -} - -const FuzzTestRunner = struct { - run: *Run, - ctx: FuzzContext, - coverage_id: ?u64, - - instances: []Instance, - /// The indexes of this are layed out such that it is effectively an array - /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr. - batch: Io.Batch, - /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter. - pending_broadcasts: std.ArrayList(u8), - broadcast: std.ArrayList(u8), - broadcast_undelivered: u32, - - const Instance = struct { - child: process.Child, - message: std.ArrayListAligned(u8, .@"4"), - broadcast_written: usize, - stderr: std.ArrayList(u8), - stdin_vec: [1][]u8, - stdout_vec: [1][]u8, - stderr_vec: [1][]u8, - progress_node: std.Progress.Node, - - fn messageHeader(instance: *Instance) InHeader { - assert(instance.message.items.len >= @sizeOf(InHeader)); - const header_ptr: *InHeader = @ptrCast(instance.message.items); - var header = header_ptr.*; - if (std.builtin.Endian.native != .little) { - std.mem.byteSwapAllFields(InHeader, &header); - } - return header; - } - }; - - const PendingBroadcastFooter = struct { - from_id: u32, - body_len: u32, - }; - - const InHeader = std.zig.Server.Message.Header; - const OutHeader = std.zig.Client.Message.Header; - - const stdin_i = 0; - const stdout_i = 1; - const stderr_i = 2; - - fn init( - run: *Run, - ctx: FuzzContext, - progress_node: std.Progress.Node, - spawn_options: process.SpawnOptions, - ) !FuzzTestRunner { - const step_owner = run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - - const n_instances = switch (ctx.fuzz.mode) { - .forever => step_owner.graph.max_jobs orelse @min( - std.Thread.getCpuCount() catch 1, - (std.math.maxInt(u32) - 2) / 3, - ), - .limit => 1, - }; - const instances = try gpa.alloc(Instance, n_instances); - errdefer gpa.free(instances); - const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3); - errdefer gpa.free(batch_storage); - - @memset(instances, .{ - .child = undefined, - .message = .empty, - .broadcast_written = undefined, - .stderr = .empty, - .stdin_vec = undefined, - .stdout_vec = undefined, - .stderr_vec = undefined, - .progress_node = undefined, - }); - for (0.., instances) |id, *instance| { - errdefer for (instances[0..id]) |*spawned| { - spawned.child.kill(io); - spawned.progress_node.end(); - }; - instance.child = try process.spawn(io, spawn_options); - instance.progress_node = progress_node.start("starting fuzzer", 0); - } - - return .{ - .run = run, - .ctx = ctx, - .coverage_id = null, - - .instances = instances, - .batch = .init(batch_storage), - .pending_broadcasts = .empty, - .broadcast = .empty, - .broadcast_undelivered = 0, - }; - } - - fn deinit(f: *FuzzTestRunner) void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - - f.batch.cancel(io); - gpa.free(f.batch.storage); - var total_rss: usize = 0; - for (f.instances) |*instance| { - instance.child.kill(io); - instance.message.deinit(gpa); - instance.stderr.deinit(gpa); - instance.progress_node.end(); - total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; - } - f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); - gpa.free(f.instances); - } - - fn startInstances(f: *FuzzTestRunner) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - - for (0.., f.instances) |id, *instance| { - const id32: u32 = @intCast(id); - (switch (f.ctx.fuzz.mode) { - .forever => sendRunFuzzTestMessage( - io, - instance.child.stdin.?, - f.run.fuzz_tests.items, - .forever, - id32, - ), - .limit => |limit| sendRunFuzzTestMessage( - io, - instance.child.stdin.?, - f.run.fuzz_tests.items, - .iterations, - limit.amount, - ), - }) catch |write_err| { - // The runner unexpectedly closed stdin, which means it crashed during initialization. - // Clean up everything and wait for the child to exit. - instance.child.stdin.?.close(io); - instance.child.stdin = null; - const term = try instance.child.wait(io); - return f.run.step.fail( - "unable to write stdin ({t}); test process unexpectedly {f}", - .{ write_err, fmtTerm(term) }, - ); - }; - - try f.addStdoutRead(id32, @sizeOf(InHeader)); - try f.addStderrRead(id32); - } - } - - fn listen(f: *FuzzTestRunner) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - - while (true) { - try f.batch.awaitConcurrent(io, .none); - while (f.batch.next()) |completion| { - const id = completion.index / 3; - const result = completion.result; - switch (completion.index % 3) { - 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) { - // Avoid calling `instanceEos` until EndOfStream is seen with stderr so - // that all stderr is collected. - error.BrokenPipe => continue, - else => |write_e| return write_e, - }), - 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) { - // Avoid calling `instanceEos` until EndOfStream is seen with stderr so - // that all stderr is collected. - error.EndOfStream => continue, - else => |read_e| return read_e, - }), - 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { - error.EndOfStream => return f.instanceEos(id), - else => |read_e| return read_e, - }), - else => unreachable, - } - } - } - } - - fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - const instance = &f.instances[id]; - - instance.message.items.len += n; - const total_read = instance.message.items.len; - if (total_read < @sizeOf(InHeader)) { - try f.addStdoutRead(id, @sizeOf(InHeader)); - return; - } - - const header = instance.messageHeader(); - const body = instance.message.items[@sizeOf(InHeader)..]; - if (body.len != header.bytes_len) { - try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len); - return; - } - - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - }, - .coverage_id => { - var body_r: Io.Reader = .fixed(body); - f.coverage_id = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable; - - const fuzz = f.ctx.fuzz; - fuzz.queue_mutex.lockUncancelable(io); - defer fuzz.queue_mutex.unlock(io); - try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ - .id = f.coverage_id.?, - .cumulative = .{ - .runs = cumulative_runs, - .unique = cumulative_unique, - .coverage = cumulative_coverage, - }, - .run = f.run, - } }); - fuzz.queue_cond.signal(io); - }, - .fuzz_start_addr => { - var body_r: Io.Reader = .fixed(body); - const fuzz = f.ctx.fuzz; - const addr = body_r.takeInt(u64, .little) catch unreachable; - - fuzz.queue_mutex.lockUncancelable(io); - defer fuzz.queue_mutex.unlock(io); - try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ - .addr = addr, - .coverage_id = f.coverage_id.?, - } }); - fuzz.queue_cond.signal(io); - }, - .fuzz_test_change => { - const test_i = std.mem.readInt(u32, body[0..4], .little); - instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); - }, - .broadcast_fuzz_input => { - if (f.instances.len == 1) { - // No other processes to broadcast to. - } else if (f.broadcast_undelivered == 0) { - try f.instanceBroadcast(id, body); - } else { - const footer: PendingBroadcastFooter = .{ - .from_id = id, - .body_len = @intCast(body.len), - }; - // There is another broadcast in progress so add this one to the queue. - const size = @sizeOf(PendingBroadcastFooter) + body.len; - try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); - f.pending_broadcasts.appendSliceAssumeCapacity(body); - f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); - } - }, - else => {}, // ignore other messages - } - - instance.message.clearRetainingCapacity(); - try f.addStdoutRead(id, @sizeOf(InHeader)); - } - - fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void { - const instance = &f.instances[id]; - instance.stderr.items.len += n; - try f.addStderrRead(id); - } - - fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void { - const instance = &f.instances[id]; - - instance.broadcast_written += n; - if (instance.broadcast_written == f.broadcast.items.len) { - f.broadcast_undelivered -= 1; - if (f.broadcast_undelivered == 0) { - try f.broadcastComplete(); - } - } else { - f.addStdinWrite(id); - } - } - - fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const instance = &f.instances[id]; - - try instance.message.ensureTotalCapacity(gpa, end); - const start = instance.message.items.len; - instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]}; - f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{ - .file = instance.child.stdout.?, - .data = &instance.stdout_vec, - } }); - } - - fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const instance = &f.instances[id]; - - try instance.stderr.ensureUnusedCapacity(gpa, 1); - instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()}; - f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{ - .file = instance.child.stderr.?, - .data = &instance.stderr_vec, - } }); - } - - fn addStdinWrite(f: *FuzzTestRunner, id: u32) void { - const instance = &f.instances[id]; - - assert(f.broadcast.items.len != instance.broadcast_written); - instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]}; - f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{ - .file = instance.child.stdin.?, - .data = &instance.stdin_vec, - } }); - } - - fn instanceEos(f: *FuzzTestRunner, id: u32) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - const instance = &f.instances[id]; - - instance.child.stdin.?.close(io); - instance.child.stdin = null; - const term = try instance.child.wait(io); - if (!termMatches(.{ .exited = 0 }, term)) { - f.run.step.result_stderr = try f.mergedStderr(); - try f.saveCrash(id, term); - return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); - } - } - - fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { - const step = &f.run.step; - const b = step.owner; - const io = b.graph.io; - - if (f.coverage_id == null) return; - - // Search for the input file corresponding to the instance - const InputHeader = Build.abi.fuzz.MmapInputHeader; - var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; - var in_r: Io.File.Reader = undefined; - var in_f: Io.File = undefined; - var in_name_buf: [12]u8 = undefined; - var in_name: []const u8 = undefined; - var i: u32 = 0; - const header: InputHeader = while (true) : ({ - if (i == std.math.maxInt(u32)) return; - i += 1; - }) { - const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; - in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; - in_f = b.cache_root.handle.openFile(io, in_name, .{ - .lock = .exclusive, - .lock_nonblocking = true, - }) catch |e| switch (e) { - error.FileNotFound => return, - error.WouldBlock => continue, // Can not be from - // the crashed instance since it is still locked. - else => return step.fail("failed to open file '{f}{s}': {t}", .{ - b.cache_root, in_name, e, - }), - }; - - in_r = in_f.readerStreaming(io, &in_r_buf); - const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { - in_f.close(io); - switch (e) { - error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, - }), - error.EndOfStream => continue, - } - }; - - if (header.pc_digest == f.coverage_id.? and - header.instance_id == id and - header.test_i < f.run.fuzz_tests.items.len) - { - break header; - } - - in_f.close(io); - }; - defer in_f.close(io); - - // Save it to a seperate file - const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; - const out = b.cache_root.handle.createFile(io, crash_name, .{ - .lock = .exclusive, // Multiple run steps could have found a crash at the same time - }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ - b.cache_root, crash_name, e, - }); - defer out.close(io); - - var out_w_buf: [512]u8 = undefined; - var out_w = out.writerStreaming(io, &out_w_buf); - _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { - error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, - }), - error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ - b.cache_root, crash_name, out_w.err.?, - }), - }; - - return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ - f.run.fuzz_tests.items[header.test_i], - fmtTerm(term), - b.cache_root, - crash_name, - }); - } - - fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void { - assert(f.instances.len > 1); - assert(f.broadcast_undelivered == 0); // no other broadcast is progress - assert(f.broadcast.items.len == 0); - assert(from_id < f.instances.len); - - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - - var out_header: OutHeader = .{ - .tag = .new_fuzz_input, - .bytes_len = @intCast(bytes.len), - }; - if (std.builtin.Endian.native != .little) { - std.mem.byteSwapAllFields(OutHeader, &out_header); - } - try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len); - f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header)); - f.broadcast.appendSliceAssumeCapacity(bytes); - - f.broadcast_undelivered = @intCast(f.instances.len - 1); - for (0.., f.instances) |to_id, *instance| { - if (to_id == from_id) continue; - instance.broadcast_written = 0; - f.addStdinWrite(@intCast(to_id)); - } - } - - fn broadcastComplete(f: *FuzzTestRunner) !void { - assert(f.instances.len > 1); - assert(f.broadcast_undelivered == 0); - f.broadcast.clearRetainingCapacity(); - - const pending = &f.pending_broadcasts; - if (pending.items.len != 0) { - // Another broadcast is pending; copy it over to `broadcast` - - const footer_len = @sizeOf(PendingBroadcastFooter); - const footer_bytes = pending.items[pending.items.len - footer_len ..]; - const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes); - pending.items.len -= footer_len; - - const body = pending.items[pending.items.len - footer.body_len ..]; - try f.instanceBroadcast(footer.from_id, body); - pending.items.len -= body.len; - } - } - - fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { - const step_owner = f.run.step.owner; - const arena = step_owner.allocator; - - // Collect any available stderr - while (f.batch.next()) |completion| { - if (completion.index % 3 != 2) continue; - const len = completion.result.file_read_streaming catch continue; - f.instances[completion.index / 3].stderr.items.len += len; - } - - var stderr_len: usize = 0; - for (f.instances) |*instance| stderr_len += instance.stderr.items.len; - const stderr = try arena.alloc(u8, stderr_len); - - stderr_len = 0; - for (f.instances) |*instance| { - @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items); - stderr_len += instance.stderr.items.len; - } - return stderr; - } -}; - -fn evalFuzzTest( - run: *Run, - spawn_options: process.SpawnOptions, - options: Step.MakeOptions, - fuzz_context: FuzzContext, -) !void { - var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); - defer f.deinit(); - try f.startInstances(); - try f.listen(); -} - -const TestMetadata = struct { - names: []const u32, - ns_per_test: []u64, - expected_panic_msgs: []const u32, - string_bytes: []const u8, - next_index: u32, - prog_node: std.Progress.Node, - - fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { - return .{ - .names = tm.names, - .string_bytes = tm.string_bytes, - }; - } - - fn testName(tm: TestMetadata, index: u32) []const u8 { - return tm.toCachedTestMetadata().testName(index); - } -}; - -pub const CachedTestMetadata = struct { - names: []const u32, - string_bytes: []const u8, - - pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { - return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); - } -}; - -fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { - while (metadata.next_index < metadata.names.len) { - const i = metadata.next_index; - metadata.next_index += 1; - - if (metadata.expected_panic_msgs[i] != 0) continue; - - const name = metadata.testName(i); - if (sub_prog_node.*) |n| n.end(); - sub_prog_node.* = metadata.prog_node.start(name, 0); - - try sendRunTestMessage(io, in, .run_test, i); - return; - } else { - metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done - try sendMessage(io, in, .exit); - } -} - -fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 0, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 4, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u32, index, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -fn sendRunFuzzTestMessage( - io: Io, - file: Io.File, - test_names: []const []const u8, - kind: std.Build.abi.fuzz.LimitKind, - amount_or_instance: u64, -) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = .start_fuzzing, - .bytes_len = 1 + 8 + 4 + count: { - var c: u32 = @intCast(test_names.len * 4); - for (test_names) |name| { - c += @intCast(name.len); - } - break :count c; - }, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - for (test_names) |test_name| { - w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeAll(test_name) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - } -} - -fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { - const b = run.step.owner; - const io = b.graph.io; - const arena = b.allocator; - const gpa = b.allocator; - - var child = try process.spawn(io, spawn_options); - defer child.kill(io); - - switch (run.stdin) { - .bytes => |bytes| { - child.stdin.?.writeStreamingAll(io, bytes) catch |err| { - return run.step.fail("unable to write stdin: {t}", .{err}); - }; - child.stdin.?.close(io); - child.stdin = null; - }, - .lazy_path => |lazy_path| { - const path = lazy_path.getPath3(b, &run.step); - const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { - return run.step.fail("unable to open stdin file: {t}", .{err}); - }; - defer file.close(io); - // TODO https://github.com/ziglang/zig/issues/23955 - var read_buffer: [1024]u8 = undefined; - var file_reader = file.reader(io, &read_buffer); - var write_buffer: [1024]u8 = undefined; - var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); - _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { - error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ - path, file_reader.err.?, - }), - error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ - stdin_writer.err.?, - }), - }; - stdin_writer.interface.flush() catch |err| switch (err) { - error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ - stdin_writer.err.?, - }), - }; - child.stdin.?.close(io); - child.stdin = null; - }, - .none => {}, - } - - var stdout_bytes: ?[]const u8 = null; - var stderr_bytes: ?[]const u8 = null; - - if (child.stdout) |stdout| { - if (child.stderr) |stderr| { - var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; - var multi_reader: Io.File.MultiReader = undefined; - multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); - defer multi_reader.deinit(); - - const stdout_reader = multi_reader.reader(0); - const stderr_reader = multi_reader.reader(1); - - while (multi_reader.fill(64, .none)) |_| { - if (run.stdio_limit.toInt()) |limit| { - if (stdout_reader.buffered().len > limit) - return error.StdoutStreamTooLong; - if (stderr_reader.buffered().len > limit) - return error.StderrStreamTooLong; - } - } else |err| switch (err) { - error.Timeout => unreachable, - error.EndOfStream => {}, - else => |e| return e, - } - - try multi_reader.checkAnyError(); - - // TODO: this string can leak since alloc below can return error. - stdout_bytes = try multi_reader.toOwnedSlice(0); - // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. - stderr_bytes = try multi_reader.toOwnedSlice(1); - } else { - var stdout_reader = stdout.readerStreaming(io, &.{}); - stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => return stdout_reader.err.?, - error.StreamTooLong => return error.StdoutStreamTooLong, - }; - } - } else if (child.stderr) |stderr| { - var stderr_reader = stderr.readerStreaming(io, &.{}); - stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => return stderr_reader.err.?, - error.StreamTooLong => return error.StderrStreamTooLong, - }; - } - - if (stderr_bytes) |bytes| if (bytes.len > 0) { - // Treat stderr as an error message. - const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { - .check => |checks| !checksContainStderr(checks.items), - else => true, - }; - if (stderr_is_diagnostic) { - run.step.result_stderr = bytes; - } - }; - - run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; - - return .{ - .term = try child.wait(io), - .stdout = stdout_bytes, - .stderr = stderr_bytes, - }; -} - -fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { - const b = run.step.owner; - const compiles = artifact.getCompileDependencies(true); - for (compiles) |compile| { - if (compile.root_module.resolved_target.?.result.os.tag == .windows and - compile.isDynamicLibrary()) - { - addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); - } - } -} - -fn failForeign( - run: *Run, - suggested_flag: []const u8, - argv0: []const u8, - exe: *Step.Compile, -) error{ MakeFailed, MakeSkipped, OutOfMemory } { - switch (run.stdio) { - .check, .zig_test => { - if (run.skip_foreign_checks) - return error.MakeSkipped; - - const b = run.step.owner; - const host_name = try b.graph.host.result.zigTriple(b.allocator); - const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); - - return run.step.fail( - \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) - \\ consider using {s} or enabling skip_foreign_checks in the Run step - , .{ argv0, foreign_name, host_name, suggested_flag }); - }, - else => { - return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); - }, - } -} - -fn hashStdIo(hh: *std.Build.Cache.HashHelper, stdio: StdIo) void { - switch (stdio) { - .infer_from_args, .inherit, .zig_test => {}, - .check => |checks| for (checks.items) |check| { - hh.add(@as(std.meta.Tag(StdIo.Check), check)); - switch (check) { - .expect_stderr_exact, - .expect_stderr_match, - .expect_stdout_exact, - .expect_stdout_match, - => |s| hh.addBytes(s), - - .expect_term => |term| { - hh.add(@as(std.meta.Tag(process.Child.Term), term)); - switch (term) { - inline .exited, .signal, .stopped => |x| hh.add(x), - .unknown => |x| hh.add(x), - } - }, - } - }, - } -} diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index fd14090812ab8730822724792fe2233a420ecf5e..90d9e28155fabf1a853808d19bd3b81097fa745e 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -6,7 +6,7 @@ const mem = std.mem; const TranslateC = @This(); -pub const base_id: Step.Id = .translate_c; +pub const base_tag: Step.Tag = .translate_c; step: Step, source: std.Build.LazyPath, @@ -31,7 +31,7 @@ pub fn create(owner: *std.Build, options: Options) *TranslateC { const source = options.root_source_file.dupe(owner); translate_c.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "translate-c", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig index 0cc3b787c3f8416c284b0977ea9e73b9bf405fd7..b41cca59d9d82a6c959929101a4ebf203cc2cc1e 100644 --- a/lib/std/Build/Step/UpdateSourceFiles.zig +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -14,7 +14,7 @@ const ArrayList = std.ArrayList; step: Step, output_source_files: std.ArrayList(OutputSourceFile), -pub const base_id: Step.Id = .update_source_files; +pub const base_tag: Step.Tag = .update_source_files; pub const OutputSourceFile = struct { contents: Contents, @@ -30,7 +30,7 @@ pub fn create(owner: *std.Build) *UpdateSourceFiles { const usf = owner.allocator.create(UpdateSourceFiles) catch @panic("OOM"); usf.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "UpdateSourceFiles", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 3613fa3fef8fa5ac1a5587a4c4cdcca96661b7b6..06f030efbc8003fb31334141be311e2487bb0c52 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -18,7 +18,7 @@ directories: std.ArrayList(Directory), generated_directory: std.Build.GeneratedFile, mode: Mode = .whole_cached, -pub const base_id: Step.Id = .write_file; +pub const base_tag: Step.Tag = .write_file; pub const Mode = union(enum) { /// Default mode. Integrates with the cache system. The directory should be @@ -89,10 +89,9 @@ pub fn create(owner: *std.Build) *WriteFile { const write_file = owner.allocator.create(WriteFile) catch @panic("OOM"); write_file.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "WriteFile", .owner = owner, - .makeFn = make, }), .files = .empty, .directories = .empty, @@ -191,209 +190,3 @@ fn maybeUpdateName(write_file: *WriteFile) void { } } } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const graph = b.graph; - const io = graph.io; - const arena = b.allocator; - const gpa = graph.cache.gpa; - const write_file: *WriteFile = @fieldParentPtr("step", step); - - const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); - var open_dirs_count: usize = 0; - defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); - - switch (write_file.mode) { - .whole_cached => { - step.clearWatchInputs(); - - // The cache is used here not really as a way to speed things up - because writing - // the data to a file would probably be very fast - but as a way to find a canonical - // location to put build artifacts. - - // If, for example, a hard-coded path was used as the location to put WriteFile - // files, then two WriteFiles executing in parallel might clobber each other. - - var man = b.graph.cache.obtain(); - defer man.deinit(); - - for (write_file.files.items) |file| { - man.hash.addBytes(file.sub_path); - - switch (file.contents) { - .bytes => |bytes| { - man.hash.addBytes(bytes); - }, - .copy => |lazy_path| { - const path = lazy_path.getPath3(b, step); - _ = try man.addFilePath(path, null); - try step.addWatchInput(lazy_path); - }, - } - } - - for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { - man.hash.addBytes(dir.sub_path); - for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); - if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); - - const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); - const src_dir_path = dir.source.getPath3(b, step); - - var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {s}", .{ - src_dir_path, @errorName(err), - }); - }; - open_dir_cache_elem.* = src_dir; - open_dirs_count += 1; - - var it = try src_dir.walk(gpa); - defer it.deinit(); - while (try it.next(io)) |entry| { - if (!dir.options.pathIncluded(entry.path)) continue; - - switch (entry.kind) { - .directory => { - if (need_derived_inputs) { - const entry_path = try src_dir_path.join(arena, entry.path); - try step.addDirectoryWatchInputFromPath(entry_path); - } - }, - .file => { - const entry_path = try src_dir_path.join(arena, entry.path); - _ = try man.addFilePath(entry_path, null); - }, - else => continue, - } - } - } - - if (try step.cacheHit(&man)) { - const digest = man.final(); - write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); - assert(step.result_cached); - return; - } - - const digest = man.final(); - const cache_path = "o" ++ Dir.path.sep_str ++ digest; - - write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); - - try operate(write_file, open_dir_cache, .{ - .root_dir = b.cache_root, - .sub_path = cache_path, - }); - - try step.writeManifest(&man); - }, - .tmp => { - step.result_cached = false; - - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - - write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); - - try operate(write_file, open_dir_cache, .{ - .root_dir = b.cache_root, - .sub_path = tmp_dir_sub_path, - }); - }, - .mutate => |lp| { - step.result_cached = false; - const root_path = try lp.getPath4(b, step); - write_file.generated_directory.path = try root_path.toString(arena); - try operate(write_file, open_dir_cache, root_path); - }, - } -} - -fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { - const step = &write_file.step; - const b = step.owner; - const io = b.graph.io; - const gpa = b.graph.cache.gpa; - const arena = b.allocator; - - var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| - return step.fail("unable to make path {f}: {t}", .{ root_path, err }); - defer cache_dir.close(io); - - for (write_file.files.items) |file| { - if (Dir.path.dirname(file.sub_path)) |dirname| { - cache_dir.createDirPath(io, dirname) catch |err| { - return step.fail("unable to make path '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, dirname, err, - }); - }; - } - switch (file.contents) { - .bytes => |bytes| { - cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, file.sub_path, err, - }); - }; - }, - .copy => |file_source| { - const source_path = file_source.getPath2(b, step); - const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { - return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ - source_path, root_path, Dir.path.sep, file.sub_path, err, - }); - }; - // At this point we already will mark the step as a cache miss. - // But this is kind of a partial cache hit since individual - // file copies may be avoided. Oh well, this information is - // discarded. - _ = prev_status; - }, - } - } - - for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { - const src_dir_path = dir.source.getPath3(b, step); - const dest_dirname = dir.sub_path; - - if (dest_dirname.len != 0) { - cache_dir.createDirPath(io, dest_dirname) catch |err| { - return step.fail("unable to make path '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, dest_dirname, err, - }); - }; - } - - var it = try already_open_dir.walk(gpa); - defer it.deinit(); - while (try it.next(io)) |entry| { - if (!dir.options.pathIncluded(entry.path)) continue; - - const src_entry_path = try src_dir_path.join(arena, entry.path); - const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); - switch (entry.kind) { - .directory => try cache_dir.createDirPath(io, dest_path), - .file => { - const prev_status = Io.Dir.updateFile( - src_entry_path.root_dir.handle, - io, - src_entry_path.sub_path, - cache_dir, - dest_path, - .{}, - ) catch |err| { - return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ - src_entry_path, root_path, Dir.path.sep, dest_path, err, - }); - }; - _ = prev_status; - }, - else => continue, - } - } - } -} diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 9c2c956582d70a6582a268641702f3b41f7c08c7..75ef9c9b63195335b78c5cfa04d4d75742532905 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -11,8 +11,6 @@ const Writer = std.Io.Writer; const tokenizer = @import("zig/tokenizer.zig"); -/// The serialized output of configure phase ingested by make phase. -pub const Configuration = @import("zig/Configuration.zig"); pub const ErrorBundle = @import("zig/ErrorBundle.zig"); pub const Server = @import("zig/Server.zig"); pub const Client = @import("zig/Client.zig"); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 86bf82fa3d0c73117dc85755b36a240e76fbc8c1..589828295d70f3a2daffd4307f40636daf4a234a 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -3,22 +3,240 @@ const Configuration = @This(); const std = @import("../std.zig"); const Io = std.Io; const Allocator = std.mem.Allocator; +const assert = std.debug.assert; string_bytes: []u8, steps: []Step, path_deps_base: []Path.Base, path_deps_sub: []String, unlazy_deps: []String, +extra: []u32, +/// The field order here matches `Configuration` which documents the order in +/// the serialized format. pub const Header = extern struct { string_bytes_len: u32, steps_len: u32, path_deps_len: u32, unlazy_deps_len: u32, + extra_len: u32, + + /// Index into `steps`. + default_step: u32, +}; + +pub const Wip = struct { + gpa: Allocator, + string_table: StringTable = .empty, + deps_table: DepsTable = .empty, + + string_bytes: std.ArrayList(u8) = .empty, + unlazy_deps: std.ArrayList(String) = .empty, + steps: std.ArrayList(Step) = .empty, + path_deps: std.MultiArrayList(Path) = .empty, + extra: std.ArrayList(u32) = .empty, + + const DepsTable = std.HashMapUnmanaged(Deps, void, DepsTableContext, std.hash_map.default_max_load_percentage); + + const DepsTableContext = struct { + extra: []const u32, + + pub fn eql(ctx: @This(), a: Deps, b: Deps) bool { + const len_a = ctx.extra[@intFromEnum(a)]; + const len_b = ctx.extra[@intFromEnum(b)]; + const slice_a = ctx.extra[@intFromEnum(a) + 1 ..][0..len_a]; + const slice_b = ctx.extra[@intFromEnum(b) + 1 ..][0..len_b]; + return std.mem.eql(u32, slice_a, slice_b); + } + + pub fn hash(ctx: @This(), key: Deps) u64 { + const len = ctx.extra[@intFromEnum(key)]; + const slice = ctx.extra[@intFromEnum(key) + 1 ..][0..len]; + return std.hash_map.hashString(@ptrCast(slice)); + } + }; + + const StringTable = std.HashMapUnmanaged(String, void, StringTableContext, std.hash_map.default_max_load_percentage); + const StringTableContext = struct { + bytes: []const u8, + + pub fn eql(_: @This(), a: String, b: String) bool { + return a == b; + } + + pub fn hash(ctx: @This(), key: String) u64 { + return std.hash_map.hashString(std.mem.sliceTo(ctx.bytes[@intFromEnum(key)..], 0)); + } + }; + + const StringTableIndexAdapter = struct { + bytes: []const u8, + + pub fn eql(ctx: @This(), a: []const u8, b: String) bool { + return std.mem.eql(u8, a, std.mem.sliceTo(ctx.bytes[@intFromEnum(b)..], 0)); + } + + pub fn hash(_: @This(), adapted_key: []const u8) u64 { + assert(std.mem.indexOfScalar(u8, adapted_key, 0) == null); + return std.hash_map.hashString(adapted_key); + } + }; + + pub fn init(gpa: Allocator) Wip { + return .{ .gpa = gpa }; + } + + pub fn deinit(wip: *Wip) void { + const gpa = wip.gpa; + wip.string_bytes.deinit(gpa); + wip.unlazy_deps.deinit(gpa); + wip.steps.deinit(gpa); + wip.path_deps.deinit(gpa); + wip.extra.deinit(gpa); + wip.* = undefined; + } + + pub const Static = struct { + default_step: u32, + }; + + pub fn write(wip: *Wip, w: *Io.Writer, static: Static) Io.Writer.Error!void { + const header: Header = .{ + .string_bytes_len = @intCast(wip.string_bytes.items.len), + .steps_len = @intCast(wip.steps.items.len), + .path_deps_len = @intCast(wip.path_deps.len), + .unlazy_deps_len = @intCast(wip.unlazy_deps.items.len), + .extra_len = @intCast(wip.extra.items.len), + + .default_step = static.default_step, + }; + var buffers = [_][]const u8{ + @ptrCast(&header), + wip.string_bytes.items, + @ptrCast(wip.steps.items), + @ptrCast(wip.path_deps.items(.base)), + @ptrCast(wip.path_deps.items(.sub)), + @ptrCast(wip.unlazy_deps.items), + @ptrCast(wip.extra.items), + }; + try w.writeVecAll(&buffers); + } + + pub fn addString(wip: *Wip, bytes: []const u8) Allocator.Error!String { + const gpa = wip.gpa; + assert(std.mem.indexOfScalar(u8, bytes, 0) == null); + const gop = try wip.string_table.getOrPutContextAdapted( + gpa, + @as([]const u8, bytes), + @as(StringTableIndexAdapter, .{ .bytes = wip.string_bytes.items }), + @as(StringTableContext, .{ .bytes = wip.string_bytes.items }), + ); + if (gop.found_existing) return gop.key_ptr.*; + + try wip.string_bytes.ensureUnusedCapacity(gpa, bytes.len + 1); + const new_off: String = @enumFromInt(wip.string_bytes.items.len); + + wip.string_bytes.appendSliceAssumeCapacity(bytes); + wip.string_bytes.appendAssumeCapacity(0); + + gop.key_ptr.* = new_off; + + return new_off; + } + + pub fn prepareDeps(wip: *Wip, n: usize) Allocator.Error![]u32 { + const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1); + slice[0] = @intCast(n); + return slice[1..]; + } + + pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps { + const gpa = wip.gpa; + const gop = try wip.deps_table.getOrPutContext(gpa, deps, @as(DepsTableContext, .{ + .extra = wip.extra.items, + })); + if (gop.found_existing) { + wip.extra.items.len = @intFromEnum(deps); + return gop.key_ptr.*; + } else { + return deps; + } + } + + pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { + const gpa = wip.gpa; + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + try wip.extra.ensureUnusedCapacity(gpa, fields.len); + return addExtraAssumeCapacity(wip, extra); + } + + pub fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 { + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + const result: u32 = @intCast(wip.extra.items.len); + wip.extra.items.len += fields.len; + setExtra(wip, result, extra); + return result; + } + + fn setExtra(wip: *Wip, index: usize, extra: anytype) void { + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + var i = index; + inline for (fields) |field| { + wip.extra.items[i] = switch (field.type) { + u32 => @field(extra, field.name), + String, Deps => @intFromEnum(@field(extra, field.name)), + else => @compileError("bad field type"), + }; + i += 1; + } + } }; pub const Step = extern struct { name: String, + flags: Flags, + deps: Deps, + /// Points into `extra` for step-specific data. + extra_index: u32, + + pub const Flags = packed struct(u32) { + tag: Tag, + _: u24 = 0, + }; + + pub const Index = enum(u32) { + _, + }; + + pub const Tag = enum(u8) { + top_level, + compile, + install_artifact, + install_file, + install_dir, + remove_dir, + fail, + fmt, + translate_c, + write_file, + update_source_files, + run, + check_file, + check_object, + config_header, + objcopy, + options, + }; + + pub const TopLevel = struct { + description: String, + }; +}; + +/// Points into `extra`, where the first element is number of deps, +/// following elements is `Step.Index` per dep. +pub const Deps = enum(u32) { + _, }; pub const Path = extern struct { @@ -27,8 +245,8 @@ pub const Path = extern struct { pub const Base = enum(u8) { cwd, - global_cache, local_cache, + global_cache, build_root, }; @@ -40,6 +258,7 @@ pub const Path = extern struct { } }; +/// Points into `string_bytes`, null-terminated. pub const String = enum(u32) { _, @@ -49,24 +268,29 @@ pub const String = enum(u32) { } }; -pub const LoadError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; +pub const LoadFileError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; -pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration { +pub fn loadFile(arena: Allocator, io: Io, file: Io.File) LoadFileError!Configuration { var buffer: [2000]u8 = undefined; var fr = file.reader(io, &buffer); - const header = fr.interface.takeStruct(Header, .little) catch |err| switch (err) { + return load(arena, &fr.interface) catch |err| switch (err) { error.ReadFailed => return fr.err.?, else => |e| return e, }; +} +pub const LoadError = Io.Reader.Error || Allocator.Error; + +pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { + const header = try reader.takeStruct(Header, .little); var result: Configuration = .{ .string_bytes = try arena.alloc(u8, header.string_bytes_len), .steps = try arena.alloc(Step, header.steps_len), .path_deps_sub = try arena.alloc(String, header.path_deps_len), .path_deps_base = try arena.alloc(Path.Base, header.path_deps_len), .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), + .extra = try arena.alloc(u32, header.extra_len), }; - var vecs = [_][]u8{ result.string_bytes, @ptrCast(result.steps), @@ -74,10 +298,6 @@ pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration { @ptrCast(result.path_deps_sub), @ptrCast(result.unlazy_deps), }; - fr.interface.readVecAll(&vecs) catch |err| switch (err) { - error.ReadFailed => return fr.err.?, - else => |e| return e, - }; - + try reader.readVecAll(&vecs); return result; } diff --git a/src/main.zig b/src/main.zig index 7648cadeeaa7a19581beac9e2f6175ddec486157..5f2776cf38a7b7a5c3572115f7ead563dc0733b9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -298,7 +298,11 @@ fn mainArgs( return process.exit(try llvmArMain(arena, args)); } else if (mem.eql(u8, cmd, "build")) { dev.check(.build_command); - return cmdBuild(gpa, arena, io, cmd_args, environ_map); + var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ + .child_allocator = arena, + .io = io, + }; + return cmdBuild(gpa, thread_safe_arena.allocator(), io, cmd_args, environ_map); } else if (mem.eql(u8, cmd, "clang") or mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) { @@ -4941,6 +4945,7 @@ test sanitizeExampleName { fn cmdBuild( gpa: Allocator, + /// Needs a thread-safe arena. arena: Allocator, io: Io, args: []const []const u8, @@ -4973,28 +4978,34 @@ fn cmdBuild( var debug_target: ?[]const u8 = null; var debug_libc_paths_file: ?[]const u8 = null; - const argv_index_exe = configure_argv.items.len; - _ = try configure_argv.addOne(arena); - const self_exe_path = try process.executablePathAlloc(io, arena); - try configure_argv.append(arena, self_exe_path); + const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}); + try configure_argv.ensureUnusedCapacity(arena, 16); + + const argv_index_exe = configure_argv.items.len; + _ = configure_argv.addOneAssumeCapacity(); + + configure_argv.appendAssumeCapacity("--zig"); + configure_argv.appendAssumeCapacity(self_exe_path); + + configure_argv.appendAssumeCapacity("--zig-lib-dir"); const argv_index_zig_lib_dir = configure_argv.items.len; - _ = try configure_argv.addOne(arena); + _ = configure_argv.addOneAssumeCapacity(); + configure_argv.appendAssumeCapacity("--build-root"); const argv_index_build_file = configure_argv.items.len; - _ = try configure_argv.addOne(arena); + _ = configure_argv.addOneAssumeCapacity(); + configure_argv.appendAssumeCapacity("--local-cache"); const argv_index_cache_dir = configure_argv.items.len; - _ = try configure_argv.addOne(arena); + _ = configure_argv.addOneAssumeCapacity(); + configure_argv.appendAssumeCapacity("--global-cache"); const argv_index_global_cache_dir = configure_argv.items.len; - _ = try configure_argv.addOne(arena); + _ = configure_argv.addOneAssumeCapacity(); - try configure_argv.appendSlice(arena, &.{ - "--seed", - try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}), - }); + configure_argv.appendSliceAssumeCapacity(&.{ "--seed", default_seed }); const argv_index_seed = configure_argv.items.len - 1; const argv_index_configuration_file = make_argv.items.len; @@ -5192,14 +5203,6 @@ fn cmdBuild( ); try setThreadLimit(arena, thread_limit); - // Kick off an optimized compilation of the make runner. - var make_runner_task = io.async(compileMakeRunner, .{ io, .{ - .dirs = &dirs, - .optimize = .ReleaseSafe, - .parent_prog_node = root_prog_node, - } }); - defer if (make_runner_task.cancel(io)) |mr| mr.deinit(io) else |_| {}; - // Cache lookup for configure options. If we get a match, we can skip // execution of the configure script. If not, we get the file path to pass // to the configure process. @@ -5255,6 +5258,19 @@ fn cmdBuild( break :lci lci; }; + // Kick off an optimized compilation of the make runner. + var make_runner_task = io.async(compileMakeRunner, .{ gpa, arena, io, .{ + .dirs = &dirs, + .environ_map = environ_map, + .parent_prog_node = root_prog_node, + .resolved_target = resolved_target, + .libc_installation = libc_installation, + .thread_limit = thread_limit, + .self_exe_path = self_exe_path, + .color = color, + } }); + defer _ = make_runner_task.cancel(io) catch {}; + configure_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; configure_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; configure_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; @@ -5305,7 +5321,7 @@ fn cmdBuild( .root_src_path = fs.path.basename(runner), } else .{ .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), - .root_src_path = "build_runner.zig", + .root_src_path = "configure_runner.zig", }; const config = try Compilation.Config.resolve(.{ @@ -5533,7 +5549,7 @@ fn cmdBuild( const comp = Compilation.create(gpa, arena, io, &create_diag, .{ .libc_installation = libc_installation, .dirs = dirs, - .root_name = "build", + .root_name = "configure", .config = config, .root_mod = root_mod, .main_mod = build_mod, @@ -5554,7 +5570,7 @@ fn cmdBuild( .environ_map = environ_map, }) catch |err| switch (err) { error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - else => fatal("failed to create compilation: {t}", .{err}), + else => |e| fatal("failed to create compilation: {t}", .{e}), }; defer comp.destroy(); @@ -5625,7 +5641,7 @@ fn cmdBuild( // add them to `config_man` before obtaining the final digest. // * If it contains a set of lazy packages that need to be // fetched, we need to fetch those now and re-run configure. - var configuration = std.zig.Configuration.load(arena, io, config_tmp_file) catch |err| + var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); if (configuration.unlazy_deps.len != 0) { @@ -5661,7 +5677,7 @@ fn cmdBuild( } for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { - const conf_path: std.zig.Configuration.Path = .{ .base = base, .sub = sub }; + const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); } @@ -5707,7 +5723,6 @@ fn cmdBuild( const make_runner = make_runner_task.await(io) catch |err| fatal("failed to compile maker: {t}", .{err}); - defer make_runner.deinit(io); make_argv.items[0] = try make_runner.exe_path.toString(arena); make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); @@ -5748,22 +5763,86 @@ const MakeRunner = struct { exe_path: Path, const Options = struct { + environ_map: *const process.Environ.Map, dirs: *Compilation.Directories, - optimize: std.builtin.OptimizeMode, parent_prog_node: std.Progress.Node, + resolved_target: Package.Module.ResolvedTarget, + libc_installation: ?*const LibCInstallation, + self_exe_path: []const u8, + thread_limit: usize, + color: Color, }; - - fn deinit(mr: MakeRunner, io: Io) void { - _ = mr; - _ = io; - @panic("TODO"); - } }; -fn compileMakeRunner(io: Io, options: MakeRunner.Options) !MakeRunner { - _ = io; - _ = options; - @panic("TODO"); +fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner { + const compile_prog_node = options.parent_prog_node.start("Compile Maker", 0); + defer compile_prog_node.end(); + + const optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(options.environ_map)) + .Debug + else + .ReleaseSafe; + const strip = optimize_mode != .Debug; + + const main_mod_paths: Package.Module.CreateOptions.Paths = .{ + .root = try .fromRoot(arena, options.dirs.*, .zig_lib, "compiler"), + .root_src_path = "maker.zig", + }; + + const config = try Compilation.Config.resolve(.{ + .output_mode = .Exe, + .root_strip = strip, + .root_optimize_mode = optimize_mode, + .resolved_target = options.resolved_target, + .have_zcu = true, + .emit_bin = true, + .is_test = false, + }); + + const root_mod = try Package.Module.create(arena, .{ + .paths = main_mod_paths, + .fully_qualified_name = "root", + .cc_argv = &.{}, + .inherited = .{ + .resolved_target = options.resolved_target, + .optimize_mode = optimize_mode, + .strip = strip, + }, + .global = config, + .parent = null, + }); + + var create_diag: Compilation.CreateDiagnostic = undefined; + const comp = Compilation.create(gpa, arena, io, &create_diag, .{ + .dirs = options.dirs.*, + .root_name = "maker", + .config = config, + .root_mod = root_mod, + .main_mod = root_mod, + .emit_bin = .yes_cache, + .self_exe_path = options.self_exe_path, + .thread_limit = options.thread_limit, + .cache_mode = .whole, + .environ_map = options.environ_map, + }) catch |err| switch (err) { + error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), + error.Canceled => |e| return e, + else => |e| fatal("failed to create compilation: {t}", .{e}), + }; + defer comp.destroy(); + + try updateModule(comp, options.color, compile_prog_node); + + const exe_path: Path = .{ + .root_dir = options.dirs.global_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ + &Cache.binToHex(comp.digest.?), comp.emit_bin.?, + }), + }; + + return .{ + .exe_path = exe_path, + }; } const Fork = struct { @@ -5972,7 +6051,7 @@ fn jitCmdInner( .environ_map = environ_map, }) catch |err| switch (err) { error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - else => fatal("failed to create compilation: {s}", .{@errorName(err)}), + else => fatal("failed to create compilation: {t}", .{err}), }; defer comp.destroy(); -- 2.54.0 From 0b3ca115206c0332284c80f7f7d42358d96379c1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Feb 2026 16:04:44 -0800 Subject: [PATCH 003/179] std.Io.Writer: placeholder code for new {q} format character intendend to print something with double quotes, escaped the same as a zig string literal. --- lib/std/Io/Writer.zig | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 813142a7a35e48c1de0eaba49bea2a5f47ec1139..c489a5c93d6ef3b4adcf94a2eca16daff8a56eee 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -1172,6 +1172,25 @@ pub fn printValue( }, else => invalidFmtError(fmt, value), }, + // TODO make this print double quotes and quote-escape the string + // according to zig string syntax rules + 'q' => switch (@typeInfo(T)) { + .pointer => |info| switch (info.size) { + .one, .slice => { + const slice: []const u8 = value; + return w.alignBufferOptions(slice, options); + }, + .many, .c => { + const slice: [:0]const u8 = std.mem.span(value); + return w.alignBufferOptions(slice, options); + }, + }, + .array => { + const slice: []const u8 = &value; + return w.alignBufferOptions(slice, options); + }, + else => invalidFmtError(fmt, value), + }, 'B' => switch (@typeInfo(T)) { .int, .comptime_int => return w.printByteSize(value, .decimal, options), .@"struct" => return value.formatByteSize(w, .decimal), -- 2.54.0 From 83a34758874dbe98d3566e297b56cd3be0e66900 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Feb 2026 18:47:38 -0800 Subject: [PATCH 004/179] configure runner: implement serialization of InstallArtifact --- lib/compiler/configure_runner.zig | 107 +++++++++++++++------ lib/std/Build.zig | 1 - lib/std/Build/Step/InstallArtifact.zig | 4 +- lib/std/Build/Step/Run.zig | 7 +- lib/std/zig/Configuration.zig | 124 +++++++++++++++++++++++-- src/main.zig | 23 ++++- 6 files changed, 219 insertions(+), 47 deletions(-) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 78d3dff4c399392b1519d2d3d78368b5b4e2ffcd..07caacc4e2dcfd71af2343f38f4a96a165d1cd8c 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -140,16 +140,13 @@ pub fn main(init: process.Init.Minimal) !void { if (try builder.addUserInputFlag(option_contents)) fatal(" access the help menu with 'zig build -h'", .{}); } - } else if (mem.startsWith(u8, arg, "-fsys=")) { - const name = arg["-fsys=".len..]; + } else if (mem.cutPrefix(u8, arg, "-fsys=")) |name| { graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); - } else if (mem.startsWith(u8, arg, "-fno-sys=")) { - const name = arg["-fno-sys=".len..]; + } else if (mem.cutPrefix(u8, arg, "-fno-sys=")) |name| { graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); } else if (mem.eql(u8, arg, "--release")) { graph.release_mode = .any; - } else if (mem.startsWith(u8, arg, "--release=")) { - const text = arg["--release=".len..]; + } else if (mem.cutPrefix(u8, arg, "--release=")) |text| { graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ arg, text, @@ -175,23 +172,11 @@ pub fn main(init: process.Init.Minimal) !void { multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); }; - } else if (mem.eql(u8, arg, "--seed")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u32 after '{s}'", .{arg}); - graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ - next_arg, @errorName(err), - }); - }; } else if (mem.eql(u8, arg, "--build-id")) { builder.build_id = .fast; - } else if (mem.startsWith(u8, arg, "--build-id=")) { - const style = arg["--build-id=".len..]; - builder.build_id = std.zig.BuildId.parse(style) catch |err| { - fatal("unable to parse --build-id style '{s}': {s}", .{ - style, @errorName(err), - }); - }; + } else if (mem.cutPrefix(u8, arg, "--build-id=")) |style| { + builder.build_id = std.zig.BuildId.parse(style) catch |err| + fatal("unable to parse --build-id style '{s}': {t}", .{ style, err }); } else if (mem.eql(u8, arg, "--debug-rt")) { graph.debug_compiler_runtime_libs = true; } else if (mem.eql(u8, arg, "--debug-compile-errors")) { @@ -203,11 +188,6 @@ pub fn main(init: process.Init.Minimal) !void { // but it is handled by the parent process. The build runner // only sees this flag. graph.system_package_mode = true; - } else if (mem.cutPrefix(u8, arg, "-j")) |text| { - const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| - fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); - if (n < 1) fatal("number of jobs must be at least 1", .{}); - threaded.setAsyncLimit(.limited(n)); } else { fatalWithHint("unrecognized argument: '{s}'", .{arg}); } @@ -281,6 +261,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .name = try wc.addString(step.name), .flags = .{ .tag = step.tag }, .deps = deps, + .max_rss = .fromBytes(step.max_rss), .extra_index = switch (step.tag) { .top_level => e: { const top_level: *Step.TopLevel = @fieldParentPtr("step", step); @@ -289,7 +270,21 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { })); }, .compile => @panic("TODO"), - .install_artifact => @panic("TODO"), + .install_artifact => e: { + const ia: *Step.InstallArtifact = @fieldParentPtr("step", step); + break :e try wc.addExtra(@as(Configuration.Step.InstallArtifact, .{ + .dest_dir = try addInstallDir(wc, ia.dest_dir), + .dest_sub_path = try wc.addString(ia.dest_sub_path), + .emitted_bin = try addOptionalLazyPath(wc, ia.emitted_bin), + .implib_dir = try addInstallDir(wc, ia.implib_dir), + .emitted_implib = try addOptionalLazyPath(wc, ia.emitted_implib), + .pdb_dir = try addInstallDir(wc, ia.pdb_dir), + .emitted_pdb = try addOptionalLazyPath(wc, ia.emitted_pdb), + .h_dir = try addInstallDir(wc, ia.h_dir), + .emitted_h = try addOptionalLazyPath(wc, ia.emitted_h), + .artifact = stepIndex(&step_map, &ia.artifact.step), + })); + }, .install_file => @panic("TODO"), .install_dir => @panic("TODO"), .remove_dir => @panic("TODO"), @@ -315,10 +310,66 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { } try wc.write(writer, .{ - .default_step = @intCast(step_map.getIndex(b.default_step).?), + .default_step = stepIndex(&step_map, b.default_step), }); } +fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configuration.OptionalLazyPath { + return @enumFromInt(switch (lp orelse return .none) { + .src_path => |src_path| i: { + const owner = builderToPackage(src_path.owner); + const sub_path = try wc.addString(src_path.sub_path); + break :i try wc.addExtra(@as(Configuration.OptionalLazyPath.SourcePath, .{ + .flags = .{}, + .owner = owner, + .sub_path = sub_path, + })); + }, + .generated => |generated| i: { + const sub_path = try wc.addString(generated.sub_path); + break :i try wc.addExtra(@as(Configuration.OptionalLazyPath.Generated, .{ + .flags = .{ .up = @intCast(generated.up) }, + .sub_path = sub_path, + })); + }, + .cwd_relative => |cwd_relative_sub_path| i: { + const sub_path = try wc.addString(cwd_relative_sub_path); + break :i try wc.addExtra(@as(Configuration.OptionalLazyPath.Relative, .{ + .flags = .{ .base = .cwd }, + .sub_path = sub_path, + })); + }, + .dependency => |dependency| i: { + const owner = builderToPackage(dependency.dependency.builder); + const sub_path = try wc.addString(dependency.sub_path); + break :i try wc.addExtra(@as(Configuration.OptionalLazyPath.SourcePath, .{ + .flags = .{}, + .owner = owner, + .sub_path = sub_path, + })); + }, + }); +} + +fn builderToPackage(b: *std.Build) Configuration.Package { + _ = b; + @panic("TODO"); +} + +fn addInstallDir(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !Configuration.InstallDir { + switch (install_dir orelse return .none) { + .prefix => return .prefix, + .lib => return .lib, + .bin => return .bin, + .header => return .header, + .custom => |sub_path| return .initCustom(try wc.addString(sub_path)), + } +} + +fn stepIndex(step_map: *const std.AutoArrayHashMapUnmanaged(*Step, void), step: *Step) Configuration.Step.Index { + return @enumFromInt(step_map.getIndex(step).?); +} + /// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which /// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { diff --git a/lib/std/Build.zig b/lib/std/Build.zig index f0edfb7b817eb2dbc0d03dff4a0267c2da519ae6..6fe78cb1d6c7ad681379413d204275b1446d6d70 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -103,7 +103,6 @@ pub const Graph = struct { needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .empty, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, - random_seed: u32 = 0, dependency_cache: InitializedDepMap = .empty, allow_so_scripts: ?bool = null, /// Steps should use `io` to limit the number of jobs, however in the case of diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig index f4e9b1d8185f665a08e90983514593b43357bdb8..820de6b142c2a0f2d909935fdf782d7647672b97 100644 --- a/lib/std/Build/Step/InstallArtifact.zig +++ b/lib/std/Build/Step/InstallArtifact.zig @@ -1,8 +1,8 @@ +const InstallArtifact = @This(); + const std = @import("std"); const Step = std.Build.Step; const InstallDir = std.Build.InstallDir; -const InstallArtifact = @This(); -const fs = std.fs; const LazyPath = std.Build.LazyPath; step: Step, diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index b7dd256d8ed34189e9f59fbcf1b41b1bdcb4eb15..f1cde9d5336eb0f3a316ee6911b7c1d20d80688f 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -87,6 +87,7 @@ captured_stderr: ?*CapturedStdIo, dep_output_file: ?*Output, has_side_effects: bool, +test_runner_mode: bool = false, /// Populated during the fuzz phase if this run step corresponds to a unit test /// executable that contains fuzz tests. @@ -234,13 +235,11 @@ pub fn setName(run: *Run, name: []const u8) void { } pub fn enableTestRunnerMode(run: *Run) void { + if (run.test_runner_mode) return; const b = run.step.owner; run.stdio = .zig_test; run.addPrefixedDirectoryArg("--cache-dir=", .{ .cwd_relative = b.cache_root.path orelse "." }); - run.addArgs(&.{ - b.fmt("--seed=0x{x}", .{b.graph.random_seed}), - "--listen=-", - }); + run.test_runner_mode = true; } pub fn addArtifactArg(run: *Run, artifact: *Step.Compile) void { diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 589828295d70f3a2daffd4307f40636daf4a234a..ae83e63ef4d56e362a9e1f5856ebe8cc2ce781cf 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -4,6 +4,7 @@ const std = @import("../std.zig"); const Io = std.Io; const Allocator = std.mem.Allocator; const assert = std.debug.assert; +const maxInt = std.math.maxInt; string_bytes: []u8, steps: []Step, @@ -21,8 +22,7 @@ pub const Header = extern struct { unlazy_deps_len: u32, extra_len: u32, - /// Index into `steps`. - default_step: u32, + default_step: Step.Index, }; pub const Wip = struct { @@ -97,7 +97,7 @@ pub const Wip = struct { } pub const Static = struct { - default_step: u32, + default_step: Step.Index, }; pub fn write(wip: *Wip, w: *Io.Writer, static: Static) Io.Writer.Error!void { @@ -182,10 +182,12 @@ pub const Wip = struct { const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; var i = index; inline for (fields) |field| { - wip.extra.items[i] = switch (field.type) { - u32 => @field(extra, field.name), - String, Deps => @intFromEnum(@field(extra, field.name)), - else => @compileError("bad field type"), + comptime assert(@sizeOf(field.type) == @sizeOf(u32)); + wip.extra.items[i] = switch (@typeInfo(field.type)) { + .int => @field(extra, field.name), + .@"enum" => @intFromEnum(@field(extra, field.name)), + .@"struct" => @bitCast(@field(extra, field.name)), + else => @compileError("bad field type: " ++ @typeName(field.type)), }; i += 1; } @@ -196,6 +198,7 @@ pub const Step = extern struct { name: String, flags: Flags, deps: Deps, + max_rss: MaxRss, /// Points into `extra` for step-specific data. extra_index: u32, @@ -231,6 +234,98 @@ pub const Step = extern struct { pub const TopLevel = struct { description: String, }; + + pub const InstallArtifact = struct { + dest_dir: InstallDir, + dest_sub_path: String, + emitted_bin: OptionalLazyPath, + + implib_dir: InstallDir, + emitted_implib: OptionalLazyPath, + + pdb_dir: InstallDir, + emitted_pdb: OptionalLazyPath, + + h_dir: InstallDir, + emitted_h: OptionalLazyPath, + + /// Always a compile step. + artifact: Step.Index, + + const Flags = packed struct(u32) { + tag: Tag = .install_artifact, + dylib_symlinks: bool, + _: u23 = 0, + }; + }; +}; + +pub const MaxRss = enum(u32) { + none = 0, + _, + + pub fn toBytes(mr: MaxRss) usize { + const x: usize = @intFromEnum(mr); + return x << 8; + } + + pub fn fromBytes(bytes: usize) MaxRss { + return @enumFromInt(bytes >> 8); + } +}; + +/// An index into `extra`. +pub const OptionalLazyPath = enum(u32) { + none = maxInt(u32), + _, + + pub const Tag = enum(u8) { + /// A source file path relative to build root. + source_path, + generated, + relative, + }; + + pub const SourcePath = struct { + flags: Flags, + owner: Package, + sub_path: String, + + pub const Flags = packed struct(u32) { + tag: Tag = .source_path, + _: u24 = 0, + }; + }; + + pub const Generated = struct { + flags: Flags, + /// Applied after `up`. + sub_path: String, + + pub const Flags = packed struct(u32) { + tag: Tag = .generated, + /// The number of parent directories to go up. + /// 0 means the generated file itself. + /// 1 means the directory of the generated file. + /// 2 means the parent of that directory, and so on. + up: u24, + }; + }; + + pub const Relative = struct { + flags: Flags, + sub_path: String, + + pub const Flags = packed struct(u32) { + tag: Tag = .relative, + base: Path.Base, + _: u16 = 0, + }; + }; +}; + +pub const Package = enum(u32) { + _, }; /// Points into `extra`, where the first element is number of deps, @@ -258,6 +353,21 @@ pub const Path = extern struct { } }; +pub const InstallDir = enum(u32) { + none = maxInt(u32) - 4, + prefix = maxInt(u32) - 3, + lib = maxInt(u32) - 2, + bin = maxInt(u32) - 1, + header = maxInt(u32), + /// A `String` path relative to the prefix. + _, + + pub fn initCustom(sub_path: String) InstallDir { + assert(@intFromEnum(sub_path) < @intFromEnum(InstallDir.none)); + return @enumFromInt(@intFromEnum(sub_path)); + } +}; + /// Points into `string_bytes`, null-terminated. pub const String = enum(u32) { _, diff --git a/src/main.zig b/src/main.zig index 5f2776cf38a7b7a5c3572115f7ead563dc0733b9..a0296f8a79e30d3ec7f018f8695873867aba25ce 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4982,34 +4982,47 @@ fn cmdBuild( const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}); try configure_argv.ensureUnusedCapacity(arena, 16); + try make_argv.ensureUnusedCapacity(arena, 16); const argv_index_exe = configure_argv.items.len; _ = configure_argv.addOneAssumeCapacity(); + _ = make_argv.addOneAssumeCapacity(); configure_argv.appendAssumeCapacity("--zig"); configure_argv.appendAssumeCapacity(self_exe_path); + make_argv.appendAssumeCapacity("--zig"); + make_argv.appendAssumeCapacity(self_exe_path); + configure_argv.appendAssumeCapacity("--zig-lib-dir"); + make_argv.appendAssumeCapacity("--zig-lib-dir"); const argv_index_zig_lib_dir = configure_argv.items.len; _ = configure_argv.addOneAssumeCapacity(); + _ = make_argv.addOneAssumeCapacity(); configure_argv.appendAssumeCapacity("--build-root"); + make_argv.appendAssumeCapacity("--build-root"); const argv_index_build_file = configure_argv.items.len; _ = configure_argv.addOneAssumeCapacity(); + _ = make_argv.addOneAssumeCapacity(); configure_argv.appendAssumeCapacity("--local-cache"); + make_argv.appendAssumeCapacity("--local-cache"); const argv_index_cache_dir = configure_argv.items.len; _ = configure_argv.addOneAssumeCapacity(); + _ = make_argv.addOneAssumeCapacity(); configure_argv.appendAssumeCapacity("--global-cache"); + make_argv.appendAssumeCapacity("--global-cache"); const argv_index_global_cache_dir = configure_argv.items.len; _ = configure_argv.addOneAssumeCapacity(); + _ = make_argv.addOneAssumeCapacity(); - configure_argv.appendSliceAssumeCapacity(&.{ "--seed", default_seed }); - const argv_index_seed = configure_argv.items.len - 1; + make_argv.appendSliceAssumeCapacity(&.{ "--configuration", undefined }); + const argv_index_configuration_file = make_argv.items.len - 1; - const argv_index_configuration_file = make_argv.items.len; - _ = try make_argv.addOne(arena); + make_argv.appendSliceAssumeCapacity(&.{ "--seed", default_seed }); + const argv_index_seed = make_argv.items.len - 1; var color: Color = .auto; var n_jobs: ?u32 = null; @@ -5151,7 +5164,7 @@ fn cmdBuild( } else if (mem.eql(u8, arg, "--seed")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; - configure_argv.items[argv_index_seed] = args[i]; + make_argv.items[argv_index_seed] = args[i]; continue; } else if (mem.eql(u8, arg, "--")) { // The rest of the args are supposed to get passed onto -- 2.54.0 From 6b040d631fd5090d85c56b39006162f2aa4991b1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Feb 2026 21:32:33 -0800 Subject: [PATCH 005/179] configure runner: add Step.Run serialization --- lib/compiler/configure_runner.zig | 62 +++++++++++++-- lib/compiler/maker/Step/Run.zig | 3 + lib/compiler/maker/Watch.zig | 4 +- lib/std/Build/Step/Run.zig | 26 +------ lib/std/zig/Configuration.zig | 124 +++++++++++++++++++++++++++--- 5 files changed, 178 insertions(+), 41 deletions(-) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 07caacc4e2dcfd71af2343f38f4a96a165d1cd8c..a687af87650b584290e0fb521895b74302300a71 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -206,6 +206,7 @@ pub fn main(init: process.Init.Minimal) !void { var wc: Configuration.Wip = .init(gpa); defer wc.deinit(); + assert(try wc.addString("") == .empty); var stdout_buffer: [1024]u8 = undefined; var file_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); @@ -259,7 +260,6 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { try wc.steps.ensureTotalCapacity(gpa, step_map.entries.capacity); wc.steps.appendAssumeCapacity(.{ .name = try wc.addString(step.name), - .flags = .{ .tag = step.tag }, .deps = deps, .max_rss = .fromBytes(step.max_rss), .extra_index = switch (step.tag) { @@ -273,6 +273,9 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .install_artifact => e: { const ia: *Step.InstallArtifact = @fieldParentPtr("step", step); break :e try wc.addExtra(@as(Configuration.Step.InstallArtifact, .{ + .flags = .{ + .dylib_symlinks = ia.dylib_symlinks != null, + }, .dest_dir = try addInstallDir(wc, ia.dest_dir), .dest_sub_path = try wc.addString(ia.dest_sub_path), .emitted_bin = try addOptionalLazyPath(wc, ia.emitted_bin), @@ -293,7 +296,54 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .translate_c => @panic("TODO"), .write_file => @panic("TODO"), .update_source_files => @panic("TODO"), - .run => @panic("TODO"), + .run => e: { + const run: *Step.Run = @fieldParentPtr("step", step); + + const captured_stdout: Configuration.OptionalString = if (run.captured_stdout) |cs| + .init(try wc.addString(cs.output.basename)) + else + .none; + + const captured_stderr: Configuration.OptionalString = if (run.captured_stderr) |cs| + .init(try wc.addString(cs.output.basename)) + else + .none; + + const extra_index = try wc.addExtra(@as(Configuration.Step.Run, .{ + .flags = .{ + .disable_zig_progress = run.disable_zig_progress, + .skip_foreign_checks = run.skip_foreign_checks, + .failing_to_execute_foreign_is_an_error = run.failing_to_execute_foreign_is_an_error, + .has_side_effects = run.has_side_effects, + .test_runner_mode = run.test_runner_mode, + .color = run.color, + .stdio = switch (run.stdio) { + .infer_from_args => .infer_from_args, + .inherit => .inherit, + .check => .check, + .zig_test => .zig_test, + }, + .stdin = switch (run.stdin) { + .none => .none, + .bytes => .bytes, + .lazy_path => .lazy_path, + }, + .stdout_trim_whitespace = if (run.captured_stdout) |cs| cs.trim_whitespace else .none, + .stderr_trim_whitespace = if (run.captured_stderr) |cs| cs.trim_whitespace else .none, + .stdio_limit = run.stdio_limit != .unlimited, + .producer = run.producer != null, + }, + .file_inputs_len = @intCast(run.file_inputs.items.len), + .args_len = @intCast(run.argv.items.len), + .cwd = try addOptionalLazyPath(wc, run.cwd), + .captured_stdout = captured_stdout, + .captured_stderr = captured_stderr, + })); + + std.log.err("TODO serialize the trailing Run step data", .{}); + + break :e extra_index; + }, .check_file => @panic("TODO"), .check_object => @panic("TODO"), .config_header => @panic("TODO"), @@ -319,7 +369,7 @@ fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configu .src_path => |src_path| i: { const owner = builderToPackage(src_path.owner); const sub_path = try wc.addString(src_path.sub_path); - break :i try wc.addExtra(@as(Configuration.OptionalLazyPath.SourcePath, .{ + break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ .flags = .{}, .owner = owner, .sub_path = sub_path, @@ -327,14 +377,14 @@ fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configu }, .generated => |generated| i: { const sub_path = try wc.addString(generated.sub_path); - break :i try wc.addExtra(@as(Configuration.OptionalLazyPath.Generated, .{ + break :i try wc.addExtra(@as(Configuration.LazyPath.Generated, .{ .flags = .{ .up = @intCast(generated.up) }, .sub_path = sub_path, })); }, .cwd_relative => |cwd_relative_sub_path| i: { const sub_path = try wc.addString(cwd_relative_sub_path); - break :i try wc.addExtra(@as(Configuration.OptionalLazyPath.Relative, .{ + break :i try wc.addExtra(@as(Configuration.LazyPath.Relative, .{ .flags = .{ .base = .cwd }, .sub_path = sub_path, })); @@ -342,7 +392,7 @@ fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configu .dependency => |dependency| i: { const owner = builderToPackage(dependency.dependency.builder); const sub_path = try wc.addString(dependency.sub_path); - break :i try wc.addExtra(@as(Configuration.OptionalLazyPath.SourcePath, .{ + break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ .flags = .{}, .owner = owner, .sub_path = sub_path, diff --git a/lib/compiler/maker/Step/Run.zig b/lib/compiler/maker/Step/Run.zig index 79b4d57b7e6fa772191b434bc90d7432b3442af3..4ba04092e02221fffc55729a64b44aa41a5b9d91 100644 --- a/lib/compiler/maker/Step/Run.zig +++ b/lib/compiler/maker/Step/Run.zig @@ -20,6 +20,9 @@ const Step = @import("../Step.zig"); fuzz_tests: std.ArrayList([]const u8), cached_test_metadata: ?CachedTestMetadata = null, +/// Populated during the fuzz phase if this run step corresponds to a unit test +/// executable that contains fuzz tests. +rebuilt_executable: ?Path, fn make(step: *Step, options: Step.MakeOptions) !void { const b = step.owner; diff --git a/lib/compiler/maker/Watch.zig b/lib/compiler/maker/Watch.zig index ad3e787f854ac54ba255a13906489e2e94e783e0..a38f4ac601755382ba248c6415e78d0d0d53a757 100644 --- a/lib/compiler/maker/Watch.zig +++ b/lib/compiler/maker/Watch.zig @@ -1,12 +1,12 @@ +const Watch = @This(); const builtin = @import("builtin"); -const std = @import("../std.zig"); +const std = @import("std"); const Io = std.Io; const Step = std.Build.Step; const Allocator = std.mem.Allocator; const assert = std.debug.assert; const fatal = std.process.fatal; -const Watch = @This(); const FsEvents = @import("Watch/FsEvents.zig"); os: Os, diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index f1cde9d5336eb0f3a316ee6911b7c1d20d80688f..8254a7aaf397439bf89c97d93f5def2accf6e894 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -89,26 +89,10 @@ dep_output_file: ?*Output, has_side_effects: bool, test_runner_mode: bool = false, -/// Populated during the fuzz phase if this run step corresponds to a unit test -/// executable that contains fuzz tests. -rebuilt_executable: ?Path, - /// If this Run step was produced by a Compile step, it is tracked here. producer: ?*Step.Compile, -pub const Color = enum { - /// `CLICOLOR_FORCE` is set, and `NO_COLOR` is unset. - enable, - /// `NO_COLOR` is set, and `CLICOLOR_FORCE` is unset. - disable, - /// If the build runner is using color, equivalent to `.enable`. Otherwise, equivalent to `.disable`. - inherit, - /// If stderr is captured or checked, equivalent to `.disable`. Otherwise, equivalent to `.inherit`. - auto, - /// The build runner does not modify the `CLICOLOR_FORCE` or `NO_COLOR` environment variables. - /// They are treated like normal variables, so can be controlled through `setEnvironmentVariable`. - manual, -}; +pub const Color = std.Build.Configuration.Step.Run.Color; pub const StdIn = union(enum) { none, @@ -192,12 +176,7 @@ pub const CapturedStdIo = struct { trim_whitespace: TrimWhitespace = .none, }; - pub const TrimWhitespace = enum { - none, - all, - leading, - trailing, - }; + pub const TrimWhitespace = std.Build.Configuration.Step.Run.TrimWhitespace; }; pub fn create(owner: *std.Build, name: []const u8) *Run { @@ -223,7 +202,6 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .captured_stderr = null, .dep_output_file = null, .has_side_effects = false, - .rebuilt_executable = null, .producer = null, }; return run; diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index ae83e63ef4d56e362a9e1f5856ebe8cc2ce781cf..9ee057891018f97b0653c39adeec01ed6ed87c7b 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -196,17 +196,12 @@ pub const Wip = struct { pub const Step = extern struct { name: String, - flags: Flags, deps: Deps, max_rss: MaxRss, - /// Points into `extra` for step-specific data. + /// Points into `extra` for step-specific data. First element has flags + /// with `Tag`. extra_index: u32, - pub const Flags = packed struct(u32) { - tag: Tag, - _: u24 = 0, - }; - pub const Index = enum(u32) { _, }; @@ -232,10 +227,18 @@ pub const Step = extern struct { }; pub const TopLevel = struct { + flags: Flags = .{}, description: String, + + pub const Flags = packed struct(u32) { + tag: Tag = .top_level, + _: u24 = 0, + }; }; pub const InstallArtifact = struct { + flags: Flags, + dest_dir: InstallDir, dest_sub_path: String, emitted_bin: OptionalLazyPath, @@ -252,12 +255,96 @@ pub const Step = extern struct { /// Always a compile step. artifact: Step.Index, - const Flags = packed struct(u32) { + pub const Flags = packed struct(u32) { tag: Tag = .install_artifact, dylib_symlinks: bool, _: u23 = 0, }; }; + + /// Trailing: + /// * LazyPath for each file_inputs_len + /// * Arg for each args_len + /// * environ_map if corresponding flag is set + /// * stdin: Bytes, // if StdIn.bytes is chosen + /// * stdin: LazyPath, // if StdIn.lazy_path is chosen + /// * checks: Checks, // if StdIo.check is chosen + /// * stdio_limit: u64, // if stdio_limit is set + /// * producer: Step.Index, // if producer is set. always compile step + pub const Run = struct { + flags: Flags, + file_inputs_len: u32, + args_len: u32, + cwd: OptionalLazyPath, + captured_stdout: OptionalString, // basename + captured_stderr: OptionalString, // basename + + /// Trailing: + /// * String if prefix set + /// * String if suffix set + /// * String if basename set + /// * Step.Index which is always a compile step if tag is artifact + /// * LazyPath if tag is path_file, path_directory, or file_content + pub const Arg = struct { + flags: Arg.Flags, + + pub const Flags = packed struct(u32) { + tag: Arg.Tag, + prefix: bool, + suffix: bool, + basename: bool, + /// Implies Tag is output_file + dep_file: bool, + _: u20 = 0, + }; + + pub const Tag = enum(u8) { + artifact, + path_file, + path_directory, + file_content, + bytes, + output_file, + output_directory, + }; + }; + + pub const Color = enum(u4) { + /// `CLICOLOR_FORCE` is set, and `NO_COLOR` is unset. + enable, + /// `NO_COLOR` is set, and `CLICOLOR_FORCE` is unset. + disable, + /// If the build runner is using color, equivalent to `.enable`. Otherwise, equivalent to `.disable`. + inherit, + /// If stderr is captured or checked, equivalent to `.disable`. Otherwise, equivalent to `.inherit`. + auto, + /// The build runner does not modify the `CLICOLOR_FORCE` or `NO_COLOR` environment variables. + /// They are treated like normal variables, so can be controlled through `setEnvironmentVariable`. + manual, + }; + + pub const StdIn = enum(u2) { none, bytes, lazy_path }; + pub const TrimWhitespace = enum(u2) { none, all, leading, trailing }; + pub const StdIo = enum(u2) { infer_from_args, inherit, check, zig_test }; + + pub const Flags = packed struct(u32) { + tag: Tag = .run, + + disable_zig_progress: bool, + skip_foreign_checks: bool, + failing_to_execute_foreign_is_an_error: bool, + has_side_effects: bool, + test_runner_mode: bool, + color: Color, + stdin: StdIn, + stdio: StdIo, + stdout_trim_whitespace: TrimWhitespace, + stderr_trim_whitespace: TrimWhitespace, + stdio_limit: bool, + producer: bool, + _: u5 = 0, + }; + }; }; pub const MaxRss = enum(u32) { @@ -274,10 +361,15 @@ pub const MaxRss = enum(u32) { } }; -/// An index into `extra`. +/// An index into `extra`, or `null`. pub const OptionalLazyPath = enum(u32) { none = maxInt(u32), _, +}; + +/// An index into `extra`. +pub const LazyPath = enum(u32) { + _, pub const Tag = enum(u8) { /// A source file path relative to build root. @@ -368,8 +460,22 @@ pub const InstallDir = enum(u32) { } }; +/// Points into `string_bytes`, null-terminated. +pub const OptionalString = enum(u32) { + empty = 0, + none = maxInt(u32), + _, + + pub fn init(s: String) OptionalString { + const result: OptionalString = @enumFromInt(@intFromEnum(s)); + assert(result != .none); + return result; + } +}; + /// Points into `string_bytes`, null-terminated. pub const String = enum(u32) { + empty = 0, _, pub fn slice(index: String, c: *const Configuration) [:0]const u8 { -- 2.54.0 From 648e0e0cc0d6d1940d3bb2ebc265067f7eb62432 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Feb 2026 23:23:17 -0800 Subject: [PATCH 006/179] configure runner: serialization of compile step --- lib/compiler/configure_runner.zig | 106 +++++++++++- lib/compiler/maker/Step/Compile.zig | 130 +++++++++++++- lib/std/Build/Step/Compile.zig | 164 ++---------------- lib/std/zig.zig | 4 +- lib/std/zig/Configuration.zig | 258 +++++++++++++++++++++++++++- 5 files changed, 500 insertions(+), 162 deletions(-) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index a687af87650b584290e0fb521895b74302300a71..64ebf844aece882f9dd545a20c0db2ed12d1e359 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -269,7 +269,105 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .description = try wc.addString(top_level.description), })); }, - .compile => @panic("TODO"), + .compile => e: { + const c: *Step.Compile = @fieldParentPtr("step", step); + const extra_index = try wc.addExtra(@as(Configuration.Step.Compile, .{ + .flags = .{ + .filters_len = c.filters.len != 0, + .exec_cmd_args_len = if (c.exec_cmd_args) |a| a.len != 0 else false, + .installed_headers_len = c.installed_headers.items.len != 0, + .force_undefined_symbols_len = c.force_undefined_symbols.entries.len != 0, + + .verbose_link = c.verbose_link, + .verbose_cc = c.verbose_cc, + .rdynamic = c.rdynamic, + .import_memory = c.import_memory, + .export_memory = c.export_memory, + .import_symbols = c.import_symbols, + .import_table = c.import_table, + .export_table = c.export_table, + .shared_memory = c.shared_memory, + .link_eh_frame_hdr = c.link_eh_frame_hdr, + .link_emit_relocs = c.link_emit_relocs, + .link_function_sections = c.link_function_sections, + .link_data_sections = c.link_data_sections, + .linker_dynamicbase = c.linker_dynamicbase, + .link_z_notext = c.link_z_notext, + .link_z_relro = c.link_z_relro, + .link_z_lazy = c.link_z_lazy, + .link_z_defs = c.link_z_defs, + .headerpad_max_install_names = c.headerpad_max_install_names, + .dead_strip_dylibs = c.dead_strip_dylibs, + .force_load_objc = c.force_load_objc, + .discard_local_symbols = c.discard_local_symbols, + .mingw_unicode_entry_point = c.mingw_unicode_entry_point, + }, + .flags2 = .{ + .pie = .init(c.pie), + .formatted_panics = .init(c.formatted_panics), + .bundle_compiler_rt = .init(c.bundle_compiler_rt), + .bundle_ubsan_rt = .init(c.bundle_ubsan_rt), + .each_lib_rpath = .init(c.each_lib_rpath), + .link_gc_sections = .init(c.link_gc_sections), + .linker_allow_shlib_undefined = .init(c.linker_allow_shlib_undefined), + .linker_allow_undefined_version = .init(c.linker_allow_undefined_version), + .linker_enable_new_dtags = .init(c.linker_enable_new_dtags), + .dll_export_fns = .init(c.dll_export_fns), + .use_llvm = .init(c.use_llvm), + .use_lld = .init(c.use_lld), + .use_new_linker = .init(c.use_new_linker), + .allow_so_scripts = .init(c.allow_so_scripts), + .sanitize_coverage_trace_pc_guard = .init(c.sanitize_coverage_trace_pc_guard), + .linkage = .init(c.linkage), + }, + .flags3 = .{ + .is_linking_libc = c.is_linking_libc, + .is_linking_libcpp = c.is_linking_libcpp, + .version = c.version != null, + .compress_debug_sections = c.compress_debug_sections, + .initial_memory = c.initial_memory != null, + .max_memory = c.max_memory != null, + .kind = c.kind, + .global_base = c.global_base != null, + .test_runner_mode = if (c.test_runner) |tr| switch (tr.mode) { + .simple => .simple, + .server => .server, + } else .default, + .wasi_exec_model = .init(c.wasi_exec_model), + .win32_manifest = c.win32_manifest != null, + .win32_module_definition = c.win32_module_definition != null, + .zig_lib_dir = c.zig_lib_dir != null, + .rc_includes = c.rc_includes, + .image_base = c.image_base != null, + .build_id = .init(c.build_id), + .entry = switch (c.entry) { + .default => .default, + .disabled => .disabled, + .enabled => .enabled, + .symbol_name => .symbol_name, + }, + .lto = .init(c.lto), + .subsystem = .init(c.subsystem), + }, + .flags4 = .{ + .libc_file = c.libc_file != null, + .link_z_common_page_size = c.link_z_common_page_size != null, + .link_z_max_page_size = c.link_z_max_page_size != null, + .pagezero_size = c.pagezero_size != null, + .stack_size = c.stack_size != null, + .headerpad_size = c.headerpad_size != null, + .error_limit = c.error_limit != null, + .install_name = c.install_name != null, + .entitlements = c.entitlements != null, + }, + .root_module = try addModule(wc, c.root_module), + .root_name = try wc.addString(c.name), + })); + + std.log.err("TODO serialize the trailing Compile step data", .{}); + + break :e extra_index; + }, .install_artifact => e: { const ia: *Step.InstallArtifact = @fieldParentPtr("step", step); break :e try wc.addExtra(@as(Configuration.Step.InstallArtifact, .{ @@ -364,6 +462,12 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }); } +fn addModule(wc: *Configuration.Wip, module: *std.Build.Module) !Configuration.Module { + _ = wc; + _ = module; + @panic("TODO"); +} + fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configuration.OptionalLazyPath { return @enumFromInt(switch (lp orelse return .none) { .src_path => |src_path| i: { diff --git a/lib/compiler/maker/Step/Compile.zig b/lib/compiler/maker/Step/Compile.zig index e3727025e25d189080e18cce93e48b13e0a226d1..5ffbc23cfc15d6d6de5df4f4585585364052da6d 100644 --- a/lib/compiler/maker/Step/Compile.zig +++ b/lib/compiler/maker/Step/Compile.zig @@ -98,8 +98,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { } { - var symbol_it = compile.force_undefined_symbols.keyIterator(); - while (symbol_it.next()) |symbol_name| { + for (compile.force_undefined_symbols.keys()) |symbol_name| { try zig_args.append("--force_undefined"); try zig_args.append(symbol_name.*); } @@ -1071,4 +1070,131 @@ fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult { }; } +fn checkCompileErrors(compile: *Compile) !void { + // Clear this field so that it does not get printed by the build runner. + const actual_eb = compile.step.result_error_bundle; + compile.step.result_error_bundle = .empty; + + const arena = compile.step.owner.allocator; + + const actual_errors = ae: { + var aw: std.Io.Writer.Allocating = .init(arena); + defer aw.deinit(); + try actual_eb.renderToWriter(.{ + .include_reference_trace = false, + .include_source_line = false, + }, &aw.writer); + break :ae try aw.toOwnedSlice(); + }; + + // Render the expected lines into a string that we can compare verbatim. + var expected_generated: std.ArrayList(u8) = .empty; + const expect_errors = compile.expect_errors.?; + + var actual_line_it = mem.splitScalar(u8, actual_errors, '\n'); + + // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile + switch (expect_errors) { + .starts_with => |expect_starts_with| { + if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return; + return compile.step.fail( + \\ + \\========= should start with: ============ + \\{s} + \\========= but not found: ================ + \\{s} + \\========================================= + , .{ expect_starts_with, actual_errors }); + }, + .contains => |expect_line| { + while (actual_line_it.next()) |actual_line| { + if (!matchCompileError(actual_line, expect_line)) continue; + return; + } + + return compile.step.fail( + \\ + \\========= should contain: =============== + \\{s} + \\========= but not found: ================ + \\{s} + \\========================================= + , .{ expect_line, actual_errors }); + }, + .stderr_contains => |expect_line| { + const actual_stderr: []const u8 = if (compile.step.result_error_msgs.items.len > 0) + compile.step.result_error_msgs.items[0] + else + &.{}; + compile.step.result_error_msgs.clearRetainingCapacity(); + + var stderr_line_it = mem.splitScalar(u8, actual_stderr, '\n'); + + while (stderr_line_it.next()) |actual_line| { + if (!matchCompileError(actual_line, expect_line)) continue; + return; + } + + return compile.step.fail( + \\ + \\========= should contain: =============== + \\{s} + \\========= but not found: ================ + \\{s} + \\========================================= + , .{ expect_line, actual_stderr }); + }, + .exact => |expect_lines| { + for (expect_lines) |expect_line| { + const actual_line = actual_line_it.next() orelse { + try expected_generated.appendSlice(arena, expect_line); + try expected_generated.append(arena, '\n'); + continue; + }; + if (matchCompileError(actual_line, expect_line)) { + try expected_generated.appendSlice(arena, actual_line); + try expected_generated.append(arena, '\n'); + continue; + } + try expected_generated.appendSlice(arena, expect_line); + try expected_generated.append(arena, '\n'); + } + + if (mem.eql(u8, expected_generated.items, actual_errors)) return; + + return compile.step.fail( + \\ + \\========= expected: ===================== + \\{s} + \\========= but found: ==================== + \\{s} + \\========================================= + , .{ expected_generated.items, actual_errors }); + }, + } +} + +fn matchCompileError(actual: []const u8, expected: []const u8) bool { + if (mem.endsWith(u8, actual, expected)) return true; + if (mem.startsWith(u8, expected, ":?:?: ")) { + if (mem.endsWith(u8, actual, expected[":?:?: ".len..])) return true; + } + // We scan for /?/ in expected line and if there is a match, we match everything + // up to and after /?/. + const expected_trim = mem.trim(u8, expected, " "); + if (mem.find(u8, expected_trim, "/?/")) |index| { + const actual_trim = mem.trim(u8, actual, " "); + const lhs = expected_trim[0..index]; + const rhs = expected_trim[index + "/?/".len ..]; + if (mem.startsWith(u8, actual_trim, lhs) and mem.endsWith(u8, actual_trim, rhs)) return true; + } + return false; +} + +fn moduleNeedsCliArg(mod: *const Module) bool { + return for (mod.link_objects.items) |o| switch (o) { + .c_source_file, .c_source_files, .assembly_file, .win32_resource_file => break true, + else => continue, + } else false; +} diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 2897fddb747f199a72381070dfbfd28787e41734..8b580f3a5da9a11903540524f7b98a445b2d9573 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -60,7 +60,7 @@ filters: []const []const u8, test_runner: ?TestRunner, wasi_exec_model: ?std.builtin.WasiExecModel = null, -installed_headers: std.array_list.Managed(HeaderInstallation), +installed_headers: std.ArrayList(HeaderInstallation), /// This step is used to create an include tree that dependent modules can add to their include /// search paths. Installed headers are copied to this step. @@ -83,8 +83,6 @@ win32_manifest: ?LazyPath = null, /// Set via options; intended to be read-only after that. win32_module_definition: ?LazyPath = null, -installed_path: ?[]const u8, - /// Base address for an executable image. image_base: ?u64 = null, @@ -189,7 +187,7 @@ entry: Entry = .default, /// List of symbols forced as undefined in the symbol table /// thus forcing their resolution by the linker. /// Corresponds to `-u ` for ELF/MachO and `/include:` for COFF/PE. -force_undefined_symbols: std.StringHashMap(void), +force_undefined_symbols: std.StringArrayHashMapUnmanaged(void), /// Overrides the default stack size stack_size: ?u64 = null, @@ -290,20 +288,7 @@ pub const Options = struct { win32_module_definition: ?LazyPath = null, }; -pub const Kind = enum { - exe, - lib, - obj, - @"test", - test_obj, - - pub fn isTest(kind: Kind) bool { - return switch (kind) { - .exe, .lib, .obj => false, - .@"test", .test_obj => true, - }; - } -}; +pub const Kind = std.Build.Configuration.Step.Compile.Kind; pub const HeaderInstallation = union(enum) { file: File, @@ -424,14 +409,13 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .out_lib_filename = undefined, .major_only_filename = null, .name_only_filename = null, - .installed_headers = std.array_list.Managed(HeaderInstallation).init(owner.allocator), + .installed_headers = .empty, .zig_lib_dir = null, .exec_cmd_args = null, .filters = options.filters, .test_runner = null, // set below .rdynamic = false, - .installed_path = null, - .force_undefined_symbols = StringHashMap(void).init(owner.allocator), + .force_undefined_symbols = .empty, .emit_directory = null, .generated_docs = null, @@ -519,7 +503,7 @@ pub fn installHeader(cs: *Compile, source: LazyPath, dest_rel_path: []const u8) .source = source.dupe(b), .dest_rel_path = b.dupePath(dest_rel_path), } }; - cs.installed_headers.append(installation) catch @panic("OOM"); + cs.installed_headers.append(b.allocator, installation) catch @panic("OOM"); cs.addHeaderInstallationToIncludeTree(installation); installation.getSource().addStepDependencies(&cs.step); } @@ -539,7 +523,7 @@ pub fn installHeadersDirectory( .dest_rel_path = b.dupePath(dest_rel_path), .options = options.dupe(b), } }; - cs.installed_headers.append(installation) catch @panic("OOM"); + cs.installed_headers.append(b.allocator, installation) catch @panic("OOM"); cs.addHeaderInstallationToIncludeTree(installation); installation.getSource().addStepDependencies(&cs.step); } @@ -556,9 +540,10 @@ pub fn installConfigHeader(cs: *Compile, config_header: *Step.ConfigHeader) void /// module's include search path. pub fn installLibraryHeaders(cs: *Compile, lib: *Compile) void { assert(lib.kind == .lib); + const arena = cs.owner.allocator; for (lib.installed_headers.items) |installation| { const installation_copy = installation.dupe(lib.step.owner); - cs.installed_headers.append(installation_copy) catch @panic("OOM"); + cs.installed_headers.append(arena, installation_copy) catch @panic("OOM"); cs.addHeaderInstallationToIncludeTree(installation_copy); installation_copy.getSource().addStepDependencies(&cs.step); } @@ -618,7 +603,8 @@ pub fn setVersionScript(compile: *Compile, source: LazyPath) void { pub fn forceUndefinedSymbol(compile: *Compile, symbol_name: []const u8) void { const b = compile.step.owner; - compile.force_undefined_symbols.put(b.dupe(symbol_name), {}) catch @panic("OOM"); + const arena = b.allocator; + compile.force_undefined_symbols.put(arena, b.dupe(symbol_name), {}) catch @panic("OOM"); } /// Returns whether the library, executable, or object depends on a particular system library. @@ -867,139 +853,11 @@ fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifa return out_dir.joinString(arena, name) catch @panic("OOM"); } -fn checkCompileErrors(compile: *Compile) !void { - // Clear this field so that it does not get printed by the build runner. - const actual_eb = compile.step.result_error_bundle; - compile.step.result_error_bundle = .empty; - - const arena = compile.step.owner.allocator; - - const actual_errors = ae: { - var aw: std.Io.Writer.Allocating = .init(arena); - defer aw.deinit(); - try actual_eb.renderToWriter(.{ - .include_reference_trace = false, - .include_source_line = false, - }, &aw.writer); - break :ae try aw.toOwnedSlice(); - }; - - // Render the expected lines into a string that we can compare verbatim. - var expected_generated: std.ArrayList(u8) = .empty; - const expect_errors = compile.expect_errors.?; - - var actual_line_it = mem.splitScalar(u8, actual_errors, '\n'); - - // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile - switch (expect_errors) { - .starts_with => |expect_starts_with| { - if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return; - return compile.step.fail( - \\ - \\========= should start with: ============ - \\{s} - \\========= but not found: ================ - \\{s} - \\========================================= - , .{ expect_starts_with, actual_errors }); - }, - .contains => |expect_line| { - while (actual_line_it.next()) |actual_line| { - if (!matchCompileError(actual_line, expect_line)) continue; - return; - } - - return compile.step.fail( - \\ - \\========= should contain: =============== - \\{s} - \\========= but not found: ================ - \\{s} - \\========================================= - , .{ expect_line, actual_errors }); - }, - .stderr_contains => |expect_line| { - const actual_stderr: []const u8 = if (compile.step.result_error_msgs.items.len > 0) - compile.step.result_error_msgs.items[0] - else - &.{}; - compile.step.result_error_msgs.clearRetainingCapacity(); - - var stderr_line_it = mem.splitScalar(u8, actual_stderr, '\n'); - - while (stderr_line_it.next()) |actual_line| { - if (!matchCompileError(actual_line, expect_line)) continue; - return; - } - - return compile.step.fail( - \\ - \\========= should contain: =============== - \\{s} - \\========= but not found: ================ - \\{s} - \\========================================= - , .{ expect_line, actual_stderr }); - }, - .exact => |expect_lines| { - for (expect_lines) |expect_line| { - const actual_line = actual_line_it.next() orelse { - try expected_generated.appendSlice(arena, expect_line); - try expected_generated.append(arena, '\n'); - continue; - }; - if (matchCompileError(actual_line, expect_line)) { - try expected_generated.appendSlice(arena, actual_line); - try expected_generated.append(arena, '\n'); - continue; - } - try expected_generated.appendSlice(arena, expect_line); - try expected_generated.append(arena, '\n'); - } - - if (mem.eql(u8, expected_generated.items, actual_errors)) return; - - return compile.step.fail( - \\ - \\========= expected: ===================== - \\{s} - \\========= but found: ==================== - \\{s} - \\========================================= - , .{ expected_generated.items, actual_errors }); - }, - } -} - -fn matchCompileError(actual: []const u8, expected: []const u8) bool { - if (mem.endsWith(u8, actual, expected)) return true; - if (mem.startsWith(u8, expected, ":?:?: ")) { - if (mem.endsWith(u8, actual, expected[":?:?: ".len..])) return true; - } - // We scan for /?/ in expected line and if there is a match, we match everything - // up to and after /?/. - const expected_trim = mem.trim(u8, expected, " "); - if (mem.find(u8, expected_trim, "/?/")) |index| { - const actual_trim = mem.trim(u8, actual, " "); - const lhs = expected_trim[0..index]; - const rhs = expected_trim[index + "/?/".len ..]; - if (mem.startsWith(u8, actual_trim, lhs) and mem.endsWith(u8, actual_trim, rhs)) return true; - } - return false; -} - pub fn rootModuleTarget(c: *Compile) std.Target { // The root module is always given a target, so we know this to be non-null. return c.root_module.resolved_target.?.result; } -fn moduleNeedsCliArg(mod: *const Module) bool { - return for (mod.link_objects.items) |o| switch (o) { - .c_source_file, .c_source_files, .assembly_file, .win32_resource_file => break true, - else => continue, - } else false; -} - /// Return the full set of `Step.Compile` which `start` depends on, recursively. `start` itself is /// always returned as the first element. If `chase_dynamic` is `false`, then dynamic libraries are /// not included, and their dependencies are not considered; if `chase_dynamic` is `true`, dynamic diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 75ef9c9b63195335b78c5cfa04d4d75742532905..5fdcd67de4c87b4d93f89ffb0e5fe5af99f817d0 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -374,9 +374,9 @@ pub const Subsystem = enum { pub const EfiRuntimeDriver: Subsystem = .efi_runtime_driver; }; -pub const CompressDebugSections = enum { none, zlib, zstd }; +pub const CompressDebugSections = enum(u2) { none, zlib, zstd }; -pub const RcIncludes = enum { +pub const RcIncludes = enum(u2) { /// Use MSVC if available, fall back to MinGW. any, /// Use MSVC include paths (MSVC install + Windows SDK, must be present on the system). diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 9ee057891018f97b0653c39adeec01ed6ed87c7b..9009156191ee32aa1cbc22a793fe69797f1e7d33 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -206,7 +206,7 @@ pub const Step = extern struct { _, }; - pub const Tag = enum(u8) { + pub const Tag = enum(u5) { top_level, compile, install_artifact, @@ -232,7 +232,7 @@ pub const Step = extern struct { pub const Flags = packed struct(u32) { tag: Tag = .top_level, - _: u24 = 0, + _: u27 = 0, }; }; @@ -258,7 +258,7 @@ pub const Step = extern struct { pub const Flags = packed struct(u32) { tag: Tag = .install_artifact, dylib_symlinks: bool, - _: u23 = 0, + _: u26 = 0, }; }; @@ -342,7 +342,253 @@ pub const Step = extern struct { stderr_trim_whitespace: TrimWhitespace, stdio_limit: bool, producer: bool, - _: u5 = 0, + _: u8 = 0, + }; + }; + + /// Trailing: + /// * filters_len: u32, // if flag is set + /// * exec_cmd_args_len: u32, // if flag is set + /// * installed_headers_len: u32, // if flag is set + /// * force_undefined_symbols_len: u32, // if flag is set + /// * exacts_len: u32 if expected_compile_errors is exact + /// * filter: String for each filters_len + /// * exec_cmd_arg: String for each exec_cmd_args_len + /// * InstalledHeader for each installed_headers_len + /// * force_undefined_symbol: String for each force_undefined_symbols_len + /// * String for each exacts_len + /// * linker_script: LazyPath if flag is set + /// * version_script: LazyPath if flag is set + /// * zig_lib_dir: LazyPath if flag is set + /// * libc_file: LazyPath if flag is set + /// * test_runner: LazyPath if test_runner_mode is not default + /// * win32_manifest: LazyPath if flag is set + /// * win32_module_definition: LazyPath if flag is set + /// * entitlements: LazyPath if flag is set + /// * version: String if flag is set (semantic version string) + /// * entry: String if entry is symbol + /// * install_name: String if flag is set + /// * String if expected_compile_errors is contains, starts_with, or stderr_contains + /// * initial_memory: u64 if flag is set + /// * max_memory: u64 if flag is set + /// * global_base: u64 if flag is set + /// * image_base: u64 if flag is set + /// * link_z_common_page_size if flag is set + /// * link_z_max_page_size if flag is set + /// * pagezero_size if flag is set + /// * stack_size if flag is set + /// * headerpad_size if flag is set + /// * error_limit if flag is set + /// * Hexstring if build_id is hexstring + pub const Compile = struct { + flags: Flags, + flags2: Flags2, + flags3: Flags3, + flags4: Flags4, + + root_module: Module, + root_name: String, + + pub const ExpectedCompileErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; + pub const TestRunnerMode = enum(u2) { default, simple, server }; + pub const Entry = enum(u2) { default, disabled, enabled, symbol_name }; + + pub const Lto = enum(u2) { + none, + full, + thin, + default, + + pub fn init(lto: ?std.zig.LtoMode) Lto { + return switch (lto orelse return .default) { + .none => .none, + .full => .full, + .thin => .thin, + }; + } + }; + + pub const BuildId = enum(u3) { + none, + fast, + uuid, + sha1, + md5, + hexstring, + default, + + pub fn init(build_id: ?std.zig.BuildId) BuildId { + return switch (build_id orelse return .default) { + .none => .none, + .fast => .fast, + .uuid => .uuid, + .sha1 => .sha1, + .md5 => .md5, + .hexstring => .hexstring, + }; + } + }; + pub const WasiExecModel = enum(u2) { + default, + command, + reactor, + + pub fn init(wasi_exec_model: ?std.builtin.WasiExecModel) WasiExecModel { + return switch (wasi_exec_model orelse return .default) { + .command => .command, + .reactor => .reactor, + }; + } + }; + pub const Linkage = enum(u2) { + static, + dynamic, + default, + + pub fn init(link_mode: ?std.builtin.LinkMode) Linkage { + return switch (link_mode orelse return .default) { + .static => .static, + .dynamic => .dynamic, + }; + } + }; + pub const Kind = enum(u3) { + exe, + lib, + obj, + @"test", + test_obj, + + pub fn isTest(kind: Kind) bool { + return switch (kind) { + .exe, .lib, .obj => false, + .@"test", .test_obj => true, + }; + } + }; + pub const DefaultingBool = enum(u2) { + false, + true, + default, + + pub fn init(b: ?bool) DefaultingBool { + return switch (b orelse return .default) { + false => .false, + true => .true, + }; + } + }; + + pub const Subsystem = enum(u4) { + console, + windows, + posix, + native, + efi_application, + efi_boot_service_driver, + efi_rom, + efi_runtime_driver, + default, + + pub fn init(subsystem: ?std.zig.Subsystem) Subsystem { + return switch (subsystem orelse return .default) { + .console => .console, + .windows => .windows, + .posix => .posix, + .native => .native, + .efi_application => .efi_application, + .efi_boot_service_driver => .efi_boot_service_driver, + .efi_rom => .efi_rom, + .efi_runtime_driver => .efi_runtime_driver, + }; + } + }; + + pub const Flags = packed struct(u32) { + tag: Tag = .compile, + + filters_len: bool, + exec_cmd_args_len: bool, + installed_headers_len: bool, + force_undefined_symbols_len: bool, + + verbose_link: bool, + verbose_cc: bool, + rdynamic: bool, + import_memory: bool, + export_memory: bool, + import_symbols: bool, + import_table: bool, + export_table: bool, + shared_memory: bool, + link_eh_frame_hdr: bool, + link_emit_relocs: bool, + link_function_sections: bool, + link_data_sections: bool, + linker_dynamicbase: bool, + link_z_notext: bool, + link_z_relro: bool, + link_z_lazy: bool, + link_z_defs: bool, + headerpad_max_install_names: bool, + dead_strip_dylibs: bool, + force_load_objc: bool, + discard_local_symbols: bool, + mingw_unicode_entry_point: bool, + }; + + pub const Flags2 = packed struct(u32) { + pie: DefaultingBool, + formatted_panics: DefaultingBool, + bundle_compiler_rt: DefaultingBool, + bundle_ubsan_rt: DefaultingBool, + each_lib_rpath: DefaultingBool, + link_gc_sections: DefaultingBool, + linker_allow_shlib_undefined: DefaultingBool, + linker_allow_undefined_version: DefaultingBool, + linker_enable_new_dtags: DefaultingBool, + dll_export_fns: DefaultingBool, + use_llvm: DefaultingBool, + use_lld: DefaultingBool, + use_new_linker: DefaultingBool, + allow_so_scripts: DefaultingBool, + sanitize_coverage_trace_pc_guard: DefaultingBool, + linkage: Linkage, + }; + + pub const Flags3 = packed struct(u32) { + is_linking_libc: bool, + is_linking_libcpp: bool, + version: bool, + initial_memory: bool, + max_memory: bool, + kind: Kind, + compress_debug_sections: std.zig.CompressDebugSections, + global_base: bool, + test_runner_mode: TestRunnerMode, + wasi_exec_model: WasiExecModel, + win32_manifest: bool, + win32_module_definition: bool, + zig_lib_dir: bool, + rc_includes: std.zig.RcIncludes, + image_base: bool, + build_id: BuildId, + entry: Entry, + lto: Lto, + subsystem: Subsystem, + }; + + pub const Flags4 = packed struct(u32) { + libc_file: bool, + link_z_common_page_size: bool, + link_z_max_page_size: bool, + pagezero_size: bool, + stack_size: bool, + headerpad_size: bool, + error_limit: bool, + install_name: bool, + entitlements: bool, + _: u23 = 0, }; }; }; @@ -420,6 +666,10 @@ pub const Package = enum(u32) { _, }; +pub const Module = enum(u32) { + _, +}; + /// Points into `extra`, where the first element is number of deps, /// following elements is `Step.Index` per dep. pub const Deps = enum(u32) { -- 2.54.0 From 3e6bebbbca2acdcde7578c12e618cbcea233dc49 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 16 Feb 2026 16:02:31 -0800 Subject: [PATCH 007/179] configure runner: serialization of Module --- lib/compiler/configure_runner.zig | 99 +++- lib/std/Build/Module.zig | 15 +- lib/std/lang.zig | 2 +- lib/std/zig/Configuration.zig | 744 +++++++++++++++++++++++++++++- 4 files changed, 824 insertions(+), 36 deletions(-) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 64ebf844aece882f9dd545a20c0db2ed12d1e359..b61d2b57fabc8c3762fb2b68b130b0b6650f7734 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -227,6 +227,9 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { const arena = graph.arena; const gpa = wc.gpa; + var module_map: std.AutoArrayHashMapUnmanaged(*std.Build.Module, Configuration.Module.Index) = .empty; + defer module_map.deinit(gpa); + // Starting from all top-level steps in `b`, traverse the entire step graph // and add all step dependencies implied by module graphs. const top_level_steps = b.top_level_steps.values(); @@ -360,7 +363,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .install_name = c.install_name != null, .entitlements = c.entitlements != null, }, - .root_module = try addModule(wc, c.root_module), + .root_module = try addModule(wc, &module_map, c.root_module), .root_name = try wc.addString(c.name), })); @@ -462,20 +465,97 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }); } -fn addModule(wc: *Configuration.Wip, module: *std.Build.Module) !Configuration.Module { - _ = wc; - _ = module; - @panic("TODO"); +fn addModule( + wc: *Configuration.Wip, + module_map: *std.AutoArrayHashMapUnmanaged(*std.Build.Module, Configuration.Module.Index), + m: *std.Build.Module, +) !Configuration.Module.Index { + if (module_map.get(m)) |index| return index; + + const gpa = wc.gpa; + const import_table: Configuration.ImportTable = @enumFromInt(wc.extra.items.len); + const import_table_extra_len = 1 + 2 * m.import_table.entries.len; + try wc.extra.ensureUnusedCapacity(gpa, import_table_extra_len); + wc.extra.items.len += import_table_extra_len; + wc.extra.appendAssumeCapacity(@intCast(m.import_table.entries.len)); + wc.extra.items[@intFromEnum(import_table)] = @intCast(m.import_table.entries.len); + for ( + m.import_table.keys(), + @intFromEnum(import_table) + 1.., + ) |mod_name, extra_index| { + wc.extra.items[extra_index] = @intFromEnum(try wc.addString(mod_name)); + } + for ( + m.import_table.values(), + @intFromEnum(import_table) + 1 + m.import_table.entries.len.., + ) |dep, extra_index| { + // TODO module dependencies can be cyclic + wc.extra.items[extra_index] = @intFromEnum(try addModule(wc, module_map, dep)); + } + + const module_index: Configuration.Module.Index = @enumFromInt(try wc.addExtra(@as(Configuration.Module, .{ + .flags = .{ + .optimize = .init(m.optimize), + .strip = .init(m.strip), + .unwind_tables = .init(m.unwind_tables), + .dwarf_format = .init(m.dwarf_format), + .single_threaded = .init(m.strip), + .stack_protector = .init(m.strip), + .stack_check = .init(m.strip), + .sanitize_c = .init(m.sanitize_c), + .sanitize_thread = .init(m.strip), + .fuzz = .init(m.strip), + .code_model = m.code_model, + .c_macros = m.c_macros.items.len != 0, + .include_dirs = m.include_dirs.items.len != 0, + .lib_paths = m.lib_paths.items.len != 0, + .rpaths = m.rpaths.items.len != 0, + .frameworks = m.frameworks.entries.len != 0, + .link_objects = m.link_objects.items.len != 0, + .export_symbol_names = m.export_symbol_names.len != 0, + }, + .flags2 = .{ + .valgrind = .init(m.strip), + .pic = .init(m.strip), + .red_zone = .init(m.strip), + .omit_frame_pointer = .init(m.strip), + .error_tracing = .init(m.strip), + .link_libc = .init(m.strip), + .link_libcpp = .init(m.strip), + .no_builtin = .init(m.strip), + }, + .owner = builderToPackage(m.owner), + .root_source_file = try addOptionalLazyPath(wc, m.root_source_file), + .import_table = import_table, + .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target), + }))); + + std.log.err("TODO serialize the trailing Module data", .{}); + + try module_map.putNoClobber(gpa, m, module_index); + + return module_index; +} + +fn addOptionalResolvedTarget( + wc: *Configuration.Wip, + optional_resolved_target: ?std.Build.ResolvedTarget, +) !Configuration.ResolvedTarget.OptionalIndex { + const resolved_target = optional_resolved_target orelse return .none; + // TODO dedupe + return @enumFromInt(try wc.addExtra(@as(Configuration.ResolvedTarget, .{ + .query = try wc.addTargetQuery(resolved_target.query), + .result = try wc.addTarget(resolved_target.result), + }))); } fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configuration.OptionalLazyPath { return @enumFromInt(switch (lp orelse return .none) { .src_path => |src_path| i: { - const owner = builderToPackage(src_path.owner); const sub_path = try wc.addString(src_path.sub_path); break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ .flags = .{}, - .owner = owner, + .owner = builderToPackage(src_path.owner), .sub_path = sub_path, })); }, @@ -494,18 +574,17 @@ fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configu })); }, .dependency => |dependency| i: { - const owner = builderToPackage(dependency.dependency.builder); const sub_path = try wc.addString(dependency.sub_path); break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ .flags = .{}, - .owner = owner, + .owner = builderToPackage(dependency.dependency.builder), .sub_path = sub_path, })); }, }); } -fn builderToPackage(b: *std.Build) Configuration.Package { +fn builderToPackage(b: *std.Build) Configuration.Package.Index { _ = b; @panic("TODO"); } diff --git a/lib/std/Build/Module.zig b/lib/std/Build/Module.zig index 077c255a504340c3c1ba5a1b04c166074611a3af..7b42c9a1100334ca4fcfbd28a3da5db1d772b591 100644 --- a/lib/std/Build/Module.zig +++ b/lib/std/Build/Module.zig @@ -1,3 +1,11 @@ +const Module = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const LazyPath = std.Build.LazyPath; +const Step = std.Build.Step; +const ArrayList = std.ArrayList; + /// The one responsible for creating this module. owner: *std.Build, root_source_file: ?LazyPath, @@ -703,10 +711,3 @@ pub fn getGraph(root: *Module) Graph { root.cached_graph = result; return result; } - -const Module = @This(); -const std = @import("std"); -const assert = std.debug.assert; -const LazyPath = std.Build.LazyPath; -const Step = std.Build.Step; -const ArrayList = std.ArrayList; diff --git a/lib/std/lang.zig b/lib/std/lang.zig index 0275de552a8144e513ef809c92a5f74ee96bdc6d..b515a5180d62c534ce37f643608e2a93dfb74358 100644 --- a/lib/std/lang.zig +++ b/lib/std/lang.zig @@ -93,7 +93,7 @@ pub const AtomicRmwOp = enum { /// /// This data structure is used by the Zig language code generation and /// therefore must be kept in sync with the compiler implementation. -pub const CodeModel = enum { +pub const CodeModel = enum(u4) { default, extreme, kernel, diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 9009156191ee32aa1cbc22a793fe69797f1e7d33..61ac838b12c3bc3c955f8ac2e741b914b6e7ceee 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -29,6 +29,7 @@ pub const Wip = struct { gpa: Allocator, string_table: StringTable = .empty, deps_table: DepsTable = .empty, + targets_table: TargetsTable = .empty, string_bytes: std.ArrayList(u8) = .empty, unlazy_deps: std.ArrayList(String) = .empty, @@ -37,6 +38,7 @@ pub const Wip = struct { extra: std.ArrayList(u32) = .empty, const DepsTable = std.HashMapUnmanaged(Deps, void, DepsTableContext, std.hash_map.default_max_load_percentage); + const TargetsTable = std.HashMapUnmanaged(TargetQuery.Index, void, TargetsTableContext, std.hash_map.default_max_load_percentage); const DepsTableContext = struct { extra: []const u32, @@ -56,6 +58,21 @@ pub const Wip = struct { } }; + const TargetsTableContext = struct { + extra: []const u32, + + pub fn eql(ctx: @This(), a: TargetQuery.Index, b: TargetQuery.Index) bool { + const slice_a = a.extraSlice(ctx.extra); + const slice_b = b.extraSlice(ctx.extra); + return std.mem.eql(u32, slice_a, slice_b); + } + + pub fn hash(ctx: @This(), key: TargetQuery.Index) u64 { + const slice = key.extraSlice(ctx.extra); + return std.hash_map.hashString(@ptrCast(slice)); + } + }; + const StringTable = std.HashMapUnmanaged(String, void, StringTableContext, std.hash_map.default_max_load_percentage); const StringTableContext = struct { bytes: []const u8, @@ -144,6 +161,157 @@ pub const Wip = struct { return new_off; } + pub fn addSemVer(wip: *Wip, sv: std.SemanticVersion) Allocator.Error!String { + var buffer: [256]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buffer); + sv.format(&writer) catch return error.OutOfMemory; + return addString(wip, writer.buffered()); + } + + pub fn addTargetQuery(wip: *Wip, q: std.Target.Query) !TargetQuery.OptionalIndex { + if (q.isNative()) return .none; + const gpa = wip.gpa; + const cpu_name: ?String = switch (q.cpu_model) { + .native, .baseline, .determined_by_arch_os => null, + .explicit => |model| try wip.addString(model.name), + }; + const os_version_min: ?u32 = if (q.os_version_min) |ver| switch (ver) { + .none => null, + .semver => |sem_ver| @intFromEnum(try wip.addSemVer(sem_ver)), + .windows => |win_ver| @intFromEnum(win_ver), + } else null; + const os_version_max: ?u32 = if (q.os_version_max) |ver| switch (ver) { + .none => null, + .semver => |sem_ver| @intFromEnum(try wip.addSemVer(sem_ver)), + .windows => |win_ver| @intFromEnum(win_ver), + } else null; + const glibc_version: ?String = if (q.glibc_version) |sem_ver| try wip.addSemVer(sem_ver) else null; + const dynamic_linker: ?String = if (q.dynamic_linker) |*dl| + if (dl.get()) |s| try wip.addString(s) else .empty + else + null; + const cpu_features_add_empty = q.cpu_features_add.isEmpty(); + const cpu_features_sub_empty = q.cpu_features_sub.isEmpty(); + try wip.extra.ensureUnusedCapacity(gpa, @typeInfo(TargetQuery).@"struct".fields.len + 6 + + 2 * ((@sizeOf(std.Target.Cpu.Feature.Set) + 3) / 4)); + const result_index: TargetQuery.Index = @enumFromInt(wip.addExtraAssumeCapacity(@as(TargetQuery, .{ + .flags = .{ + .cpu_arch = .init(q.cpu_arch), + .cpu_model = .init(q.cpu_model), + .cpu_features_add = !cpu_features_add_empty, + .cpu_features_sub = !cpu_features_sub_empty, + .os_tag = .init(q.os_tag), + .abi = .init(q.abi), + .object_format = .init(q.ofmt), + .os_version_min = .init(q.os_version_min), + .os_version_max = .init(q.os_version_max), + .glibc_version = q.glibc_version != null, + .android_api_level = q.android_api_level != null, + .dynamic_linker = q.dynamic_linker != null, + }, + }))); + if (!cpu_features_add_empty) wip.extra.appendSliceAssumeCapacity(@ptrCast(&q.cpu_features_add.ints)); + if (!cpu_features_sub_empty) wip.extra.appendSliceAssumeCapacity(@ptrCast(&q.cpu_features_sub.ints)); + wip.addExtraOptionalStringAssumeCapacity(cpu_name); + if (os_version_min) |v| wip.extra.appendAssumeCapacity(v); + if (os_version_max) |v| wip.extra.appendAssumeCapacity(v); + wip.addExtraOptionalStringAssumeCapacity(glibc_version); + if (q.android_api_level) |x| wip.extra.appendAssumeCapacity(x); + wip.addExtraOptionalStringAssumeCapacity(dynamic_linker); + + // Deduplicate. + const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ + .extra = wip.extra.items, + })); + if (gop.found_existing) { + wip.extra.items.len = @intFromEnum(result_index); + return .init(gop.key_ptr.*); + } else { + return .init(result_index); + } + } + + pub fn addTarget(wip: *Wip, t: std.Target) !TargetQuery.Index { + const gpa = wip.gpa; + const cpu_name: String = try wip.addString(t.cpu.model.name); + + const os_version_min: ?u32, const os_version_max: ?u32, const glibc_version: ?String, const android_api_level: ?u32 = switch (t.os.versionRange()) { + .none => .{ + null, + null, + null, + null, + }, + .semver => |range| .{ + @intFromEnum(try wip.addSemVer(range.min)), + @intFromEnum(try wip.addSemVer(range.max)), + null, + null, + }, + .hurd => |hurd| .{ + @intFromEnum(try wip.addSemVer(hurd.range.min)), + @intFromEnum(try wip.addSemVer(hurd.range.max)), + try wip.addSemVer(hurd.glibc), + null, + }, + .linux => |linux| .{ + @intFromEnum(try wip.addSemVer(linux.range.min)), + @intFromEnum(try wip.addSemVer(linux.range.max)), + try wip.addSemVer(linux.glibc), + linux.android, + }, + .windows => |range| .{ + @intFromEnum(range.min), + @intFromEnum(range.max), + null, + null, + }, + }; + const dynamic_linker: ?String = if (t.dynamic_linker.get()) |dl| try wip.addString(dl) else null; + const cpu_features_add_empty = t.cpu.features.isEmpty(); + const os_version: TargetQuery.OsVersion = switch (t.os.versionRange()) { + .none => .none, + .semver, .linux, .hurd => .semver, + .windows => .windows, + }; + try wip.extra.ensureUnusedCapacity(gpa, @typeInfo(TargetQuery).@"struct".fields.len + 6 + + 2 * ((@sizeOf(std.Target.Cpu.Feature.Set) + 3) / 4)); + const result_index: TargetQuery.Index = @enumFromInt(wip.addExtraAssumeCapacity(@as(TargetQuery, .{ + .flags = .{ + .cpu_arch = .init(t.cpu.arch), + .cpu_model = .explicit, + .cpu_features_add = !cpu_features_add_empty, + .cpu_features_sub = false, + .os_tag = .init(t.os.tag), + .abi = .init(t.abi), + .object_format = .init(t.ofmt), + .os_version_min = os_version, + .os_version_max = os_version, + .glibc_version = glibc_version != null, + .android_api_level = android_api_level != null, + .dynamic_linker = dynamic_linker != null, + }, + }))); + if (!cpu_features_add_empty) wip.extra.appendSliceAssumeCapacity(@ptrCast(&t.cpu.features.ints)); + wip.addExtraOptionalStringAssumeCapacity(cpu_name); + if (os_version_min) |v| wip.extra.appendAssumeCapacity(v); + if (os_version_max) |v| wip.extra.appendAssumeCapacity(v); + wip.addExtraOptionalStringAssumeCapacity(glibc_version); + if (android_api_level) |x| wip.extra.appendAssumeCapacity(x); + wip.addExtraOptionalStringAssumeCapacity(dynamic_linker); + + // Deduplicate. + const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ + .extra = wip.extra.items, + })); + if (gop.found_existing) { + wip.extra.items.len = @intFromEnum(result_index); + return gop.key_ptr.*; + } else { + return result_index; + } + } + pub fn prepareDeps(wip: *Wip, n: usize) Allocator.Error![]u32 { const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1); slice[0] = @intCast(n); @@ -178,6 +346,11 @@ pub const Wip = struct { return result; } + fn addExtraOptionalStringAssumeCapacity(wip: *Wip, optional_string: ?String) void { + const string = optional_string orelse return; + wip.extra.appendAssumeCapacity(@intFromEnum(string)); + } + fn setExtra(wip: *Wip, index: usize, extra: anytype) void { const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; var i = index; @@ -386,7 +559,7 @@ pub const Step = extern struct { flags3: Flags3, flags4: Flags4, - root_module: Module, + root_module: Module.Index, root_name: String, pub const ExpectedCompileErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; @@ -466,19 +639,6 @@ pub const Step = extern struct { }; } }; - pub const DefaultingBool = enum(u2) { - false, - true, - default, - - pub fn init(b: ?bool) DefaultingBool { - return switch (b orelse return .default) { - false => .false, - true => .true, - }; - } - }; - pub const Subsystem = enum(u4) { console, windows, @@ -626,7 +786,7 @@ pub const LazyPath = enum(u32) { pub const SourcePath = struct { flags: Flags, - owner: Package, + owner: Package.Index, sub_path: String, pub const Flags = packed struct(u32) { @@ -662,11 +822,168 @@ pub const LazyPath = enum(u32) { }; }; -pub const Package = enum(u32) { - _, +pub const Package = extern struct { + hash: String, + build_root: OptionalString, + + pub const Index = enum(u32) { + root = maxInt(u32), + _, + }; +}; + +/// Trailing: +/// * c_macros: LengthPrefixedList(String), // if flag is set +/// * lib_paths: LengthPrefixedList(LazyPath), // if flag is set +/// * export_symbol_names: LengthPrefixedList(String), // if flag is set +/// * frameworks: FlagsPrefixedList(FrameworkFlags), // if flag is set +/// * include_dirs: UnionList(IncludeDir), // if flag is set +/// * rpaths: UnionList(RPath), // if flag is set +/// * link_objects: UnionList(LinkObject), // if flag is set +pub const Module = struct { + flags: Flags, + flags2: Flags2, + owner: Package.Index, + root_source_file: OptionalLazyPath, + import_table: ImportTable, + resolved_target: ResolvedTarget.OptionalIndex, + + pub const Optimize = enum(u3) { + debug, + safe, + fast, + small, + default, + + pub fn init(o: ?std.builtin.OptimizeMode) Optimize { + return switch (o orelse return .default) { + .Debug => .debug, + .ReleaseSafe => .safe, + .ReleaseFast => .fast, + .ReleaseSmall => .small, + }; + } + }; + + pub const UnwindTables = enum(u2) { + none, + sync, + async, + default, + + pub fn init(ut: ?std.builtin.UnwindTables) UnwindTables { + return switch (ut orelse return .default) { + .none => .none, + .sync => .sync, + .async => .async, + }; + } + }; + + pub const SanitizeC = enum(u2) { + off, + trap, + full, + default, + + pub fn init(sc: ?std.zig.SanitizeC) SanitizeC { + return switch (sc orelse return .default) { + .off => .off, + .trap => .trap, + .full => .full, + }; + } + }; + + pub const DwarfFormat = enum(u2) { + @"32", + @"64", + default, + + pub fn init(df: ?std.dwarf.Format) DwarfFormat { + return switch (df orelse return .default) { + .@"32" => .@"32", + .@"64" => .@"64", + }; + } + }; + + pub const Index = enum(u32) { + _, + }; + + pub const Flags = packed struct(u32) { + optimize: Optimize, + strip: DefaultingBool, + unwind_tables: UnwindTables, + dwarf_format: DwarfFormat, + single_threaded: DefaultingBool, + stack_protector: DefaultingBool, + stack_check: DefaultingBool, + sanitize_c: SanitizeC, + sanitize_thread: DefaultingBool, + fuzz: DefaultingBool, + code_model: std.builtin.CodeModel, + c_macros: bool, + include_dirs: bool, + lib_paths: bool, + rpaths: bool, + frameworks: bool, + link_objects: bool, + export_symbol_names: bool, + }; + + pub const Flags2 = packed struct(u32) { + valgrind: DefaultingBool, + pic: DefaultingBool, + red_zone: DefaultingBool, + omit_frame_pointer: DefaultingBool, + error_tracing: DefaultingBool, + link_libc: DefaultingBool, + link_libcpp: DefaultingBool, + no_builtin: DefaultingBool, + _: u16 = 0, + }; + + pub const IncludeDir = union(enum(u3)) { + path: LazyPath, + path_system: LazyPath, + path_after: LazyPath, + framework_path: LazyPath, + framework_path_system: LazyPath, + /// Always `Step.Tag.compile`. + other_step: Step.Index, + /// Always `Step.Tag.config_header`. + config_header_step: Step.Index, + embed_path: LazyPath, + }; + + pub const RPath = union(enum(u1)) { + lazy_path: LazyPath, + special: String, + }; + + pub const LinkObject = union(enum(u3)) { + static_path: LazyPath, + /// Always `Step.Tag.compile`. + other_step: Step.Index, + system_lib: SystemLib, + assembly_file: LazyPath, + c_source_file: CSourceFile.Index, + c_source_files: CSourceFiles.Index, + win32_resource_file: RcSourceFile.Index, + }; + + pub const FrameworkFlags = packed struct(u2) { + needed: bool, + weak: bool, + }; }; -pub const Module = enum(u32) { +/// Points into `extra`, first element is len, then: +/// * import_name: String, // for each len +/// * Module.Index, // for each len +pub const ImportTable = enum(u32) { _, }; @@ -734,6 +1051,397 @@ pub const String = enum(u32) { } }; +pub const DefaultingBool = enum(u2) { + false, + true, + default, + + pub fn init(b: ?bool) DefaultingBool { + return switch (b orelse return .default) { + false => .false, + true => .true, + }; + } +}; + +pub const SystemLib = struct { + name: String, + flags: Flags, + + pub const Index = enum(u32) { + _, + }; + + pub const UsePkgConfig = enum(u2) { no, yes, force }; + pub const LinkMode = enum { static, dynamic }; + + pub const Flags = packed struct(u32) { + needed: bool, + weak: bool, + use_pkg_config: UsePkgConfig, + preferred_link_mode: LinkMode, + search_strategy: SearchStrategy, + }; + + pub const SearchStrategy = enum(u2) { paths_first, mode_first, no_fallback }; +}; + +/// Trailing: +/// * flag: String, // for each flags_len +/// * sub_path: String, // for each files_len +pub const CSourceFiles = struct { + root: LazyPath, + files_len: u32, + flags: Flags, + + pub const Index = enum(u32) { + _, + }; + + pub const Flags = packed struct(u32) { + /// C compiler CLI flags. + flags_len: u29, + lang: OptionalCSourceLanguage, + }; +}; + +/// Trailing: +/// * flag: String, // for each flags_len +pub const CSourceFile = struct { + file: LazyPath, + flags: Flags, + + pub const Index = enum(u32) { + _, + }; + + pub const Flags = packed struct(u32) { + /// C compiler CLI flags. + flags_len: u29, + lang: OptionalCSourceLanguage, + }; +}; + +pub const OptionalCSourceLanguage = enum(u3) { + c, + cpp, + objective_c, + objective_cpp, + assembly, + assembly_with_preprocessor, + default, +}; + +pub const RcSourceFile = struct { + file: LazyPath, + /// Any option that rc.exe accepts will work here, with the exception of: + /// - `/fo`: The output filename is set by the build system + /// - `/p`: Only running the preprocessor is not supported in this context + /// - `/:no-preprocess` (non-standard option): Not supported in this context + /// - Any MUI-related option + /// https://learn.microsoft.com/en-us/windows/win32/menurc/using-rc-the-rc-command-line- + /// + /// Implicitly defined options: + /// /x (ignore the INCLUDE environment variable) + /// /D_DEBUG or /DNDEBUG depending on the optimization mode + flags: []const []const u8 = &.{}, + /// Include paths that may or may not exist yet and therefore need to be + /// specified as a LazyPath. Each path will be appended to the flags + /// as `/I `. + include_paths: []const LazyPath = &.{}, + + pub const Index = enum(u32) { + _, + }; +}; + +pub const ResolvedTarget = struct { + /// none indicates host. + query: TargetQuery.OptionalIndex, + /// defaults will be resolved. + result: TargetQuery.Index, + + pub const Index = enum(u32) { + _, + }; + + pub const OptionalIndex = enum(u32) { + none = maxInt(u32), + _, + }; +}; + +/// Trailing: +/// * cpu_features_add: std.Target.Feature.Set, // if flag set +/// * cpu_features_sub: std.Target.Feature.Set, // if flag set +/// * cpu_name: String, // if cpu_model is explicit +/// * os_version_min: WindowsVersion // if os_version_min is windows +/// * os_version_min: String // if os_version_min is semver +/// * os_version_max: WindowsVersion // if os_version_max is windows +/// * os_version_max: String // if os_version_max is semver +/// * glibc_version: String, // if flag is set +/// * android_api_level: u32, // if flag is set +/// * dynamic_linker: String, // if flag is set +pub const TargetQuery = struct { + flags: Flags, + + pub const Index = enum(u32) { + _, + + pub fn extraSlice(i: Index, extra: []const u32) []const u32 { + return extra[@intFromEnum(i)..][0..length(i, extra)]; + } + + pub fn length(i: Index, extra: []const u32) usize { + //const flags = getExtra(extra, @intFromEnum(i), TargetQuery).flags; + const flags: Flags = @bitCast(extra[@intFromEnum(i)]); + const feature_set_size: usize = (@sizeOf(std.Target.Cpu.Feature.Set) + 3) / 4; + return @typeInfo(TargetQuery).@"struct".fields.len + + (if (flags.cpu_features_add) feature_set_size else 0) + + (if (flags.cpu_features_sub) feature_set_size else 0) + + @intFromBool(flags.cpu_model == .explicit) + + @as(usize, switch (flags.os_version_min) { + .semver, .windows => 1, + else => 0, + }) + + @as(usize, switch (flags.os_version_max) { + .semver, .windows => 1, + else => 0, + }) + + @intFromBool(flags.glibc_version) + + @intFromBool(flags.android_api_level) + + @intFromBool(flags.dynamic_linker); + } + }; + + pub const OptionalIndex = enum(u32) { + none = maxInt(u32), + _, + + pub fn init(i: Index) OptionalIndex { + const result: OptionalIndex = @enumFromInt(@intFromEnum(i)); + assert(result != .none); + return result; + } + }; + + pub const CpuModel = enum(u2) { + native, + baseline, + determined_by_arch_os, + explicit, + + pub fn init(x: std.Target.Query.CpuModel) @This() { + return switch (x) { + .native => .native, + .baseline => .baseline, + .determined_by_arch_os => .determined_by_arch_os, + .explicit => .explicit, + }; + } + }; + pub const OsVersion = enum(u2) { + none, + semver, + windows, + default, + + pub fn init(x: ?std.Target.Query.OsVersion) @This() { + return switch (x orelse return .default) { + .none => .none, + .semver => .semver, + .windows => .windows, + }; + } + }; + pub const Abi = enum(u5) { + none, + gnu, + gnuabin32, + gnuabi64, + gnueabi, + gnueabihf, + gnuf32, + gnusf, + gnux32, + eabi, + eabihf, + ilp32, + android, + androideabi, + musl, + muslabin32, + muslabi64, + musleabi, + musleabihf, + muslf32, + muslsf, + muslx32, + msvc, + itanium, + simulator, + ohos, + ohoseabi, + + default, + + pub fn init(x: ?std.Target.Abi) @This() { + // TODO comptime assert the enums match + return @enumFromInt(@intFromEnum(x orelse return .default)); + } + }; + pub const CpuArch = enum(u6) { + aarch64, + aarch64_be, + alpha, + amdgcn, + arc, + arceb, + arm, + armeb, + avr, + bpfeb, + bpfel, + csky, + hexagon, + hppa, + hppa64, + kalimba, + kvx, + lanai, + loongarch32, + loongarch64, + m68k, + microblaze, + microblazeel, + mips, + mipsel, + mips64, + mips64el, + msp430, + nvptx, + nvptx64, + or1k, + powerpc, + powerpcle, + powerpc64, + powerpc64le, + propeller, + riscv32, + riscv32be, + riscv64, + riscv64be, + s390x, + sh, + sheb, + sparc, + sparc64, + spirv32, + spirv64, + thumb, + thumbeb, + ve, + wasm32, + wasm64, + x86_16, + x86, + x86_64, + xcore, + xtensa, + xtensaeb, + + default, + + pub fn init(x: ?std.Target.Cpu.Arch) @This() { + // TODO comptime assert the enums match + return @enumFromInt(@intFromEnum(x orelse return .default)); + } + }; + pub const OsTag = enum(u6) { + freestanding, + other, + contiki, + fuchsia, + hermit, + managarm, + haiku, + hurd, + illumos, + linux, + plan9, + rtems, + serenity, + dragonfly, + freebsd, + netbsd, + openbsd, + driverkit, + ios, + maccatalyst, + macos, + tvos, + visionos, + watchos, + windows, + uefi, + @"3ds", + ps3, + ps4, + ps5, + vita, + emscripten, + wasi, + amdhsa, + amdpal, + cuda, + mesa3d, + nvcl, + opencl, + opengl, + vulkan, + + default, + + pub fn init(x: ?std.Target.Os.Tag) @This() { + // TODO comptime assert the enums match + return @enumFromInt(@intFromEnum(x orelse return .default)); + } + }; + pub const ObjectFormat = enum(u4) { + c, + coff, + elf, + hex, + macho, + plan9, + raw, + spirv, + wasm, + + default, + + pub fn init(x: ?std.Target.ObjectFormat) @This() { + // TODO comptime assert the enums match + return @enumFromInt(@intFromEnum(x orelse return .default)); + } + }; + + pub const Flags = packed struct(u32) { + cpu_arch: CpuArch, + cpu_model: CpuModel, + cpu_features_add: bool, + cpu_features_sub: bool, + os_tag: OsTag, + abi: Abi, + object_format: ObjectFormat, + os_version_min: OsVersion, + os_version_max: OsVersion, + glibc_version: bool, + android_api_level: bool, + dynamic_linker: bool, + }; +}; + pub const LoadFileError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; pub fn loadFile(arena: Allocator, io: Io, file: Io.File) LoadFileError!Configuration { -- 2.54.0 From 2b11859a1049910a2bd8031e53c24beea305ae10 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 16 Feb 2026 16:16:23 -0800 Subject: [PATCH 008/179] configure runner: implement builderToPackage --- lib/compiler/configure_runner.zig | 12 ++++++------ lib/std/zig/Configuration.zig | 20 +++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index b61d2b57fabc8c3762fb2b68b130b0b6650f7734..460173735c57b7fda4d8db5dac70a81e7df166dd 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -524,7 +524,7 @@ fn addModule( .link_libcpp = .init(m.strip), .no_builtin = .init(m.strip), }, - .owner = builderToPackage(m.owner), + .owner = try builderToPackage(wc, m.owner), .root_source_file = try addOptionalLazyPath(wc, m.root_source_file), .import_table = import_table, .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target), @@ -555,7 +555,7 @@ fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configu const sub_path = try wc.addString(src_path.sub_path); break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ .flags = .{}, - .owner = builderToPackage(src_path.owner), + .owner = try builderToPackage(wc, src_path.owner), .sub_path = sub_path, })); }, @@ -577,16 +577,16 @@ fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configu const sub_path = try wc.addString(dependency.sub_path); break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ .flags = .{}, - .owner = builderToPackage(dependency.dependency.builder), + .owner = try builderToPackage(wc, dependency.dependency.builder), .sub_path = sub_path, })); }, }); } -fn builderToPackage(b: *std.Build) Configuration.Package.Index { - _ = b; - @panic("TODO"); +fn builderToPackage(wc: *Configuration.Wip, b: *std.Build) !Configuration.Package { + if (b.pkg_hash.len == 0) return .root; + return .fromHash(try wc.addString(b.pkg_hash)); } fn addInstallDir(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !Configuration.InstallDir { diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 61ac838b12c3bc3c955f8ac2e741b914b6e7ceee..0dee682bafe4946209e550e78038c4eed2ccbccd 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -786,7 +786,7 @@ pub const LazyPath = enum(u32) { pub const SourcePath = struct { flags: Flags, - owner: Package.Index, + owner: Package, sub_path: String, pub const Flags = packed struct(u32) { @@ -822,14 +822,16 @@ pub const LazyPath = enum(u32) { }; }; -pub const Package = extern struct { - hash: String, - build_root: OptionalString, +/// It's an OptionalString which points to the package hash. +pub const Package = enum(u32) { + root = maxInt(u32), + _, - pub const Index = enum(u32) { - root = maxInt(u32), - _, - }; + pub fn fromHash(hash: String) Package { + const result: Package = @enumFromInt(@intFromEnum(hash)); + assert(result != .root); + return result; + } }; /// Trailing: @@ -843,7 +845,7 @@ pub const Package = extern struct { pub const Module = struct { flags: Flags, flags2: Flags2, - owner: Package.Index, + owner: Package, root_source_file: OptionalLazyPath, import_table: ImportTable, resolved_target: ResolvedTarget.OptionalIndex, -- 2.54.0 From 0505318efe0d2757a344dded9ae1607f948f7511 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 17 Feb 2026 17:08:55 -0800 Subject: [PATCH 009/179] make runner gets compiled and run and --print-configuration prints some deserialized stuff --- lib/compiler/configure_runner.zig | 65 ++- lib/compiler/maker.zig | 650 ++++++++++++++++-------------- lib/compiler/maker/Fuzz.zig | 18 +- lib/compiler/maker/Graph.zig | 83 +--- lib/compiler/maker/Package.zig | 18 + lib/compiler/maker/Step.zig | 12 + lib/compiler/maker/WebServer.zig | 35 +- lib/std/Build.zig | 24 +- lib/std/zig/Configuration.zig | 104 ++++- src/Compilation.zig | 4 +- src/main.zig | 63 +-- 11 files changed, 596 insertions(+), 480 deletions(-) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 460173735c57b7fda4d8db5dac70a81e7df166dd..ad3ef25a8d33b5d64aa3ee52d68b4f14e7ea05b3 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -1,18 +1,19 @@ const builtin = @import("builtin"); const std = @import("std"); +const Allocator = std.mem.Allocator; +const Color = std.zig.Color; +const Configuration = std.Build.Configuration; +const File = std.Io.File; const Io = std.Io; +const Step = std.Build.Step; +const Writer = std.Io.Writer; const assert = std.debug.assert; +const fatal = std.process.fatal; const fmt = std.fmt; +const log = std.log; const mem = std.mem; const process = std.process; -const File = std.Io.File; -const Step = std.Build.Step; -const Allocator = std.mem.Allocator; -const fatal = std.process.fatal; -const Writer = std.Io.Writer; -const Color = std.zig.Color; -const Configuration = std.Build.Configuration; pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); @@ -95,7 +96,6 @@ pub fn main(init: process.Init.Minimal) !void { .query = .{}, .result = try std.zig.system.resolveTargetQuery(io, .{}), }, - .time_report = false, }; graph.cache.addPrefix(.{ .path = null, .handle = cwd }); @@ -141,9 +141,9 @@ pub fn main(init: process.Init.Minimal) !void { fatal(" access the help menu with 'zig build -h'", .{}); } } else if (mem.cutPrefix(u8, arg, "-fsys=")) |name| { - graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); + try graph.system_integration_options.put(arena, name, .user_enabled); } else if (mem.cutPrefix(u8, arg, "-fno-sys=")) |name| { - graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); + try graph.system_integration_options.put(arena, name, .user_disabled); } else if (mem.eql(u8, arg, "--release")) { graph.release_mode = .any; } else if (mem.cutPrefix(u8, arg, "--release=")) |text| { @@ -177,8 +177,6 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.cutPrefix(u8, arg, "--build-id=")) |style| { builder.build_id = std.zig.BuildId.parse(style) catch |err| fatal("unable to parse --build-id style '{s}': {t}", .{ style, err }); - } else if (mem.eql(u8, arg, "--debug-rt")) { - graph.debug_compiler_runtime_libs = true; } else if (mem.eql(u8, arg, "--debug-compile-errors")) { builder.debug_compile_errors = true; } else if (mem.eql(u8, arg, "--debug-incremental")) { @@ -204,10 +202,16 @@ pub fn main(init: process.Init.Minimal) !void { try builder.runBuild(root); + if (builder.validateUserInputDidItFail()) { + fatal(" access the help menu with 'zig build -h'", .{}); + } + var wc: Configuration.Wip = .init(gpa); defer wc.deinit(); assert(try wc.addString("") == .empty); + try serializeSystemIntegrationOptions(&graph, &wc); + var stdout_buffer: [1024]u8 = undefined; var file_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); serialize(builder, &wc, &file_writer.interface) catch |err| switch (err) { @@ -367,7 +371,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .root_name = try wc.addString(c.name), })); - std.log.err("TODO serialize the trailing Compile step data", .{}); + log.err("TODO serialize the trailing Compile step data", .{}); break :e extra_index; }, @@ -441,7 +445,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .captured_stderr = captured_stderr, })); - std.log.err("TODO serialize the trailing Run step data", .{}); + log.err("TODO serialize the trailing Run step data", .{}); break :e extra_index; }, @@ -489,7 +493,7 @@ fn addModule( m.import_table.values(), @intFromEnum(import_table) + 1 + m.import_table.entries.len.., ) |dep, extra_index| { - // TODO module dependencies can be cyclic + log.err("TODO module dependencies can be cyclic", .{}); wc.extra.items[extra_index] = @intFromEnum(try addModule(wc, module_map, dep)); } @@ -530,7 +534,7 @@ fn addModule( .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target), }))); - std.log.err("TODO serialize the trailing Module data", .{}); + log.err("TODO serialize the trailing Module data", .{}); try module_map.putNoClobber(gpa, m, module_index); @@ -542,7 +546,7 @@ fn addOptionalResolvedTarget( optional_resolved_target: ?std.Build.ResolvedTarget, ) !Configuration.ResolvedTarget.OptionalIndex { const resolved_target = optional_resolved_target orelse return .none; - // TODO dedupe + log.debug("TODO deduplicate resolved targets", .{}); return @enumFromInt(try wc.addExtra(@as(Configuration.ResolvedTarget, .{ .query = try wc.addTargetQuery(resolved_target.query), .result = try wc.addTarget(resolved_target.result), @@ -698,3 +702,30 @@ const Summary = enum { all, new, failures, line, none }; fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { fatal(f ++ "\n access the help menu with \"zig build -h\"", args); } + +fn serializeSystemIntegrationOptions(graph: *std.Build.Graph, wc: *Configuration.Wip) Allocator.Error!void { + const gpa = wc.gpa; + + var bad = false; + try wc.system_integrations.ensureTotalCapacityPrecise(gpa, graph.system_integration_options.entries.len); + for (graph.system_integration_options.keys(), graph.system_integration_options.values()) |k, v| { + wc.system_integrations.appendAssumeCapacity(.{ + .name = try wc.addString(k), + .status = switch (v) { + .user_disabled, .user_enabled => x: { + // The user tried to enable or disable a system library integration, but + // the configure script did not recognize that option. + log.err("system integration name not recognized by configure script: {s}", .{k}); + bad = true; + break :x .disabled; + }, + .declared_disabled => .disabled, + .declared_enabled => .enabled, + }, + }); + } + if (bad) { + log.info("access the help menu with \"zig build -h\"", .{}); + process.exit(1); + } +} diff --git a/lib/compiler/maker.zig b/lib/compiler/maker.zig index 44fe7170ab3b6043303880acb05b129c168d63d1..030bbdfe20d9a8bf606cc4631b3c13acfaa20541 100644 --- a/lib/compiler/maker.zig +++ b/lib/compiler/maker.zig @@ -1,21 +1,23 @@ const builtin = @import("builtin"); const std = @import("std"); +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; +const File = std.Io.File; const Io = std.Io; +const Path = std.Build.Cache.Path; +const Writer = std.Io.Writer; const assert = std.debug.assert; +const fatal = std.process.fatal; const fmt = std.fmt; +const log = std.log; const mem = std.mem; const process = std.process; -const File = std.Io.File; -const Allocator = std.mem.Allocator; -const fatal = std.process.fatal; -const Writer = std.Io.Writer; -const Cache = std.Build.Cache; -const Configuration = std.Build.Configuration; const Fuzz = @import("maker/Fuzz.zig"); const Graph = @import("maker/Graph.zig"); -const Step = @import("maker/Step.zig"); +const Step = void; // @import("maker/Step.zig"); const Watch = @import("maker/Watch.zig"); const WebServer = @import("maker/WebServer.zig"); @@ -48,12 +50,12 @@ pub fn main(init: process.Init.Minimal) !void { // skip my own exe name var arg_idx: usize = 1; - const zig_exe = cutArgPrefixOrFatal(args, &arg_idx, "--zig="); - const zig_lib_dir = cutArgPrefixOrFatal(args, &arg_idx, "--lib="); - const build_root = cutArgPrefixOrFatal(args, &arg_idx, "--build-root="); - const local_cache_root = cutArgPrefixOrFatal(args, &arg_idx, "--local-cache="); - const global_cache_root = cutArgPrefixOrFatal(args, &arg_idx, "--global-cache="); - const configure_path = cutArgPrefixOrFatal(args, &arg_idx, "--configure="); + const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); + const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); + const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); + const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); + const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); + const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); const cwd: Io.Dir = .cwd(); @@ -90,11 +92,6 @@ pub fn main(init: process.Init.Minimal) !void { .environ_map = try init.environ.createMap(arena), .global_cache_root = global_cache_directory, .zig_lib_directory = zig_lib_directory, - .host = .{ - .query = .{}, - .result = try std.zig.system.resolveTargetQuery(io, .{}), - }, - .time_report = false, }; graph.cache.addPrefix(.{ .path = null, .handle = cwd }); @@ -105,9 +102,13 @@ pub fn main(init: process.Init.Minimal) !void { var targets = std.array_list.Managed([]const u8).init(arena); var debug_log_scopes = std.array_list.Managed([]const u8).init(arena); - - var install_prefix: ?[]const u8 = null; - var dir_list: std.Build.DirList = .{}; + var help_menu = false; + var steps_menu = false; + var print_configuration = false; + var override_install_prefix: ?[]const u8 = null; + var override_lib_dir: ?[]const u8 = null; + var override_bin_dir: ?[]const u8 = null; + var override_include_dir: ?[]const u8 = null; var error_style: ErrorStyle = .verbose; var multiline_errors: MultilineErrors = .indent; var summary: ?Summary = null; @@ -115,8 +116,6 @@ pub fn main(init: process.Init.Minimal) !void { var skip_oom_steps = false; var test_timeout_ns: ?u64 = null; var color: Color = .auto; - var help_menu = false; - var steps_menu = false; var watch = false; var fuzz: ?Fuzz.Mode = null; var debounce_interval_ms: u16 = 50; @@ -152,51 +151,53 @@ pub fn main(init: process.Init.Minimal) !void { } } - var configuration: Configuration = undefined; - { - var file = cwd.openFile(io, configure_path, .{}) catch |err| - fatal("failed to open configuration file {f}: {t}", .{ configure_path, err }); - defer file.close(io); - configuration = Configuration.load(arena, io, file) catch |err| - fatal("failed to load configuration file {f}: {t}", .{ configure_path, err }); - } - graph.configuration = &configuration; - graph.scanConfiguration(); + const scanned_config: ScannedConfig = sc: { + const configuration = c: { + var file = cwd.openFile(io, configure_path, .{}) catch |err| + fatal("failed to open configuration file {s}: {t}", .{ configure_path, err }); + defer file.close(io); + break :c Configuration.loadFile(arena, io, file) catch |err| + fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); + }; + var top_level_steps: std.ArrayList(Configuration.Step.Index) = .empty; + for (configuration.steps, 0..) |*conf_step, step_index| { + const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]); + if (flags.tag == .top_level) { + try top_level_steps.append(arena, @enumFromInt(step_index)); + } + } + break :sc .{ + .configuration = configuration, + .top_level_steps = top_level_steps.items, + }; + }; - std.log.err("TODO handle user -D options", .{}); + log.err("TODO handle user -D options", .{}); while (nextArg(args, &arg_idx)) |arg| { if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "--verbose")) { - verbose = true; - } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { help_menu = true; - } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { - install_prefix = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { steps_menu = true; - } else if (mem.startsWith(u8, arg, "-fsys=")) { - const name = arg["-fsys=".len..]; - graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); - } else if (mem.startsWith(u8, arg, "-fno-sys=")) { - const name = arg["-fno-sys=".len..]; - graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); + } else if (mem.eql(u8, arg, "--print-configuration")) { + print_configuration = true; + } else if (mem.eql(u8, arg, "--verbose")) { + verbose = true; + } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { + override_install_prefix = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { - dir_list.lib_dir = nextArgOrFatal(args, &arg_idx); + override_lib_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { - dir_list.exe_dir = nextArgOrFatal(args, &arg_idx); + override_bin_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--prefix-include-dir")) { - dir_list.include_dir = nextArgOrFatal(args, &arg_idx); + override_include_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--sysroot")) { sysroot = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--maxrss")) { const max_rss_text = nextArgOrFatal(args, &arg_idx); - max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { - std.debug.print("invalid byte size: '{s}': {s}\n", .{ - max_rss_text, @errorName(err), - }); - process.exit(1); - }; + max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| + fatal("invalid byte size: '{s}': {t}", .{ max_rss_text, err }); } else if (mem.eql(u8, arg, "--skip-oom-steps")) { skip_oom_steps = true; } else if (mem.eql(u8, arg, "--test-timeout")) { @@ -270,9 +271,7 @@ pub fn main(init: process.Init.Minimal) !void { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected u32 after '{s}'", .{arg}); graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ - next_arg, @errorName(err), - }); + fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {t}", .{ next_arg, err }); }; } else if (mem.eql(u8, arg, "--debounce")) { const next_arg = nextArg(args, &arg_idx) orelse @@ -288,7 +287,7 @@ pub fn main(init: process.Init.Minimal) !void { const addr_str = arg["--webui=".len..]; if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { - fatal("invalid web UI address '{s}': {s}", .{ addr_str, @errorName(err) }); + fatal("invalid web UI address '{s}': {t}", .{ addr_str, err }); }; } else if (mem.eql(u8, arg, "--debug-log")) { const next_arg = nextArgOrFatal(args, &arg_idx); @@ -300,11 +299,6 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse fatal("unrecognized optimization mode: {s}", .{rest}); - } else if (mem.eql(u8, arg, "--system")) { - // The usage text shows another argument after this parameter - // but it is handled by the parent process. The build runner - // only sees this flag. - graph.system_package_mode = true; } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); @@ -383,7 +377,7 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.startsWith(u8, arg, "-freference-trace=")) { const num = arg["-freference-trace=".len..]; reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); + std.debug.print("unable to parse reference_trace count '{s}': {t}", .{ num, err }); process.exit(1); }; } else if (mem.eql(u8, arg, "-fno-reference-trace")) { @@ -414,6 +408,29 @@ pub fn main(init: process.Init.Minimal) !void { .off => .no_color, }; + if (help_menu) { + var w = initStdoutWriter(io); + scanned_config.printUsage(&graph, w) catch |err| switch (err) { + error.WriteFailed => return stdout_writer_allocation.err.?, + else => |e| return e, + }; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } else if (steps_menu) { + var w = initStdoutWriter(io); + scanned_config.printSteps(&graph, w) catch |err| switch (err) { + error.WriteFailed => return stdout_writer_allocation.err.?, + else => |e| return e, + }; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } else if (print_configuration) { + var w = initStdoutWriter(io); + scanned_config.print(w) catch return stdout_writer_allocation.err.?; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } + if (webui_listen != null) { if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); @@ -424,27 +441,33 @@ pub fn main(init: process.Init.Minimal) !void { }); defer main_progress_node.end(); - graph.resolveInstallPrefix(install_prefix, dir_list); + const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ + .root_dir = .cwd(), + .sub_path = try Io.Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), + } else if (override_install_prefix) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else .{ + .root_dir = build_root_directory, + .sub_path = "zig-out", + }; - if (graph.validateUserInputDidItFail()) { - fatal(" access the help menu with 'zig build -h'", .{}); - } + const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "lib"); - validateSystemLibraryOptions(&graph); + const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "bin"); - if (help_menu) { - var w = initStdoutWriter(io); - printUsage(&graph, w) catch return stdout_writer_allocation.err.?; - w.flush() catch return stdout_writer_allocation.err.?; - return; - } + const install_include_path: Path = if (override_include_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "include"); - if (steps_menu) { - var w = initStdoutWriter(io); - printSteps(&graph, w) catch return stdout_writer_allocation.err.?; - w.flush() catch return stdout_writer_allocation.err.?; - return; - } + if (true) @panic("TODO"); var run: Run = .{ .gpa = gpa, @@ -463,6 +486,13 @@ pub fn main(init: process.Init.Minimal) !void { .error_style = error_style, .multiline_errors = multiline_errors, .summary = summary orelse if (watch or webui_listen != null) .line else .failures, + + .install_paths = .{ + .prefix = install_prefix_path, + .lib = install_lib_path, + .bin = install_bin_path, + .include = install_include_path, + }, }; defer { run.memory_blocked_steps.deinit(gpa); @@ -628,9 +658,9 @@ fn prepare(graph: *Graph, step_names: []const []const u8, run: *Run) !void { try step_stack.ensureUnusedCapacity(gpa, step_names.len); for (0..step_names.len) |i| { const step_name = step_names[step_names.len - i - 1]; - const s = graph.top_level_steps.get(step_name) orelse { - std.log.info("access the help menu with \"zig build -h\"", .{}); - fatal("no step named '{s}'", .{step_name}); + const s = run.top_level_steps.get(step_name) orelse { + log.info("access the help menu with 'zig build -h'", .{}); + fatal("no such step: {s}", .{step_name}); }; step_stack.putAssumeCapacity(&s.step, {}); } @@ -873,11 +903,11 @@ fn runStepNames( print_node.last = true; printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {}; } else { - const last_index = if (run.summary == .all) graph.top_level_steps.count() else blk: { + const last_index = if (run.summary == .all) run.top_level_steps.count() else blk: { var i: usize = step_names.len; while (i > 0) { i -= 1; - const step = graph.top_level_steps.get(step_names[i]).?.step; + const step = run.top_level_steps.get(step_names[i]).?.step; const found = switch (run.summary) { .all, .line, .none => unreachable, .failures => step.state != .success, @@ -885,10 +915,10 @@ fn runStepNames( }; if (found) break :blk i; } - break :blk graph.top_level_steps.count(); + break :blk run.top_level_steps.count(); }; for (step_names, 0..) |step_name, i| { - const tls = graph.top_level_steps.get(step_name).?; + const tls = run.top_level_steps.get(step_name).?; print_node.last = i + 1 == last_index; printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; } @@ -1207,7 +1237,7 @@ fn constructGraphAndCheckForDependencyLoop( // We dupe to avoid shuffling the steps in the summary, it depends // on s.dependencies' order. - const deps = gpa.dupe(*Step, s.dependencies.items) catch @panic("OOM"); + const deps = try gpa.dupe(*Step, s.dependencies.items); defer gpa.free(deps); rand.shuffle(*Step, deps); @@ -1327,7 +1357,7 @@ fn makeStep( try run.max_rss_mutex.lock(io); defer run.max_rss_mutex.unlock(io); run.available_rss += s.max_rss; - dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch @panic("OOM"); + try dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len); while (run.memory_blocked_steps.getLast()) |candidate| { if (run.available_rss < candidate.max_rss) break; assert(run.memory_blocked_steps.pop() == candidate); @@ -1360,7 +1390,7 @@ fn stepReady( defer run.max_rss_mutex.unlock(io); if (run.available_rss < s.max_rss) { // Running this step right now could possibly exceed the allotted RSS. - run.memory_blocked_steps.append(run.gpa, s) catch @panic("OOM"); + try run.memory_blocked_steps.append(run.gpa, s); return; } run.available_rss -= s.max_rss; @@ -1454,178 +1484,6 @@ pub fn printErrorMessages( try writer.writeByte('\n'); } -fn printSteps(graph: *Graph, w: *Writer) !void { - const arena = graph.arena; - for (graph.top_level_steps.values()) |top_level_step| { - const name = if (&top_level_step.step == graph.default_step) - try fmt.allocPrint(arena, "{s} (default)", .{top_level_step.step.name}) - else - top_level_step.step.name; - try w.print(" {s:<28} {s}\n", .{ name, top_level_step.description }); - } -} - -fn printUsage(graph: *Graph, w: *Writer) !void { - const arena = graph.arena; - - try w.print( - \\Usage: {s} build [steps] [options] - \\ - \\Steps: - \\ - , .{graph.zig_exe}); - try printSteps(graph, w); - try w.writeAll( - \\ - \\Project-Specific Options: - \\ - ); - - if (graph.available_options_list.items.len == 0) { - try w.print(" (none)\n", .{}); - } else { - for (graph.available_options_list.items) |option| { - const name = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ option.name, option.type_id }); - try w.print("{s:<30} {s}\n", .{ name, option.description }); - if (option.enum_options) |enum_options| { - const padding: [33]u8 = @splat(' '); - try w.writeAll(padding ++ "Supported Values:\n"); - for (enum_options) |enum_option| { - try w.print(padding ++ " {s}\n", .{enum_option}); - } - } - } - } - - try w.writeAll( - \\ - \\System Integration Options: - \\ --search-prefix [path] Add a path to look for binaries, libraries, headers - \\ --sysroot [path] Set the system root directory (usually /) - \\ --libc [file] Provide a file which specifies libc paths - \\ - \\ --system [pkgdir] Disable package fetching; enable all integrations - \\ -fsys=[name] Enable a system integration - \\ -fno-sys=[name] Disable a system integration - \\ - \\ -fdarling, -fno-darling Integration with system-installed Darling to - \\ execute macOS programs on Linux hosts - \\ (default: no) - \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute - \\ foreign-architecture programs on Linux hosts - \\ (default: no) - \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc - \\ (e.g. glibc or musl) built for multiple foreign - \\ architectures, allowing execution of non-native - \\ programs that link with libc. - \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on - \\ ARM64 macOS hosts. (default: no) - \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to - \\ execute WASI binaries. (default: no) - \\ -fwine, -fno-wine Integration with system-installed Wine to execute - \\ Windows programs on Linux hosts. (default: no) - \\ - \\ Available System Integrations: Enabled: - \\ - ); - if (graph.system_library_options.entries.len == 0) { - try w.writeAll(" (none) -\n"); - } else { - for (graph.system_library_options.keys(), graph.system_library_options.values()) |k, v| { - const status = switch (v) { - .declared_enabled => "yes", - .declared_disabled => "no", - .user_enabled, .user_disabled => unreachable, // already emitted error - }; - try w.print(" {s:<43} {s}\n", .{ k, status }); - } - } - - try w.writeAll( - \\ - \\General Options: - \\ -h, --help Print this help and exit - \\ -l, --list-steps Print available steps - \\ - \\ -p, --prefix [path] Where to install files (default: zig-out) - \\ --prefix-lib-dir [path] Where to install libraries - \\ --prefix-exe-dir [path] Where to install executables - \\ --prefix-include-dir [path] Where to install C header files - \\ --release[=mode] Request release mode, optionally specifying a - \\ preferred optimization mode: fast, safe, small - \\ - \\ --verbose Print commands before executing them - \\ --color [auto|off|on] Enable or disable colored error messages - \\ --error-style [style] Control how build errors are printed - \\ verbose (Default) Report errors with full context - \\ minimal Report errors after summary, excluding context like command lines - \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update - \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update - \\ --multiline-errors [style] Control how multi-line error messages are printed - \\ indent (Default) Indent non-initial lines to align with initial line - \\ newline Include a leading newline so that the error message is on its own lines - \\ none Print as usual so the first line is misaligned - \\ --summary [mode] Control the printing of the build summary - \\ all Print the build summary in its entirety - \\ new Omit cached steps - \\ failures (Default if short-lived) Only print failed steps - \\ line (Default if long-lived) Only print the single-line summary - \\ none Do not print the build summary - \\ -j Limit concurrent jobs (default is to use all CPU cores) - \\ --maxrss Limit memory usage (default is to use available memory) - \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss - \\ --test-timeout Limit execution time of unit tests, terminating if exceeded. - \\ The timeout must include a unit: ns, us, ms, s, m, h - \\ --watch Continuously rebuild when source files are modified - \\ --debounce Delay before rebuilding after changed file detected - \\ --webui[=ip] Enable the web interface on the given IP address - \\ --fuzz[=limit] Continuously search for unit test failures with an optional - \\ limit to the max number of iterations. The argument supports - \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies - \\ '--webui' when no limit is specified. - \\ --time-report Force full rebuild and provide detailed information on - \\ compilation time of Zig source code (implies '--webui') - \\ -fincremental Enable incremental compilation - \\ -fno-incremental Disable incremental compilation - \\ - \\Package Management Options: - \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit - \\ needed (Default) Lazy dependencies are fetched as needed - \\ all Lazy dependencies are always fetched - \\ --fork=[path] Override one or more projects from dependency tree - \\ - \\Advanced Options: - \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error - \\ -fno-reference-trace Disable reference trace - \\ -fallow-so-scripts Allows .so files to be GNU ld scripts - \\ -fno-allow-so-scripts (default) .so files must be ELF files - \\ --build-file [file] Override path to build.zig - \\ --cache-dir [path] Override path to local Zig cache directory - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --zig-lib-dir [arg] Override path to Zig lib directory - \\ --build-runner [file] Override path to build runner - \\ --seed [integer] For shuffling dependency traversal order (default: random) - \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries - \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) - \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) - \\ md5 16-byte cryptographic hash (ELF) - \\ uuid 16-byte random UUID (ELF, WASM) - \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) - \\ none (default) No build ID - \\ --debug-log [scope] Enable debugging the compiler - \\ --debug-pkg-config Fail if unknown pkg-config flags encountered - \\ --debug-rt Debug compiler runtime libraries - \\ --verbose-link Enable compiler debug output for linking - \\ --verbose-air Enable compiler debug output for Zig AIR - \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR - \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC - \\ --verbose-cimport Enable compiler debug output for C imports - \\ --verbose-cc Enable compiler debug output for C compilation - \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features - \\ - ); -} - fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { if (idx.* >= args.len) return null; defer idx.* += 1; @@ -1633,17 +1491,17 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { } fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse - fatal("expected argument after {q}\n access the help menu with \"zig build -h\"", .{args[idx.* - 1]}); + return nextArg(args, idx) orelse { + log.info("access the help menu with \"zig build -h\"", .{}); + fatal("expected argument after {q}", .{args[idx.* - 1]}); + }; } -fn cutArgPrefixOrFatal(args: []const [:0]const u8, idx: *usize, prefix: []const u8) []const u8 { - if (nextArg(args, idx)) |next_arg| { - if (mem.cutPrefix(u8, next_arg, prefix)) |arg| { - return arg; - } - } - fatal("expected argument after {q} to start with {q}", .{ args[idx.* - 1], prefix }); +fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { + const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); + if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); + const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); + return arg; } fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { @@ -1674,35 +1532,8 @@ const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { - std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); - process.exit(1); -} - -fn validateSystemLibraryOptions(graph: *Graph) void { - var bad = false; - for (graph.system_library_options.keys(), graph.system_library_options.values()) |k, v| { - switch (v) { - .user_disabled, .user_enabled => { - // The user tried to enable or disable a system library integration, but - // the build script did not recognize that option. - std.debug.print("system library name not recognized by build script: '{s}'\n", .{k}); - bad = true; - }, - .declared_disabled, .declared_enabled => {}, - } - } - if (bad) { - std.debug.print(" access the help menu with 'zig build -h'\n", .{}); - process.exit(1); - } -} - -var stdio_buffer_allocation: [256]u8 = undefined; -var stdout_writer_allocation: Io.File.Writer = undefined; - -fn initStdoutWriter(io: Io) *Writer { - stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); - return &stdout_writer_allocation.interface; + log.info("access the help menu with 'zig build -h'", .{}); + fatal(f, args); } fn cleanTmpFiles(io: Io, steps: []const *Step) void { @@ -1711,7 +1542,222 @@ fn cleanTmpFiles(io: Io, steps: []const *Step) void { if (wf.mode != .tmp) continue; const path = wf.generated_directory.path orelse continue; Io.Dir.cwd().deleteTree(io, path) catch |err| { - std.log.warn("failed to delete {s}: {t}", .{ path, err }); + log.warn("failed to delete {s}: {t}", .{ path, err }); }; } } + +const InstallPaths = struct { + prefix: Path, + lib: Path, + bin: Path, + include: Path, +}; + +var stdio_buffer_allocation: [256]u8 = undefined; +var stdout_writer_allocation: Io.File.Writer = undefined; + +fn initStdoutWriter(io: Io) *Writer { + stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); + return &stdout_writer_allocation.interface; +} + +const ScannedConfig = struct { + configuration: Configuration, + top_level_steps: []const Configuration.Step.Index, + + fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { + var serializer: std.zon.Serializer = .{ .writer = w }; + var s = try serializer.beginStruct(.{}); + + try s.field("default_step", @intFromEnum(sc.configuration.default_step), .{}); + { + var tuple = try s.beginTupleField("top_level_steps", .{}); + for (sc.top_level_steps) |step| try tuple.field(@intFromEnum(step), .{}); + try tuple.end(); + } + + try s.end(); + } + + fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + const c = &sc.configuration; + for (sc.top_level_steps) |step_index| { + const step = step_index.ptr(c); + const name = step.name.slice(c); + const decorated_name = if (step_index == c.default_step) + try fmt.allocPrint(arena, "{s} (default)", .{name}) + else + name; + const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); + const description = top_level.description.slice(c); + try w.print(" {s:<28} {s}\n", .{ decorated_name, description }); + } + } + + fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + + try w.print( + \\Usage: {s} build [steps] [options] + \\ + \\Steps: + \\ + , .{graph.zig_exe}); + try printSteps(sc, graph, w); + try w.writeAll( + \\ + \\Project-Specific Options: + \\ + ); + + const available_options = sc.configuration.available_options; + if (available_options.len == 0) { + try w.print(" (none)\n", .{}); + } else { + for (available_options) |option| { + const name = option.name.slice(&sc.configuration); + const description = option.description.slice(&sc.configuration); + const help = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type }); + try w.print("{s:<30} {s}\n", .{ help, description }); + if (option.enum_options.slice(&sc.configuration)) |enum_options| { + const padding: [33]u8 = @splat(' '); + try w.writeAll(padding ++ "Supported Values:\n"); + for (enum_options) |enum_option_index| { + const enum_option = enum_option_index.slice(&sc.configuration); + try w.print(padding ++ " {s}\n", .{enum_option}); + } + } + } + } + + try w.writeAll( + \\ + \\System Integration Options: + \\ --search-prefix [path] Add a path to look for binaries, libraries, headers + \\ --sysroot [path] Set the system root directory (usually /) + \\ --libc [file] Provide a file which specifies libc paths + \\ + \\ --system [pkgdir] Disable package fetching; enable all integrations + \\ -fsys=[name] Enable a system integration + \\ -fno-sys=[name] Disable a system integration + \\ + \\ -fdarling, -fno-darling Integration with system-installed Darling to + \\ execute macOS programs on Linux hosts + \\ (default: no) + \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute + \\ foreign-architecture programs on Linux hosts + \\ (default: no) + \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc + \\ (e.g. glibc or musl) built for multiple foreign + \\ architectures, allowing execution of non-native + \\ programs that link with libc. + \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on + \\ ARM64 macOS hosts. (default: no) + \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to + \\ execute WASI binaries. (default: no) + \\ -fwine, -fno-wine Integration with system-installed Wine to execute + \\ Windows programs on Linux hosts. (default: no) + \\ + \\ Available System Integrations: Enabled: + \\ + ); + if (sc.configuration.system_integrations.len == 0) { + try w.writeAll(" (none) -\n"); + } else { + for (sc.configuration.system_integrations) |system_integration| { + const name = system_integration.name.slice(&sc.configuration); + const status = switch (system_integration.status) { + .disabled => "no", + .enabled => "yes", + }; + try w.print(" {s:<43} {s}\n", .{ name, status }); + } + } + + try w.writeAll( + \\ + \\General Options: + \\ -h, --help Print this help and exit + \\ -l, --list-steps Print available steps + \\ + \\ -p, --prefix [path] Where to install files (default: zig-out) + \\ --prefix-lib-dir [path] Where to install libraries + \\ --prefix-exe-dir [path] Where to install executables + \\ --prefix-include-dir [path] Where to install C header files + \\ --release[=mode] Request release mode, optionally specifying a + \\ preferred optimization mode: fast, safe, small + \\ + \\ --verbose Print commands before executing them + \\ --color [auto|off|on] Enable or disable colored error messages + \\ --error-style [style] Control how build errors are printed + \\ verbose (Default) Report errors with full context + \\ minimal Report errors after summary, excluding context like command lines + \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update + \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update + \\ --multiline-errors [style] Control how multi-line error messages are printed + \\ indent (Default) Indent non-initial lines to align with initial line + \\ newline Include a leading newline so that the error message is on its own lines + \\ none Print as usual so the first line is misaligned + \\ --summary [mode] Control the printing of the build summary + \\ all Print the build summary in its entirety + \\ new Omit cached steps + \\ failures (Default if short-lived) Only print failed steps + \\ line (Default if long-lived) Only print the single-line summary + \\ none Do not print the build summary + \\ -j Limit concurrent jobs (default is to use all CPU cores) + \\ --maxrss Limit memory usage (default is to use available memory) + \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss + \\ --test-timeout Limit execution time of unit tests, terminating if exceeded. + \\ The timeout must include a unit: ns, us, ms, s, m, h + \\ --watch Continuously rebuild when source files are modified + \\ --debounce Delay before rebuilding after changed file detected + \\ --webui[=ip] Enable the web interface on the given IP address + \\ --fuzz[=limit] Continuously search for unit test failures with an optional + \\ limit to the max number of iterations. The argument supports + \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies + \\ '--webui' when no limit is specified. + \\ --time-report Force full rebuild and provide detailed information on + \\ compilation time of Zig source code (implies '--webui') + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation + \\ + \\Package Management Options: + \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit + \\ needed (Default) Lazy dependencies are fetched as needed + \\ all Lazy dependencies are always fetched + \\ --fork=[path] Override one or more projects from dependency tree + \\ + \\Advanced Options: + \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error + \\ -fno-reference-trace Disable reference trace + \\ -fallow-so-scripts Allows .so files to be GNU ld scripts + \\ -fno-allow-so-scripts (default) .so files must be ELF files + \\ --build-file [file] Override path to build.zig + \\ --cache-dir [path] Override path to local Zig cache directory + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --zig-lib-dir [arg] Override path to Zig lib directory + \\ --build-runner [file] Override path to build runner + \\ --seed [integer] For shuffling dependency traversal order (default: random) + \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries + \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) + \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) + \\ md5 16-byte cryptographic hash (ELF) + \\ uuid 16-byte random UUID (ELF, WASM) + \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) + \\ none (default) No build ID + \\ --debug-log [scope] Enable debugging the compiler + \\ --debug-pkg-config Fail if unknown pkg-config flags encountered + \\ --debug-rt Debug compiler runtime libraries + \\ --verbose-link Enable compiler debug output for linking + \\ --verbose-air Enable compiler debug output for Zig AIR + \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR + \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC + \\ --verbose-cimport Enable compiler debug output for C imports + \\ --verbose-cc Enable compiler debug output for C compilation + \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features + \\ + ); + } +}; diff --git a/lib/compiler/maker/Fuzz.zig b/lib/compiler/maker/Fuzz.zig index a9b01e8111c0bf6034d740461d654f455e557813..b0bcbf8621e05cb5416f4f1240e4960b8f23e76a 100644 --- a/lib/compiler/maker/Fuzz.zig +++ b/lib/compiler/maker/Fuzz.zig @@ -1,17 +1,19 @@ -const std = @import("Std"); +const Fuzz = @This(); + +const std = @import("std"); const Io = std.Io; const Build = std.Build; -const Cache = Build.Cache; +const Cache = std.Build.Cache; const Step = std.Build.Step; const assert = std.debug.assert; const fatal = std.process.fatal; const Allocator = std.mem.Allocator; const log = std.log; const Coverage = std.debug.Coverage; -const abi = Build.abi.fuzz; +const abi = std.Build.abi.fuzz; -const Fuzz = @This(); -const build_runner = @import("root"); +const maker = @import("../maker.zig"); +const WebServer = @import("WebServer.zig"); gpa: Allocator, io: Io, @@ -33,7 +35,7 @@ queue_cond: Io.Condition, msg_queue: std.ArrayList(Msg), pub const Mode = union(enum) { - forever: struct { ws: *Build.WebServer }, + forever: struct { ws: *WebServer }, limit: Limited, pub const Limited = struct { @@ -173,7 +175,7 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod var buf: [256]u8 = undefined; const stderr = try io.lockStderr(&buf, graph.stderr_mode); defer io.unlockStderr(); - build_runner.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; } const rebuilt_bin_path = result catch |err| switch (err) { @@ -196,7 +198,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run) void { error.Canceled => return, }; defer io.unlockStderr(); - build_runner.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; return; }, else => { diff --git a/lib/compiler/maker/Graph.zig b/lib/compiler/maker/Graph.zig index ed5e297b59339ce09271cf7a4ec49b4df4046a97..ba9cad0ee17b713cd1731aff04e22a132d94afdd 100644 --- a/lib/compiler/maker/Graph.zig +++ b/lib/compiler/maker/Graph.zig @@ -6,87 +6,20 @@ const Io = std.Io; const Allocator = std.mem.Allocator; const Configuration = std.Build.Configuration; -const Step = @import("Step.zig"); -const Package = @import("Package.zig"); - io: Io, /// Process lifetime. arena: Allocator, -system_library_options: std.StringArrayHashMapUnmanaged(std.Build.SystemLibraryMode), -system_package_mode: bool, -debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, cache: std.Build.Cache, -zig_exe: [:0]const u8, +zig_exe: []const u8, environ_map: std.process.Environ.Map, global_cache_root: std.Build.Cache.Directory, zig_lib_directory: std.Build.Cache.Directory, -incremental: ?bool, -random_seed: u32, -allow_so_scripts: ?bool, -time_report: bool, + +debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, +incremental: ?bool = null, +random_seed: u32 = 0, +allow_so_scripts: ?bool = null, +time_report: bool = false, /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also /// respects the '--color' flag. -stderr_mode: ?Io.Terminal.Mode, - -configuration: *const Configuration, -top_level_steps: std.AutoArrayHashMapUnmanaged(Configuration.String, Configuration.Step.Index), - -pub const DirList = struct { - lib_dir: ?[]const u8 = null, - exe_dir: ?[]const u8 = null, - include_dir: ?[]const u8 = null, -}; - -/// This function is intended to be called by lib/build_runner.zig, not a build.zig file. -pub fn resolveInstallPrefix(graph: *Graph, p: *Package, install_prefix: ?[]const u8, dir_list: DirList) !void { - if (p.dest_dir) |dest_dir| { - p.install_prefix = install_prefix orelse "/usr"; - p.install_path = b.pathJoin(&.{ dest_dir, p.install_prefix }); - } else { - p.install_prefix = install_prefix orelse - (p.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); - b.install_path = b.install_prefix; - } - - var lib_list = [_][]const u8{ b.install_path, "lib" }; - var exe_list = [_][]const u8{ b.install_path, "bin" }; - var h_list = [_][]const u8{ b.install_path, "include" }; - - if (dir_list.lib_dir) |dir| { - if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; - lib_list[1] = dir; - } - - if (dir_list.exe_dir) |dir| { - if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; - exe_list[1] = dir; - } - - if (dir_list.include_dir) |dir| { - if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; - h_list[1] = dir; - } - - b.lib_dir = b.pathJoin(&lib_list); - b.exe_dir = b.pathJoin(&exe_list); - b.h_dir = b.pathJoin(&h_list); -} - -fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { - // Create an installation directory local to this package. This will be used when - // dependant packages require a standard prefix, such as include directories for C headers. - var hash = b.graph.cache.hash; - // Random bytes to make unique. Refresh this with new random bytes when - // implementation is modified in a non-backwards-compatible way. - hash.add(@as(u32, 0xd8cb0056)); - hash.addBytes(b.dep_prefix); - - var wyhash = std.hash.Wyhash.init(0); - hashUserInputOptionsMap(b.allocator, b.user_input_options, &wyhash); - hash.add(wyhash.final()); - - const digest = hash.final(); - const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); - b.resolveInstallPrefix(install_prefix, .{}); -} - +stderr_mode: ?Io.Terminal.Mode = null, diff --git a/lib/compiler/maker/Package.zig b/lib/compiler/maker/Package.zig index e3ad442f69d6fa215b7dde6394211bc9cda283bf..42e3b47d29cf46cae53f592f0e884ef8c931487f 100644 --- a/lib/compiler/maker/Package.zig +++ b/lib/compiler/maker/Package.zig @@ -10,3 +10,21 @@ exe_dir: []const u8, h_dir: []const u8, /// Path to the directory containing build.zig. build_root: std.Build.Cache.Path, + +fn determineAndApplyInstallPrefix(p: *Package) error{OutOfMemory}!void { + // Create an installation directory local to this package. This will be used when + // dependant packages require a standard prefix, such as include directories for C headers. + var hash = p.graph.cache.hash; + // Random bytes to make unique. Refresh this with new random bytes when + // implementation is modified in a non-backwards-compatible way. + hash.add(@as(u32, 0xd8cb0056)); + hash.addBytes(p.dep_prefix); + + var wyhash = std.hash.Wyhash.init(0); + hashUserInputOptionsMap(p.allocator, p.user_input_options, &wyhash); + hash.add(wyhash.final()); + + const digest = hash.final(); + const install_prefix = try p.cache_root.join(p.allocator, &.{ "i", &digest }); + p.resolveInstallPrefix(install_prefix, .{}); +} diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/maker/Step.zig index 295993ec245d79d5edde3444f8ed7aec7e2c6b5b..164510f3dd8d56d5477585c0226fc543940deb42 100644 --- a/lib/compiler/maker/Step.zig +++ b/lib/compiler/maker/Step.zig @@ -848,3 +848,15 @@ pub fn allocPrintCmd( return aw.toOwnedSlice(); } +pub fn getInstallPath(b: *Build, dir: InstallDir, dest_rel_path: []const u8) []const u8 { + assert(!fs.path.isAbsolute(dest_rel_path)); // Install paths must be relative to the prefix + const base_dir = switch (dir) { + .prefix => b.install_path, + .bin => b.exe_dir, + .lib => b.lib_dir, + .header => b.h_dir, + .custom => |p| b.pathJoin(&.{ b.install_path, p }), + }; + return b.pathResolve(&.{ base_dir, dest_rel_path }); +} + diff --git a/lib/compiler/maker/WebServer.zig b/lib/compiler/maker/WebServer.zig index f860b9f58f965ecdfdf260a40455a71dd518169e..9a81d6e5712c74104d40bfe0d9e7de909477187f 100644 --- a/lib/compiler/maker/WebServer.zig +++ b/lib/compiler/maker/WebServer.zig @@ -1,3 +1,21 @@ +const WebServer = @This(); + +const builtin = @import("builtin"); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Build = std.Build; +const Cache = std.Build.Cache; +const Io = std.Io; +const abi = std.Build.abi; +const assert = std.debug.assert; +const http = std.http; +const log = std.log.scoped(.web_server); +const mem = std.mem; +const net = std.Io.net; + +const Fuzz = @import("Fuzz.zig"); + gpa: Allocator, graph: *const Build.Graph, all_steps: []const *Build.Step, @@ -907,20 +925,3 @@ const cache_control_header: http.Header = .{ .name = "Cache-Control", .value = "max-age=0, must-revalidate", }; - -const builtin = @import("builtin"); - -const std = @import("std"); -const Io = std.Io; -const net = std.Io.net; -const assert = std.debug.assert; -const mem = std.mem; -const log = std.log.scoped(.web_server); -const Allocator = std.mem.Allocator; -const Build = std.Build; -const Cache = Build.Cache; -const Fuzz = Build.Fuzz; -const abi = Build.abi; -const http = std.http; - -const WebServer = @This(); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 6fe78cb1d6c7ad681379413d204275b1446d6d70..c36e93e8118c67e7f74c314290cb47df09fc5778 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -92,9 +92,8 @@ pub const Graph = struct { io: Io, /// Process lifetime. arena: Allocator, - system_library_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .empty, + system_integration_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .empty, system_package_mode: bool = false, - debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, cache: Cache, zig_exe: []const u8, environ_map: process.Environ.Map, @@ -108,7 +107,7 @@ pub const Graph = struct { /// Steps should use `io` to limit the number of jobs, however in the case of /// a single step spawning a fixed number of processes this can be used. max_jobs: ?u32 = null, - time_report: bool, + time_report: bool = false, /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also /// respects the '--color' flag. stderr_mode: ?Io.Terminal.Mode = null, @@ -322,11 +321,6 @@ fn createChild( .invalid_user_input = false, .default_step = undefined, .top_level_steps = .{}, - .install_prefix = undefined, - .lib_dir = parent.lib_dir, - .exe_dir = parent.exe_dir, - .h_dir = parent.h_dir, - .install_path = parent.install_path, .sysroot = parent.sysroot, .build_root = build_root, .cache_root = parent.cache_root, @@ -1769,18 +1763,6 @@ pub fn run(b: *Build, argv: []const []const u8) []u8 { ); } -pub fn getInstallPath(b: *Build, dir: InstallDir, dest_rel_path: []const u8) []const u8 { - assert(!fs.path.isAbsolute(dest_rel_path)); // Install paths must be relative to the prefix - const base_dir = switch (dir) { - .prefix => b.install_path, - .bin => b.exe_dir, - .lib => b.lib_dir, - .header => b.h_dir, - .custom => |p| b.pathJoin(&.{ b.install_path, p }), - }; - return b.pathResolve(&.{ base_dir, dest_rel_path }); -} - pub const Dependency = struct { builder: *Build, @@ -2572,7 +2554,7 @@ pub fn systemIntegrationOption( name: []const u8, config: SystemIntegrationOptionConfig, ) bool { - const gop = b.graph.system_library_options.getOrPut(b.allocator, name) catch @panic("OOM"); + const gop = b.graph.system_integration_options.getOrPut(b.allocator, name) catch @panic("OOM"); if (gop.found_existing) switch (gop.value_ptr.*) { .user_disabled => { gop.value_ptr.* = .declared_disabled; diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 0dee682bafe4946209e550e78038c4eed2ccbccd..0c4d45b5f6f6ca166609d69006677484ba5502bf 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -11,7 +11,10 @@ steps: []Step, path_deps_base: []Path.Base, path_deps_sub: []String, unlazy_deps: []String, +system_integrations: []SystemIntegration, +available_options: []AvailableOption, extra: []u32, +default_step: Step.Index, /// The field order here matches `Configuration` which documents the order in /// the serialized format. @@ -20,6 +23,8 @@ pub const Header = extern struct { steps_len: u32, path_deps_len: u32, unlazy_deps_len: u32, + system_integrations_len: u32, + available_options_len: u32, extra_len: u32, default_step: Step.Index, @@ -33,6 +38,8 @@ pub const Wip = struct { string_bytes: std.ArrayList(u8) = .empty, unlazy_deps: std.ArrayList(String) = .empty, + system_integrations: std.ArrayList(SystemIntegration) = .empty, + available_options: std.ArrayList(AvailableOption) = .empty, steps: std.ArrayList(Step) = .empty, path_deps: std.MultiArrayList(Path) = .empty, extra: std.ArrayList(u32) = .empty, @@ -107,6 +114,8 @@ pub const Wip = struct { const gpa = wip.gpa; wip.string_bytes.deinit(gpa); wip.unlazy_deps.deinit(gpa); + wip.system_integrations.deinit(gpa); + wip.available_options.deinit(gpa); wip.steps.deinit(gpa); wip.path_deps.deinit(gpa); wip.extra.deinit(gpa); @@ -123,6 +132,8 @@ pub const Wip = struct { .steps_len = @intCast(wip.steps.items.len), .path_deps_len = @intCast(wip.path_deps.len), .unlazy_deps_len = @intCast(wip.unlazy_deps.items.len), + .system_integrations_len = @intCast(wip.system_integrations.items.len), + .available_options_len = @intCast(wip.available_options.items.len), .extra_len = @intCast(wip.extra.items.len), .default_step = static.default_step, @@ -134,6 +145,8 @@ pub const Wip = struct { @ptrCast(wip.path_deps.items(.base)), @ptrCast(wip.path_deps.items(.sub)), @ptrCast(wip.unlazy_deps.items), + @ptrCast(wip.system_integrations.items), + @ptrCast(wip.available_options.items), @ptrCast(wip.extra.items), }; try w.writeVecAll(&buffers); @@ -367,6 +380,37 @@ pub const Wip = struct { } }; +pub const SystemIntegration = extern struct { + name: String, + status: Status, + + pub const Status = enum(u32) { + disabled = 0, + enabled = 1, + }; +}; + +pub const AvailableOption = extern struct { + name: String, + description: String, + type: Type, + /// If the `type_id` is `enum` or `enum_list` this provides the list of enum options + enum_options: OptionalStringList, + + pub const Type = enum(u8) { + bool, + int, + float, + @"enum", + enum_list, + string, + list, + build_id, + lazy_path, + lazy_path_list, + }; +}; + pub const Step = extern struct { name: String, deps: Deps, @@ -375,8 +419,19 @@ pub const Step = extern struct { /// with `Tag`. extra_index: u32, + /// Points into `steps`. pub const Index = enum(u32) { _, + + pub fn ptr(i: Index, c: *const Configuration) *const Step { + return &c.steps[@intFromEnum(i)]; + } + }; + + /// Shared by all steps. + pub const Flags = packed struct(u32) { + tag: Tag, + _: u27 = 0, }; pub const Tag = enum(u5) { @@ -400,7 +455,7 @@ pub const Step = extern struct { }; pub const TopLevel = struct { - flags: Flags = .{}, + flags: @This().Flags = .{}, description: String, pub const Flags = packed struct(u32) { @@ -410,7 +465,7 @@ pub const Step = extern struct { }; pub const InstallArtifact = struct { - flags: Flags, + flags: @This().Flags, dest_dir: InstallDir, dest_sub_path: String, @@ -445,7 +500,7 @@ pub const Step = extern struct { /// * stdio_limit: u64, // if stdio_limit is set /// * producer: Step.Index, // if producer is set. always compile step pub const Run = struct { - flags: Flags, + flags: @This().Flags, file_inputs_len: u32, args_len: u32, cwd: OptionalLazyPath, @@ -554,7 +609,7 @@ pub const Step = extern struct { /// * error_limit if flag is set /// * Hexstring if build_id is hexstring pub const Compile = struct { - flags: Flags, + flags: @This().Flags, flags2: Flags2, flags3: Flags3, flags4: Flags4, @@ -989,12 +1044,26 @@ pub const ImportTable = enum(u32) { _, }; -/// Points into `extra`, where the first element is number of deps, -/// following elements is `Step.Index` per dep. +/// Points into `extra`, where the first element is count of deps, following +/// elements is `Step.Index` per count. pub const Deps = enum(u32) { _, }; +/// Points into `extra`, where the first element is count of strings, following +/// elements is `String` per count. +/// +/// Stored identically to `Deps`. +pub const OptionalStringList = enum(u32) { + none = maxInt(u32), + _, + + pub fn slice(osl: OptionalStringList, c: *const Configuration) ?[]const String { + const len = c.extra[@intFromEnum(osl)]; + return @ptrCast(c.extra[@intFromEnum(osl) + 1 ..][0..len]); + } +}; + pub const Path = extern struct { base: Base, sub: String, @@ -1444,6 +1513,23 @@ pub const TargetQuery = struct { }; }; +pub fn extraData(c: *const Configuration, comptime T: type, index: usize) T { + const extra = c.extra; + var i: usize = index; + var result: T = undefined; + inline for (@typeInfo(T).@"struct".fields) |field| { + comptime assert(@sizeOf(field.type) == @sizeOf(u32)); + @field(result, field.name) = switch (@typeInfo(field.type)) { + .int => extra[i], + .@"enum" => @enumFromInt(extra[i]), + .@"struct" => @bitCast(extra[i]), + else => comptime unreachable, + }; + i += 1; + } + return result; +} + pub const LoadFileError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; pub fn loadFile(arena: Allocator, io: Io, file: Io.File) LoadFileError!Configuration { @@ -1465,7 +1551,10 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { .path_deps_sub = try arena.alloc(String, header.path_deps_len), .path_deps_base = try arena.alloc(Path.Base, header.path_deps_len), .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), + .system_integrations = try arena.alloc(SystemIntegration, header.system_integrations_len), + .available_options = try arena.alloc(AvailableOption, header.available_options_len), .extra = try arena.alloc(u32, header.extra_len), + .default_step = header.default_step, }; var vecs = [_][]u8{ result.string_bytes, @@ -1473,6 +1562,9 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { @ptrCast(result.path_deps_base), @ptrCast(result.path_deps_sub), @ptrCast(result.unlazy_deps), + @ptrCast(result.system_integrations), + @ptrCast(result.available_options), + @ptrCast(result.extra), }; try reader.readVecAll(&vecs); return result; diff --git a/src/Compilation.zig b/src/Compilation.zig index 9e9b66f03bb9c81dab21b305901efaee158fb8b9..870ac3f79893b9301c8daa075276c10237bce75f 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -3233,9 +3233,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE } // Failure here only means an unnecessary cache miss. - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest: {s}", .{@errorName(err)}); - }; + man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); assert(whole.lock == null); whole.lock = man.toOwnedLock(); diff --git a/src/main.zig b/src/main.zig index a0296f8a79e30d3ec7f018f8695873867aba25ce..a24acdc14ec48af7be44fe003704912c12533649 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3682,26 +3682,16 @@ fn buildOutputType( if (t.arch == target.cpu.arch and t.os == target.os.tag) { // If there's a `glibc_min`, there's also an `os_ver`. if (t.glibc_min) |glibc_min| { - std.log.info("zig can provide libc for related target {s}-{s}.{f}-{s}.{d}.{d}", .{ - @tagName(t.arch), - @tagName(t.os), - t.os_ver.?, - @tagName(t.abi), - glibc_min.major, - glibc_min.minor, + std.log.info("zig can provide libc for related target {t}-{t}.{f}-{t}.{d}.{d}", .{ + t.arch, t.os, t.os_ver.?, t.abi, glibc_min.major, glibc_min.minor, }); } else if (t.os_ver) |os_ver| { - std.log.info("zig can provide libc for related target {s}-{s}.{f}-{s}", .{ - @tagName(t.arch), - @tagName(t.os), - os_ver, - @tagName(t.abi), + std.log.info("zig can provide libc for related target {t}-{t}.{f}-{t}", .{ + t.arch, t.os, os_ver, t.abi, }); } else { - std.log.info("zig can provide libc for related target {s}-{s}-{s}", .{ - @tagName(t.arch), - @tagName(t.os), - @tagName(t.abi), + std.log.info("zig can provide libc for related target {t}-{t}-{t}", .{ + t.arch, t.os, t.abi, }); } } @@ -3710,7 +3700,7 @@ fn buildOutputType( }, else => fatal("{f}", .{create_diag}), }, - else => fatal("failed to create compilation: {s}", .{@errorName(err)}), + else => fatal("failed to create compilation: {t}", .{err}), }; var comp_destroyed = false; defer if (!comp_destroyed) comp.destroy(); @@ -4984,7 +4974,6 @@ fn cmdBuild( try configure_argv.ensureUnusedCapacity(arena, 16); try make_argv.ensureUnusedCapacity(arena, 16); - const argv_index_exe = configure_argv.items.len; _ = configure_argv.addOneAssumeCapacity(); _ = make_argv.addOneAssumeCapacity(); @@ -5069,9 +5058,7 @@ fn cmdBuild( } else if (mem.cutPrefix(u8, arg, "--fetch=")) |sub_arg| { fetch_only = true; fetch_mode = std.meta.stringToEnum(Package.Fetch.JobQueue.Mode, sub_arg) orelse - fatal("expected [needed|all] after '--fetch=', found '{s}'", .{ - sub_arg, - }); + fatal("expected [needed|all] after '--fetch=', found '{s}'", .{sub_arg}); } else if (mem.cutPrefix(u8, arg, "--fork=")) |sub_arg| { try forks.append(arena, .{ .manifest_ast = undefined, @@ -5093,7 +5080,7 @@ fn cmdBuild( continue; } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - fatal("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); + fatal("unable to parse reference_trace count '{s}': {t}", .{ num, err }); }; } else if (mem.eql(u8, arg, "-fno-reference-trace")) { reference_trace = null; @@ -5169,7 +5156,8 @@ fn cmdBuild( } else if (mem.eql(u8, arg, "--")) { // The rest of the args are supposed to get passed onto // build runner's `build.args` - try configure_argv.appendSlice(arena, args[i..]); + try configure_argv.append(arena, "--have-run-args"); + try make_argv.appendSlice(arena, args[i..]); break; } } @@ -5273,7 +5261,12 @@ fn cmdBuild( // Kick off an optimized compilation of the make runner. var make_runner_task = io.async(compileMakeRunner, .{ gpa, arena, io, .{ - .dirs = &dirs, + .dirs = .{ + .cwd = dirs.cwd, + .zig_lib = dirs.zig_lib, + .global_cache = dirs.global_cache, + .local_cache = dirs.global_cache, + }, .environ_map = environ_map, .parent_prog_node = root_prog_node, .resolved_target = resolved_target, @@ -5281,6 +5274,7 @@ fn cmdBuild( .thread_limit = thread_limit, .self_exe_path = self_exe_path, .color = color, + .reference_trace = reference_trace, } }); defer _ = make_runner_task.cancel(io) catch {}; @@ -5289,6 +5283,11 @@ fn cmdBuild( configure_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; configure_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; + make_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; + make_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; + make_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; + make_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; + // Dummy http client that is not actually used when fetch_command is unsupported. // Prevents bootstrap from depending on a bunch of unnecessary stuff. var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { @@ -5601,7 +5600,7 @@ fn cmdBuild( .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), }; _ = try config_man.addFilePath(exe_path, null); - configure_argv.items[argv_index_exe] = try exe_path.toString(arena); + configure_argv.items[0] = try exe_path.toString(arena); if (try config_man.hit()) { const digest = config_man.final(); @@ -5758,15 +5757,15 @@ fn cmdBuild( }) { .exited => |code| { if (code == 0) return cleanExit(io); - const cmd = try std.mem.join(arena, " ", configure_argv.items); + const cmd = try std.mem.join(arena, " ", make_argv.items); fatal("the following maker command failed with exit code {d}:\n{s}", .{ code, cmd }); }, .signal => |sig| { - const cmd = try std.mem.join(arena, " ", configure_argv.items); + const cmd = try std.mem.join(arena, " ", make_argv.items); fatal("the following maker command terminated with signal {t}:\n{s}", .{ sig, cmd }); }, else => { - const cmd = try std.mem.join(arena, " ", configure_argv.items); + const cmd = try std.mem.join(arena, " ", make_argv.items); fatal("the following maker command crashed:\n{s}", .{cmd}); }, } @@ -5777,13 +5776,14 @@ const MakeRunner = struct { const Options = struct { environ_map: *const process.Environ.Map, - dirs: *Compilation.Directories, + dirs: Compilation.Directories, parent_prog_node: std.Progress.Node, resolved_target: Package.Module.ResolvedTarget, libc_installation: ?*const LibCInstallation, self_exe_path: []const u8, thread_limit: usize, color: Color, + reference_trace: ?u32, }; }; @@ -5798,7 +5798,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn const strip = optimize_mode != .Debug; const main_mod_paths: Package.Module.CreateOptions.Paths = .{ - .root = try .fromRoot(arena, options.dirs.*, .zig_lib, "compiler"), + .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"), .root_src_path = "maker.zig", }; @@ -5827,7 +5827,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn var create_diag: Compilation.CreateDiagnostic = undefined; const comp = Compilation.create(gpa, arena, io, &create_diag, .{ - .dirs = options.dirs.*, + .dirs = options.dirs, .root_name = "maker", .config = config, .root_mod = root_mod, @@ -5837,6 +5837,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn .thread_limit = options.thread_limit, .cache_mode = .whole, .environ_map = options.environ_map, + .reference_trace = options.reference_trace, }) catch |err| switch (err) { error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), error.Canceled => |e| return e, -- 2.54.0 From afd7507a197d516c7240f92fc19e4f43adb0b79c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 17 Feb 2026 20:36:45 -0800 Subject: [PATCH 010/179] make runner: prepare steps for execution --- lib/compiler/configure_runner.zig | 142 +++--- lib/compiler/maker.zig | 750 ++++++++++++++++-------------- lib/compiler/maker/Package.zig | 30 -- lib/compiler/maker/Step.zig | 122 ++--- lib/std/Build.zig | 4 + lib/std/Build/Step/Run.zig | 2 + lib/std/zig/Configuration.zig | 32 +- 7 files changed, 542 insertions(+), 540 deletions(-) delete mode 100644 lib/compiler/maker/Package.zig diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index ad3ef25a8d33b5d64aa3ee52d68b4f14e7ea05b3..54113594f6429340aec1fa6c4b1236ee201393e8 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -186,6 +186,8 @@ pub fn main(init: process.Init.Minimal) !void { // but it is handled by the parent process. The build runner // only sees this flag. graph.system_package_mode = true; + } else if (mem.eql(u8, arg, "--have-run-args")) { + graph.have_run_args = true; } else { fatalWithHint("unrecognized argument: '{s}'", .{arg}); } @@ -226,13 +228,69 @@ pub fn main(init: process.Init.Minimal) !void { process.exit(0); } +const Serialize = struct { + arena: Allocator, + wc: *Configuration.Wip, + module_map: std.AutoArrayHashMapUnmanaged(*std.Build.Module, Configuration.Module.Index) = .empty, + package_map: std.AutoArrayHashMapUnmanaged(*std.Build, Configuration.Package.Index) = .empty, + + fn builderToPackage(s: *Serialize, b: *std.Build) !Configuration.Package.Index { + if (b.pkg_hash.len == 0) return .root; + const arena = s.arena; + const wc = s.wc; + const gop = try s.package_map.getOrPut(arena, b); + if (!gop.found_existing) { + gop.value_ptr.* = @enumFromInt(try wc.addExtra(@as(Configuration.Package, .{ + .hash = try wc.addString(b.pkg_hash), + .dep_prefix = try wc.addString(b.dep_prefix), + }))); + } + return gop.value_ptr.*; + } + + fn addOptionalLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.OptionalLazyPath { + const wc = s.wc; + return @enumFromInt(switch (lp orelse return .none) { + .src_path => |src_path| i: { + const sub_path = try wc.addString(src_path.sub_path); + break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ + .flags = .{}, + .owner = try s.builderToPackage(src_path.owner), + .sub_path = sub_path, + })); + }, + .generated => |generated| i: { + const sub_path = try wc.addString(generated.sub_path); + break :i try wc.addExtra(@as(Configuration.LazyPath.Generated, .{ + .flags = .{ .up = @intCast(generated.up) }, + .sub_path = sub_path, + })); + }, + .cwd_relative => |cwd_relative_sub_path| i: { + const sub_path = try wc.addString(cwd_relative_sub_path); + break :i try wc.addExtra(@as(Configuration.LazyPath.Relative, .{ + .flags = .{ .base = .cwd }, + .sub_path = sub_path, + })); + }, + .dependency => |dependency| i: { + const sub_path = try wc.addString(dependency.sub_path); + break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ + .flags = .{}, + .owner = try s.builderToPackage(dependency.dependency.builder), + .sub_path = sub_path, + })); + }, + }); + } +}; + fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { const graph = b.graph; const arena = graph.arena; const gpa = wc.gpa; - var module_map: std.AutoArrayHashMapUnmanaged(*std.Build.Module, Configuration.Module.Index) = .empty; - defer module_map.deinit(gpa); + var s: Serialize = .{ .wc = wc, .arena = arena }; // Starting from all top-level steps in `b`, traverse the entire step graph // and add all step dependencies implied by module graphs. @@ -267,6 +325,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { try wc.steps.ensureTotalCapacity(gpa, step_map.entries.capacity); wc.steps.appendAssumeCapacity(.{ .name = try wc.addString(step.name), + .owner = try s.builderToPackage(step.owner), .deps = deps, .max_rss = .fromBytes(step.max_rss), .extra_index = switch (step.tag) { @@ -367,7 +426,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .install_name = c.install_name != null, .entitlements = c.entitlements != null, }, - .root_module = try addModule(wc, &module_map, c.root_module), + .root_module = try addModule(&s, c.root_module), .root_name = try wc.addString(c.name), })); @@ -383,13 +442,13 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .dest_dir = try addInstallDir(wc, ia.dest_dir), .dest_sub_path = try wc.addString(ia.dest_sub_path), - .emitted_bin = try addOptionalLazyPath(wc, ia.emitted_bin), + .emitted_bin = try s.addOptionalLazyPath(ia.emitted_bin), .implib_dir = try addInstallDir(wc, ia.implib_dir), - .emitted_implib = try addOptionalLazyPath(wc, ia.emitted_implib), + .emitted_implib = try s.addOptionalLazyPath(ia.emitted_implib), .pdb_dir = try addInstallDir(wc, ia.pdb_dir), - .emitted_pdb = try addOptionalLazyPath(wc, ia.emitted_pdb), + .emitted_pdb = try s.addOptionalLazyPath(ia.emitted_pdb), .h_dir = try addInstallDir(wc, ia.h_dir), - .emitted_h = try addOptionalLazyPath(wc, ia.emitted_h), + .emitted_h = try s.addOptionalLazyPath(ia.emitted_h), .artifact = stepIndex(&step_map, &ia.artifact.step), })); }, @@ -440,7 +499,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .file_inputs_len = @intCast(run.file_inputs.items.len), .args_len = @intCast(run.argv.items.len), - .cwd = try addOptionalLazyPath(wc, run.cwd), + .cwd = try s.addOptionalLazyPath(run.cwd), .captured_stdout = captured_stdout, .captured_stderr = captured_stderr, })); @@ -469,13 +528,11 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }); } -fn addModule( - wc: *Configuration.Wip, - module_map: *std.AutoArrayHashMapUnmanaged(*std.Build.Module, Configuration.Module.Index), - m: *std.Build.Module, -) !Configuration.Module.Index { - if (module_map.get(m)) |index| return index; +fn addModule(s: *Serialize, m: *std.Build.Module) !Configuration.Module.Index { + if (s.module_map.get(m)) |index| return index; + const wc = s.wc; + const arena = s.arena; const gpa = wc.gpa; const import_table: Configuration.ImportTable = @enumFromInt(wc.extra.items.len); const import_table_extra_len = 1 + 2 * m.import_table.entries.len; @@ -494,7 +551,7 @@ fn addModule( @intFromEnum(import_table) + 1 + m.import_table.entries.len.., ) |dep, extra_index| { log.err("TODO module dependencies can be cyclic", .{}); - wc.extra.items[extra_index] = @intFromEnum(try addModule(wc, module_map, dep)); + wc.extra.items[extra_index] = @intFromEnum(try addModule(s, dep)); } const module_index: Configuration.Module.Index = @enumFromInt(try wc.addExtra(@as(Configuration.Module, .{ @@ -528,15 +585,15 @@ fn addModule( .link_libcpp = .init(m.strip), .no_builtin = .init(m.strip), }, - .owner = try builderToPackage(wc, m.owner), - .root_source_file = try addOptionalLazyPath(wc, m.root_source_file), + .owner = try s.builderToPackage(m.owner), + .root_source_file = try s.addOptionalLazyPath(m.root_source_file), .import_table = import_table, .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target), }))); log.err("TODO serialize the trailing Module data", .{}); - try module_map.putNoClobber(gpa, m, module_index); + try s.module_map.putNoClobber(arena, m, module_index); return module_index; } @@ -553,46 +610,6 @@ fn addOptionalResolvedTarget( }))); } -fn addOptionalLazyPath(wc: *Configuration.Wip, lp: ?std.Build.LazyPath) !Configuration.OptionalLazyPath { - return @enumFromInt(switch (lp orelse return .none) { - .src_path => |src_path| i: { - const sub_path = try wc.addString(src_path.sub_path); - break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ - .flags = .{}, - .owner = try builderToPackage(wc, src_path.owner), - .sub_path = sub_path, - })); - }, - .generated => |generated| i: { - const sub_path = try wc.addString(generated.sub_path); - break :i try wc.addExtra(@as(Configuration.LazyPath.Generated, .{ - .flags = .{ .up = @intCast(generated.up) }, - .sub_path = sub_path, - })); - }, - .cwd_relative => |cwd_relative_sub_path| i: { - const sub_path = try wc.addString(cwd_relative_sub_path); - break :i try wc.addExtra(@as(Configuration.LazyPath.Relative, .{ - .flags = .{ .base = .cwd }, - .sub_path = sub_path, - })); - }, - .dependency => |dependency| i: { - const sub_path = try wc.addString(dependency.sub_path); - break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ - .flags = .{}, - .owner = try builderToPackage(wc, dependency.dependency.builder), - .sub_path = sub_path, - })); - }, - }); -} - -fn builderToPackage(wc: *Configuration.Wip, b: *std.Build) !Configuration.Package { - if (b.pkg_hash.len == 0) return .root; - return .fromHash(try wc.addString(b.pkg_hash)); -} - fn addInstallDir(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !Configuration.InstallDir { switch (install_dir orelse return .none) { .prefix => return .prefix, @@ -665,9 +682,7 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { return nextArg(args, idx) orelse { - fatal("expected argument after {q}\n access the help menu with \"zig build -h\"", .{ - args[idx.* - 1], - }); + fatalWithHint("expected argument after: {s}", .{args[idx.* - 1]}); }; } @@ -700,7 +715,8 @@ const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { - fatal(f ++ "\n access the help menu with \"zig build -h\"", args); + log.info("to access the help menu: zig build -h", .{}); + fatal(f, args); } fn serializeSystemIntegrationOptions(graph: *std.Build.Graph, wc: *Configuration.Wip) Allocator.Error!void { @@ -725,7 +741,7 @@ fn serializeSystemIntegrationOptions(graph: *std.Build.Graph, wc: *Configuration }); } if (bad) { - log.info("access the help menu with \"zig build -h\"", .{}); + log.info("help menu contains available options: zig build -h", .{}); process.exit(1); } } diff --git a/lib/compiler/maker.zig b/lib/compiler/maker.zig index 030bbdfe20d9a8bf606cc4631b3c13acfaa20541..d35d0584847593fdfc2a38b428c90ac8906a442d 100644 --- a/lib/compiler/maker.zig +++ b/lib/compiler/maker.zig @@ -17,7 +17,7 @@ const process = std.process; const Fuzz = @import("maker/Fuzz.zig"); const Graph = @import("maker/Graph.zig"); -const Step = void; // @import("maker/Step.zig"); +const Step = @import("maker/Step.zig"); const Watch = @import("maker/Watch.zig"); const WebServer = @import("maker/WebServer.zig"); @@ -100,8 +100,8 @@ pub fn main(init: process.Init.Minimal) !void { graph.cache.addPrefix(global_cache_directory); graph.cache.hash.addBytes(builtin.zig_version_string); - var targets = std.array_list.Managed([]const u8).init(arena); - var debug_log_scopes = std.array_list.Managed([]const u8).init(arena); + var step_names: std.ArrayList([]const u8) = .empty; + var debug_log_scopes: std.ArrayList([]const u8) = .empty; var help_menu = false; var steps_menu = false; var print_configuration = false; @@ -151,29 +151,6 @@ pub fn main(init: process.Init.Minimal) !void { } } - const scanned_config: ScannedConfig = sc: { - const configuration = c: { - var file = cwd.openFile(io, configure_path, .{}) catch |err| - fatal("failed to open configuration file {s}: {t}", .{ configure_path, err }); - defer file.close(io); - break :c Configuration.loadFile(arena, io, file) catch |err| - fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); - }; - var top_level_steps: std.ArrayList(Configuration.Step.Index) = .empty; - for (configuration.steps, 0..) |*conf_step, step_index| { - const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]); - if (flags.tag == .top_level) { - try top_level_steps.append(arena, @enumFromInt(step_index)); - } - } - break :sc .{ - .configuration = configuration, - .top_level_steps = top_level_steps.items, - }; - }; - - log.err("TODO handle user -D options", .{}); - while (nextArg(args, &arg_idx)) |arg| { if (mem.startsWith(u8, arg, "-")) { if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { @@ -291,7 +268,7 @@ pub fn main(init: process.Init.Minimal) !void { }; } else if (mem.eql(u8, arg, "--debug-log")) { const next_arg = nextArgOrFatal(args, &arg_idx); - try debug_log_scopes.append(next_arg); + try debug_log_scopes.append(arena, next_arg); } else if (mem.eql(u8, arg, "--debug-pkg-config")) { debug_pkg_config = true; } else if (mem.eql(u8, arg, "--debug-rt")) { @@ -395,7 +372,7 @@ pub fn main(init: process.Init.Minimal) !void { fatalWithHint("unrecognized argument: '{s}'", .{arg}); } } else { - try targets.append(arg); + try step_names.append(arena, arg); } } @@ -408,6 +385,29 @@ pub fn main(init: process.Init.Minimal) !void { .off => .no_color, }; + const scanned_config: ScannedConfig = sc: { + const configuration = c: { + var file = cwd.openFile(io, configure_path, .{}) catch |err| + fatal("failed to open configuration file {s}: {t}", .{ configure_path, err }); + defer file.close(io); + break :c Configuration.loadFile(arena, io, file) catch |err| + fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); + }; + var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; + for (configuration.steps, 0..) |*conf_step, step_index_usize| { + const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); + const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]); + if (flags.tag == .top_level) { + const name = step_index.ptr(&configuration).name.slice(&configuration); + try top_level_steps.put(arena, name, step_index); + } + } + break :sc .{ + .configuration = configuration, + .top_level_steps = top_level_steps, + }; + }; + if (help_menu) { var w = initStdoutWriter(io); scanned_config.printUsage(&graph, w) catch |err| switch (err) { @@ -467,10 +467,17 @@ pub fn main(init: process.Init.Minimal) !void { .sub_path = cwd_relative, } else try install_prefix_path.join(arena, "include"); - if (true) @panic("TODO"); - var run: Run = .{ .gpa = gpa, + .graph = &graph, + .scanned_config = &scanned_config, + .install_paths = .{ + .prefix = install_prefix_path, + .lib = install_lib_path, + .bin = install_bin_path, + .include = install_include_path, + }, + .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), .available_rss = max_rss, .max_rss_is_default = false, @@ -486,13 +493,6 @@ pub fn main(init: process.Init.Minimal) !void { .error_style = error_style, .multiline_errors = multiline_errors, .summary = summary orelse if (watch or webui_listen != null) .line else .failures, - - .install_paths = .{ - .prefix = install_prefix_path, - .lib = install_lib_path, - .bin = install_bin_path, - .include = install_include_path, - }, }; defer { run.memory_blocked_steps.deinit(gpa); @@ -504,17 +504,16 @@ pub fn main(init: process.Init.Minimal) !void { run.max_rss_is_default = true; } - prepare(arena, &graph, targets.items, &run) catch |err| switch (err) { + run.prepare(step_names.items) catch |err| switch (err) { error.DependencyLoopDetected, error.InsufficientMemory => { - // Perhaps in the future there could be an Advanced Options flag - // such as --debug-build-runner-leaks which would make this code - // return instead of calling exit. _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(1); }, else => |e| return e, }; + if (true) @panic("TODO"); + var w: Watch = w: { if (!watch) break :w undefined; if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); @@ -547,7 +546,7 @@ pub fn main(init: process.Init.Minimal) !void { }) { if (run.web_server) |*ws| ws.startBuild(); - try runStepNames(graph, targets.items, main_progress_node, &run, fuzz); + try run.makeStepNames(step_names, main_progress_node, fuzz); if (run.web_server) |*web_server| { if (fuzz) |mode| if (mode != .forever) fatal( @@ -628,6 +627,10 @@ fn countSubProcesses(all_steps: []const *Step) usize { const Run = struct { gpa: Allocator, + graph: *Graph, + install_paths: InstallPaths, + scanned_config: *const ScannedConfig, + steps: []Step, available_rss: usize, max_rss_is_default: bool, @@ -637,309 +640,331 @@ const Run = struct { watch: bool, web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, /// Allocated into `gpa`. - memory_blocked_steps: std.ArrayList(*Step), + memory_blocked_steps: std.ArrayList(Configuration.Step.Index), /// Allocated into `gpa`. - step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), + step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), error_style: ErrorStyle, multiline_errors: MultilineErrors, summary: Summary, -}; -fn prepare(graph: *Graph, step_names: []const []const u8, run: *Run) !void { - const arena = graph.arena; - const seed: u32 = graph.random_seed; - const gpa = run.gpa; - const step_stack = &run.step_stack; + const InstallPaths = struct { + prefix: Path, + lib: Path, + bin: Path, + include: Path, + }; - if (step_names.len == 0) { - try step_stack.put(gpa, graph.configuration.default_step, {}); - } else { - try step_stack.ensureUnusedCapacity(gpa, step_names.len); - for (0..step_names.len) |i| { - const step_name = step_names[step_names.len - i - 1]; - const s = run.top_level_steps.get(step_name) orelse { - log.info("access the help menu with 'zig build -h'", .{}); - fatal("no such step: {s}", .{step_name}); - }; - step_stack.putAssumeCapacity(&s.step, {}); - } + fn stepByIndex(run: *const Run, i: Configuration.Step.Index) *Step { + return &run.steps[@intFromEnum(i)]; } - const starting_steps = try arena.dupe(*Step, step_stack.keys()); + fn prepare(run: *Run, step_names: []const []const u8) !void { + const gpa = run.gpa; + const graph = run.graph; + const arena = graph.arena; + const seed: u32 = graph.random_seed; + const step_stack = &run.step_stack; + const c = &run.scanned_config.configuration; - var rng = std.Random.DefaultPrng.init(seed); - const rand = rng.random(); - rand.shuffle(*Step, starting_steps); + @memset(run.steps, .{}); - for (starting_steps) |s| { - try constructGraphAndCheckForDependencyLoop(gpa, s, &run.step_stack, rand); - } - - { - // Check that we have enough memory to complete the build. - var any_problems = false; - var max_needed: usize = 0; - for (step_stack.keys()) |s| { - if (s.max_rss == 0) continue; - max_needed = @max(max_needed, s.max_rss); - if (s.max_rss > run.available_rss) { - if (run.skip_oom_steps) { - s.state = .skipped_oom; - for (s.dependants.items) |dependant| { - dependant.pending_deps -= 1; - } - } else { - std.log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ - s.owner.dep_prefix, s.name, s.max_rss, run.available_rss, - }); - any_problems = true; - } - } - } - if (any_problems) { - if (run.max_rss_is_default) { - std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ - max_needed, - }); - } - return error.InsufficientMemory; - } - } -} - -fn runStepNames( - graph: *Graph, - step_names: []const []const u8, - parent_prog_node: std.Progress.Node, - run: *Run, - fuzz: ?Fuzz.Mode, -) !void { - const gpa = run.gpa; - const io = graph.io; - const step_stack = &run.step_stack; - - { - // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, - // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking - // a step is initial when it actually became ready due to an earlier initial step. - var initial_set: std.ArrayList(*Step) = .empty; - defer initial_set.deinit(gpa); - try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); - for (step_stack.keys()) |s| { - if (s.state == .precheck_done and s.pending_deps == 0) { - initial_set.appendAssumeCapacity(s); - } - } - - const step_prog = parent_prog_node.start("steps", step_stack.count()); - defer step_prog.end(); - - var group: Io.Group = .init; - defer group.cancel(io); - // Start working on all of the initial steps... - for (initial_set.items) |s| try stepReady(&group, s, step_prog, run); - // ...and `makeStep` will trigger every other step when their last dependency finishes. - try group.await(io); - } - - assert(run.memory_blocked_steps.items.len == 0); - - var test_pass_count: usize = 0; - var test_skip_count: usize = 0; - var test_fail_count: usize = 0; - var test_crash_count: usize = 0; - var test_timeout_count: usize = 0; - - var test_count: usize = 0; - - var success_count: usize = 0; - var skipped_count: usize = 0; - var failure_count: usize = 0; - var pending_count: usize = 0; - var total_compile_errors: usize = 0; - - var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); - defer cleanup_task.await(io); - - for (step_stack.keys()) |s| { - test_pass_count += s.test_results.passCount(); - test_skip_count += s.test_results.skip_count; - test_fail_count += s.test_results.fail_count; - test_crash_count += s.test_results.crash_count; - test_timeout_count += s.test_results.timeout_count; - - test_count += s.test_results.test_count; - - switch (s.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - .dependency_failure => pending_count += 1, - .success => success_count += 1, - .skipped, .skipped_oom => skipped_count += 1, - .failure => { - failure_count += 1; - const compile_errors_len = s.result_error_bundle.errorMessageCount(); - if (compile_errors_len > 0) { - total_compile_errors += compile_errors_len; - } - }, - } - } - - if (fuzz) |mode| blk: { - switch (builtin.os.tag) { - // Current implementation depends on two things that need to be ported to Windows: - // * Memory-mapping to share data between the fuzzer and build runner. - // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving - // many addresses to source locations). - .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), - else => {}, - } - if (@bitSizeOf(usize) != 64) { - // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, - // being compatible with file system's u64 return value. This is not the case - // on 32-bit platforms. - // Affects or affected by issues #5185, #22523, and #22464. - fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); - } - - switch (mode) { - .forever => break :blk, - .limit => {}, - } - - assert(mode == .limit); - var f = Fuzz.init( - gpa, - io, - step_stack.keys(), - parent_prog_node, - mode, - ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); - defer f.deinit(); - - f.start(); - try f.waitAndPrintReport(); - } - - // Every test has a state - assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); - - if (failure_count == 0) { - std.Progress.setStatus(.success); - } else { - std.Progress.setStatus(.failure); - } - - summary: { - switch (run.summary) { - .all, .new, .line => {}, - .failures => if (failure_count == 0) break :summary, - .none => break :summary, - } - - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - const t = stderr.terminal(); - const w = &stderr.file_writer.interface; - - const total_count = success_count + failure_count + pending_count + skipped_count; - t.setColor(.cyan) catch {}; - t.setColor(.bold) catch {}; - w.writeAll("Build Summary: ") catch {}; - t.setColor(.reset) catch {}; - w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; - { - t.setColor(.dim) catch {}; - var first = true; - if (skipped_count > 0) { - w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; - first = false; - } - if (failure_count > 0) { - w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; - first = false; - } - if (!first) w.writeByte(')') catch {}; - t.setColor(.reset) catch {}; - } - - if (test_count > 0) { - w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; - t.setColor(.dim) catch {}; - var first = true; - if (test_skip_count > 0) { - w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; - first = false; - } - if (test_fail_count > 0) { - w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; - first = false; - } - if (test_crash_count > 0) { - w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; - first = false; - } - if (test_timeout_count > 0) { - w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; - first = false; - } - if (!first) w.writeByte(')') catch {}; - t.setColor(.reset) catch {}; - } - - w.writeAll("\n") catch {}; - - if (run.summary == .line) break :summary; - - // Print a fancy tree with build results. - var step_stack_copy = try step_stack.clone(gpa); - defer step_stack_copy.deinit(gpa); - - var print_node: PrintNode = .{ .parent = null }; if (step_names.len == 0) { - print_node.last = true; - printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {}; + try step_stack.put(gpa, c.default_step, {}); } else { - const last_index = if (run.summary == .all) run.top_level_steps.count() else blk: { - var i: usize = step_names.len; - while (i > 0) { - i -= 1; - const step = run.top_level_steps.get(step_names[i]).?.step; - const found = switch (run.summary) { - .all, .line, .none => unreachable, - .failures => step.state != .success, - .new => !step.result_cached, - }; - if (found) break :blk i; - } - break :blk run.top_level_steps.count(); - }; - for (step_names, 0..) |step_name, i| { - const tls = run.top_level_steps.get(step_name).?; - print_node.last = i + 1 == last_index; - printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; - } - } - w.writeByte('\n') catch {}; + try step_stack.ensureUnusedCapacity(gpa, step_names.len); + for (0..step_names.len) |i| { + const step_name = step_names[step_names.len - i - 1]; + const s = run.scanned_config.top_level_steps.get(step_name) orelse { + log.info("to list available steps: zig build -l", .{}); + fatal("no such step: {s}", .{step_name}); + }; + step_stack.putAssumeCapacity(s, {}); + } + } + + const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys()); + + var rng = std.Random.DefaultPrng.init(seed); + const rand = rng.random(); + rand.shuffle(Configuration.Step.Index, starting_steps); + + for (starting_steps) |s| { + try constructGraphAndCheckForDependencyLoop(gpa, c, run.steps, s, &run.step_stack, rand); + } + + { + // Check that we have enough memory to complete the build. + var any_problems = false; + var max_needed: usize = 0; + for (step_stack.keys()) |step_index| { + const make_step = run.stepByIndex(step_index); + const conf_step = step_index.ptr(c); + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss == 0) continue; + max_needed = @max(max_needed, max_rss); + if (max_rss > run.available_rss) { + if (run.skip_oom_steps) { + make_step.state = .skipped_oom; + for (make_step.dependants.items) |dependant| { + dependant.pending_deps -= 1; + } + } else { + log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ + conf_step.owner.depPrefixSlice(c), + conf_step.name.slice(c), + max_rss, + run.available_rss, + }); + any_problems = true; + } + } + } + if (any_problems) { + if (run.max_rss_is_default) { + std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ + max_needed, + }); + } + return error.InsufficientMemory; + } + } } - if (run.watch or run.web_server != null) return; + fn makeStepNames( + run: *Run, + step_names: []const []const u8, + parent_prog_node: std.Progress.Node, + fuzz: ?Fuzz.Mode, + ) !void { + const graph = run.graph; + const gpa = run.gpa; + const io = graph.io; + const step_stack = &run.step_stack; + const top_level_steps = &run.scanned_config.top_level_steps; + + { + // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, + // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking + // a step is initial when it actually became ready due to an earlier initial step. + var initial_set: std.ArrayList(*Step) = .empty; + defer initial_set.deinit(gpa); + try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); + for (step_stack.keys()) |s| { + if (s.state == .precheck_done and s.pending_deps == 0) { + initial_set.appendAssumeCapacity(s); + } + } + + const step_prog = parent_prog_node.start("steps", step_stack.count()); + defer step_prog.end(); + + var group: Io.Group = .init; + defer group.cancel(io); + // Start working on all of the initial steps... + for (initial_set.items) |s| try stepReady(&group, s, step_prog, run); + // ...and `makeStep` will trigger every other step when their last dependency finishes. + try group.await(io); + } + + assert(run.memory_blocked_steps.items.len == 0); + + var test_pass_count: usize = 0; + var test_skip_count: usize = 0; + var test_fail_count: usize = 0; + var test_crash_count: usize = 0; + var test_timeout_count: usize = 0; + + var test_count: usize = 0; + + var success_count: usize = 0; + var skipped_count: usize = 0; + var failure_count: usize = 0; + var pending_count: usize = 0; + var total_compile_errors: usize = 0; + + var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); + defer cleanup_task.await(io); + + for (step_stack.keys()) |s| { + test_pass_count += s.test_results.passCount(); + test_skip_count += s.test_results.skip_count; + test_fail_count += s.test_results.fail_count; + test_crash_count += s.test_results.crash_count; + test_timeout_count += s.test_results.timeout_count; + + test_count += s.test_results.test_count; + + switch (s.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + .dependency_failure => pending_count += 1, + .success => success_count += 1, + .skipped, .skipped_oom => skipped_count += 1, + .failure => { + failure_count += 1; + const compile_errors_len = s.result_error_bundle.errorMessageCount(); + if (compile_errors_len > 0) { + total_compile_errors += compile_errors_len; + } + }, + } + } + + if (fuzz) |mode| blk: { + switch (builtin.os.tag) { + // Current implementation depends on two things that need to be ported to Windows: + // * Memory-mapping to share data between the fuzzer and build runner. + // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving + // many addresses to source locations). + .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), + else => {}, + } + if (@bitSizeOf(usize) != 64) { + // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, + // being compatible with file system's u64 return value. This is not the case + // on 32-bit platforms. + // Affects or affected by issues #5185, #22523, and #22464. + fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); + } + + switch (mode) { + .forever => break :blk, + .limit => {}, + } + + assert(mode == .limit); + var f = Fuzz.init( + gpa, + io, + step_stack.keys(), + parent_prog_node, + mode, + ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); + defer f.deinit(); + + f.start(); + try f.waitAndPrintReport(); + } + + // Every test has a state + assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); + + if (failure_count == 0) { + std.Progress.setStatus(.success); + } else { + std.Progress.setStatus(.failure); + } + + summary: { + switch (run.summary) { + .all, .new, .line => {}, + .failures => if (failure_count == 0) break :summary, + .none => break :summary, + } + + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + const t = stderr.terminal(); + const w = &stderr.file_writer.interface; - // Perhaps in the future there could be an Advanced Options flag such as - // --debug-build-runner-leaks which would make this code return instead of - // calling exit. + const total_count = success_count + failure_count + pending_count + skipped_count; + t.setColor(.cyan) catch {}; + t.setColor(.bold) catch {}; + w.writeAll("Build Summary: ") catch {}; + t.setColor(.reset) catch {}; + w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; + { + t.setColor(.dim) catch {}; + var first = true; + if (skipped_count > 0) { + w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; + first = false; + } + if (failure_count > 0) { + w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; + first = false; + } + if (!first) w.writeByte(')') catch {}; + t.setColor(.reset) catch {}; + } - const code: u8 = code: { - if (failure_count == 0) break :code 0; // success - if (run.error_style.verboseContext()) break :code 1; // failure; print build command - break :code 2; // failure; do not print build command - }; - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(code); -} + if (test_count > 0) { + w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; + t.setColor(.dim) catch {}; + var first = true; + if (test_skip_count > 0) { + w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; + first = false; + } + if (test_fail_count > 0) { + w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; + first = false; + } + if (test_crash_count > 0) { + w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; + first = false; + } + if (test_timeout_count > 0) { + w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; + first = false; + } + if (!first) w.writeByte(')') catch {}; + t.setColor(.reset) catch {}; + } + + w.writeAll("\n") catch {}; + + if (run.summary == .line) break :summary; + + // Print a fancy tree with build results. + var step_stack_copy = try step_stack.clone(gpa); + defer step_stack_copy.deinit(gpa); + + var print_node: PrintNode = .{ .parent = null }; + if (step_names.len == 0) { + print_node.last = true; + printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {}; + } else { + const last_index = if (run.summary == .all) top_level_steps.count() else blk: { + var i: usize = step_names.len; + while (i > 0) { + i -= 1; + const step = top_level_steps.get(step_names[i]).?.step; + const found = switch (run.summary) { + .all, .line, .none => unreachable, + .failures => step.state != .success, + .new => !step.result_cached, + }; + if (found) break :blk i; + } + break :blk top_level_steps.count(); + }; + for (step_names, 0..) |step_name, i| { + const tls = top_level_steps.get(step_name).?; + print_node.last = i + 1 == last_index; + printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; + } + } + w.writeByte('\n') catch {}; + } + + if (run.watch or run.web_server != null) return; + + // Perhaps in the future there could be an Advanced Options flag such as + // --debug-build-runner-leaks which would make this code return instead of + // calling exit. + + const code: u8 = code: { + if (failure_count == 0) break :code 0; // success + if (run.error_style.verboseContext()) break :code 1; // failure; print build command + break :code 2; // failure; do not print build command + }; + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(code); + } +}; const PrintNode = struct { parent: ?*PrintNode, @@ -1221,40 +1246,47 @@ fn printTreeStep( /// random order fn constructGraphAndCheckForDependencyLoop( gpa: Allocator, - s: *Step, - step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), + c: *const Configuration, + steps: []Step, + step_index: Configuration.Step.Index, + step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), rand: std.Random, -) !void { +) error{ DependencyLoopDetected, OutOfMemory }!void { + const s: *Step = &steps[@intFromEnum(step_index)]; switch (s.state) { .precheck_started => { - std.debug.print("dependency loop detected:\n {s}\n", .{s.name}); + log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)}); return error.DependencyLoopDetected; }, .precheck_unstarted => { s.state = .precheck_started; - try step_stack.ensureUnusedCapacity(gpa, s.dependencies.items.len); + const step = step_index.ptr(c); + const dependencies = step.deps.slice(c); + try step_stack.ensureUnusedCapacity(gpa, dependencies.len); // We dupe to avoid shuffling the steps in the summary, it depends - // on s.dependencies' order. - const deps = try gpa.dupe(*Step, s.dependencies.items); + // on dependencies' order. + const deps = try gpa.dupe(Configuration.Step.Index, dependencies); defer gpa.free(deps); - rand.shuffle(*Step, deps); + rand.shuffle(Configuration.Step.Index, deps); for (deps) |dep| { + const dep_step: *Step = &steps[@intFromEnum(dep)]; try step_stack.put(gpa, dep, {}); - try dep.dependants.append(gpa, s); - constructGraphAndCheckForDependencyLoop(gpa, dep, step_stack, rand) catch |err| { - if (err == error.DependencyLoopDetected) { - std.debug.print(" {s}\n", .{s.name}); - } - return err; + try dep_step.dependants.append(gpa, s); + constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) { + error.DependencyLoopDetected => { + log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); + return err; + }, + else => return err, }; } s.state = .precheck_done; - s.pending_deps = @intCast(s.dependencies.items.len); + s.pending_deps = @intCast(dependencies.len); }, .precheck_done => {}, @@ -1492,8 +1524,7 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { return nextArg(args, idx) orelse { - log.info("access the help menu with \"zig build -h\"", .{}); - fatal("expected argument after {q}", .{args[idx.* - 1]}); + fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); }; } @@ -1532,7 +1563,7 @@ const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { - log.info("access the help menu with 'zig build -h'", .{}); + log.info("to access the help menu: zig build -h", .{}); fatal(f, args); } @@ -1547,13 +1578,6 @@ fn cleanTmpFiles(io: Io, steps: []const *Step) void { } } -const InstallPaths = struct { - prefix: Path, - lib: Path, - bin: Path, - include: Path, -}; - var stdio_buffer_allocation: [256]u8 = undefined; var stdout_writer_allocation: Io.File.Writer = undefined; @@ -1564,17 +1588,20 @@ fn initStdoutWriter(io: Io) *Writer { const ScannedConfig = struct { configuration: Configuration, - top_level_steps: []const Configuration.Step.Index, + top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { + const c = &sc.configuration; var serializer: std.zon.Serializer = .{ .writer = w }; var s = try serializer.beginStruct(.{}); - try s.field("default_step", @intFromEnum(sc.configuration.default_step), .{}); + try s.field("default_step", @intFromEnum(c.default_step), .{}); { - var tuple = try s.beginTupleField("top_level_steps", .{}); - for (sc.top_level_steps) |step| try tuple.field(@intFromEnum(step), .{}); - try tuple.end(); + var ss = try s.beginStructField("top_level_steps", .{}); + for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| { + try ss.field(name, @intFromEnum(step), .{}); + } + try ss.end(); } try s.end(); @@ -1583,9 +1610,8 @@ const ScannedConfig = struct { fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { const arena = graph.arena; const c = &sc.configuration; - for (sc.top_level_steps) |step_index| { + for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| { const step = step_index.ptr(c); - const name = step.name.slice(c); const decorated_name = if (step_index == c.default_step) try fmt.allocPrint(arena, "{s} (default)", .{name}) else @@ -1679,8 +1705,8 @@ const ScannedConfig = struct { try w.writeAll( \\ \\General Options: - \\ -h, --help Print this help and exit - \\ -l, --list-steps Print available steps + \\ -h, --help Print this help to stdout and exit + \\ -l, --list-steps Print available steps to stdout and exit \\ \\ -p, --prefix [path] Where to install files (default: zig-out) \\ --prefix-lib-dir [path] Where to install libraries diff --git a/lib/compiler/maker/Package.zig b/lib/compiler/maker/Package.zig deleted file mode 100644 index 42e3b47d29cf46cae53f592f0e884ef8c931487f..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Package.zig +++ /dev/null @@ -1,30 +0,0 @@ -const Package = @This(); - -const std = @import("std"); - -install_prefix: []const u8, -install_path: []const u8, -dest_dir: ?[]const u8, -lib_dir: []const u8, -exe_dir: []const u8, -h_dir: []const u8, -/// Path to the directory containing build.zig. -build_root: std.Build.Cache.Path, - -fn determineAndApplyInstallPrefix(p: *Package) error{OutOfMemory}!void { - // Create an installation directory local to this package. This will be used when - // dependant packages require a standard prefix, such as include directories for C headers. - var hash = p.graph.cache.hash; - // Random bytes to make unique. Refresh this with new random bytes when - // implementation is modified in a non-backwards-compatible way. - hash.add(@as(u32, 0xd8cb0056)); - hash.addBytes(p.dep_prefix); - - var wyhash = std.hash.Wyhash.init(0); - hashUserInputOptionsMap(p.allocator, p.user_input_options, &wyhash); - hash.add(wyhash.final()); - - const digest = hash.final(); - const install_prefix = try p.cache_root.join(p.allocator, &.{ "i", &digest }); - p.resolveInstallPrefix(install_prefix, .{}); -} diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/maker/Step.zig index 164510f3dd8d56d5477585c0226fc543940deb42..9ede48eb2c7be2c9ffd0a8cc8d86feee937f6e0a 100644 --- a/lib/compiler/maker/Step.zig +++ b/lib/compiler/maker/Step.zig @@ -1,19 +1,27 @@ +//! The state that maker needs in order to process a step. const Step = @This(); +const builtin = @import("builtin"); + const std = @import("std"); -const Io = std.Io; const Allocator = std.mem.Allocator; const Cache = std.Build.Cache; +const Io = std.Io; +const LazyPath = std.Build.Configuration.LazyPath; +const Package = std.Build.Configuration.Package; +const Path = std.Build.Cache.Path; const assert = std.debug.assert; const WebServer = @import("WebServer.zig"); -pub const Compile = @import("Step/Compile.zig"); -pub const Run = @import("Step/Run.zig"); +pub const Compile = void; // @import("Step/Compile.zig"); +pub const Run = void; // @import("Step/Run.zig"); -state: State, -makeFn: MakeFn, -dependants: std.ArrayList(*Step), +/// Avoid false sharing. +_: void align(std.atomic.cache_line) = {}, + +state: State = .precheck_unstarted, +dependants: std.ArrayList(*Step) = .empty, /// Collects the set of files that retrigger this step to run. /// /// This is used by the build system's implementation of `--watch` but it can @@ -23,20 +31,19 @@ dependants: std.ArrayList(*Step), /// Populated within `make`. Implementation may choose to clear and repopulate, /// retain previous value, or update. inputs: Inputs = .init, -pending_deps: u32, +pending_deps: u32 = undefined, -result_error_msgs: std.ArrayList([]const u8), -result_error_bundle: std.zig.ErrorBundle, -result_stderr: []const u8, -result_cached: bool, -result_duration_ns: ?u64, +result_error_msgs: std.ArrayList([]const u8) = .empty, +result_error_bundle: std.zig.ErrorBundle = .empty, +result_stderr: []const u8 = "", +result_cached: bool = false, +result_duration_ns: ?u64 = null, /// 0 means unavailable or not reported. -result_peak_rss: usize, +result_peak_rss: usize = 0, /// If the step is failed and this field is populated, this is the command which failed. /// This field may be populated even if the step succeeded. -result_failed_command: ?[]const u8, -test_results: TestResults, - +result_failed_command: ?[]const u8 = null, +test_results: TestResults = .{}, pub const State = enum { precheck_unstarted, @@ -172,18 +179,6 @@ pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!voi } } -fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { - _ = options; - - var all_cached = true; - - for (step.dependencies.items) |dep| { - all_cached = all_cached and dep.result_cached; - } - - step.result_cached = all_cached; -} - /// Implementation detail of file watching. Prepares the step for being re-evaluated. /// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. pub fn invalidateResult(step: *Step, gpa: Allocator) bool { @@ -233,7 +228,7 @@ pub fn captureChildProcess( s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); try handleChildProcUnsupported(s); - try handleVerbose(s.owner, .inherit, argv); + try handleVerbose(s, .inherit, argv); const result = std.process.run(arena, io, .{ .argv = argv, @@ -340,7 +335,7 @@ pub fn evalZigProcess( assert(argv.len != 0); try handleChildProcUnsupported(s); - try handleVerbose(s.owner, .inherit, argv); + try handleVerbose(s, .inherit, argv); const zp = try gpa.create(ZigProcess); defer if (!watch) gpa.destroy(zp); @@ -399,11 +394,11 @@ pub fn evalZigProcess( } /// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. -pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { +pub fn installFile(s: *Step, src_lazy_path: LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { const b = s.owner; const io = b.graph.io; const src_path = src_lazy_path.getPath3(b, s); - try handleVerbose(b, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); + try handleVerbose(s, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); } @@ -412,7 +407,7 @@ pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { const b = s.owner; const io = b.graph.io; - try handleVerbose(b, .inherit, &.{ "install", "-d", dest_path }); + try handleVerbose(s, .inherit, &.{ "install", "-d", dest_path }); return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); } @@ -567,29 +562,21 @@ fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { } pub fn handleVerbose( - b: *Build, - cwd: std.process.Child.Cwd, - argv: []const []const u8, -) error{OutOfMemory}!void { - return handleVerbose2(b, cwd, null, argv); -} - -pub fn handleVerbose2( - b: *Build, + s: *Step, + arena: Allocator, cwd: std.process.Child.Cwd, opt_env: ?*const std.process.Environ.Map, argv: []const []const u8, ) error{OutOfMemory}!void { - if (b.verbose) { - const graph = b.graph; - // Intention of verbose is to print all sub-process command lines to - // stderr before spawning them. - const text = try allocPrintCmd(b.allocator, cwd, if (opt_env) |env| .{ - .child = env, - .parent = &graph.environ_map, - } else null, argv); - std.debug.print("{s}\n", .{text}); - } + if (!s.verbose) return; + const graph = s.graph; + // Intention of verbose is to print all sub-process command lines to + // stderr before spawning them. + const text = try allocPrintCmd(arena, cwd, if (opt_env) |env| .{ + .child = env, + .parent = &graph.environ_map, + } else null, argv); + std.log.scoped(.verbose).info("{s}", .{text}); } /// Asserts that the caller has already populated `s.result_failed_command`. @@ -688,7 +675,7 @@ fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void { } /// For steps that have a single input that never changes when re-running `make`. -pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { +pub fn singleUnchangingWatchInput(step: *Step, lazy_path: LazyPath) Allocator.Error!void { if (!step.inputs.populated()) try step.addWatchInput(lazy_path); } @@ -698,7 +685,7 @@ pub fn clearWatchInputs(step: *Step) void { } /// Places a *file* dependency on the path. -pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!void { +pub fn addWatchInput(step: *Step, lazy_file: LazyPath) Allocator.Error!void { switch (lazy_file) { .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), @@ -723,7 +710,7 @@ pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!voi /// Paths derived from this directory should also be manually added via /// `addDirectoryWatchInputFromPath` if and only if this function returns /// `true`. -pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool { +pub fn addDirectoryWatchInput(step: *Step, lazy_directory: LazyPath) Allocator.Error!bool { switch (lazy_directory) { .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), @@ -744,26 +731,26 @@ pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Alloc /// Any changes inside the directory will trigger invalidation. /// -/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. +/// See also `addDirectoryWatchInput` which takes a `LazyPath` instead. /// /// This function should only be called when it has been verified that the /// dependency on `path` is not already accounted for by a `Step` dependency. /// In other words, before calling this function, first check that the -/// `Build.LazyPath` which this `path` is derived from is not `generated`. +/// `LazyPath` which this `path` is derived from is not `generated`. pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void { return addWatchInputFromPath(step, path, "."); } -fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { +fn addWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { return addWatchInputFromPath(step, .{ - .root_dir = builder.build_root, + .root_dir = package.build_root, .sub_path = std.fs.path.dirname(sub_path) orelse "", }, std.fs.path.basename(sub_path)); } -fn addDirectoryWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { +fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { return addDirectoryWatchInputFromPath(step, .{ - .root_dir = builder.build_root, + .root_dir = package.build_root, .sub_path = sub_path, }); } @@ -847,16 +834,3 @@ pub fn allocPrintCmd( } return aw.toOwnedSlice(); } - -pub fn getInstallPath(b: *Build, dir: InstallDir, dest_rel_path: []const u8) []const u8 { - assert(!fs.path.isAbsolute(dest_rel_path)); // Install paths must be relative to the prefix - const base_dir = switch (dir) { - .prefix => b.install_path, - .bin => b.exe_dir, - .lib => b.lib_dir, - .header => b.h_dir, - .custom => |p| b.pathJoin(&.{ b.install_path, p }), - }; - return b.pathResolve(&.{ base_dir, dest_rel_path }); -} - diff --git a/lib/std/Build.zig b/lib/std/Build.zig index c36e93e8118c67e7f74c314290cb47df09fc5778..a6668c5c1daa5fde7ea2c52ff4433023e74c60eb 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -112,6 +112,10 @@ pub const Graph = struct { /// respects the '--color' flag. stderr_mode: ?Io.Terminal.Mode = null, release_mode: ReleaseMode = .off, + /// Whether the user passed in "--" arguments. They can be added to a child + /// process via `Step.Run` API but cannot be observed in the configure + /// phase. + have_run_args: bool = false, }; const AvailableDeps = []const struct { []const u8, []const u8 }; diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 8254a7aaf397439bf89c97d93f5def2accf6e894..73aeba93218ee65d9cc5d69cc5d4035907054d30 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -141,6 +141,8 @@ pub const Arg = union(enum) { bytes: []u8, output_file: *Output, output_directory: *Output, + /// The arguments passed after "--" on the "zig build" CLI. + cli_rest_positionals, }; pub const PrefixedArtifact = struct { diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 0c4d45b5f6f6ca166609d69006677484ba5502bf..69061bdb136f607c9186951e9ad61b00b51571bc 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -413,6 +413,7 @@ pub const AvailableOption = extern struct { pub const Step = extern struct { name: String, + owner: Package.Index, deps: Deps, max_rss: MaxRss, /// Points into `extra` for step-specific data. First element has flags @@ -534,6 +535,7 @@ pub const Step = extern struct { bytes, output_file, output_directory, + cli_rest_positionals, }; }; @@ -841,7 +843,7 @@ pub const LazyPath = enum(u32) { pub const SourcePath = struct { flags: Flags, - owner: Package, + owner: Package.Index, sub_path: String, pub const Flags = packed struct(u32) { @@ -877,16 +879,19 @@ pub const LazyPath = enum(u32) { }; }; -/// It's an OptionalString which points to the package hash. -pub const Package = enum(u32) { - root = maxInt(u32), - _, +pub const Package = struct { + dep_prefix: String, + hash: String, - pub fn fromHash(hash: String) Package { - const result: Package = @enumFromInt(@intFromEnum(hash)); - assert(result != .root); - return result; - } + pub const Index = enum(u32) { + root = maxInt(u32), + _, + + pub fn depPrefixSlice(i: Index, c: *const Configuration) [:0]const u8 { + if (i == .root) return ""; + return extraData(c, Package, @intFromEnum(i)).dep_prefix.slice(c); + } + }; }; /// Trailing: @@ -900,7 +905,7 @@ pub const Package = enum(u32) { pub const Module = struct { flags: Flags, flags2: Flags2, - owner: Package, + owner: Package.Index, root_source_file: OptionalLazyPath, import_table: ImportTable, resolved_target: ResolvedTarget.OptionalIndex, @@ -1048,6 +1053,11 @@ pub const ImportTable = enum(u32) { /// elements is `Step.Index` per count. pub const Deps = enum(u32) { _, + + pub fn slice(deps: Deps, c: *const Configuration) []Step.Index { + const len = c.extra[@intFromEnum(deps)]; + return @ptrCast(c.extra[@intFromEnum(deps) + 1 ..][0..len]); + } }; /// Points into `extra`, where the first element is count of strings, following -- 2.54.0 From 3262698fb171fb2ed33cf6786c0573e922ce9a80 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 18 Feb 2026 13:12:18 -0800 Subject: [PATCH 011/179] make runner: execute step graph --- lib/compiler/maker.zig | 768 +++++++++++++++++-------------- lib/compiler/maker/Fuzz.zig | 55 ++- lib/compiler/maker/Step.zig | 6 +- lib/compiler/maker/Watch.zig | 52 ++- lib/compiler/maker/WebServer.zig | 61 ++- 5 files changed, 517 insertions(+), 425 deletions(-) diff --git a/lib/compiler/maker.zig b/lib/compiler/maker.zig index d35d0584847593fdfc2a38b428c90ac8906a442d..a94416c54070b097007a537b11f38fa83ec35b0f 100644 --- a/lib/compiler/maker.zig +++ b/lib/compiler/maker.zig @@ -512,12 +512,10 @@ pub fn main(init: process.Init.Minimal) !void { else => |e| return e, }; - if (true) @panic("TODO"); - var w: Watch = w: { if (!watch) break :w undefined; if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); - break :w try .init(graph.cache.cwd); + break :w try .init(graph.cache.cwd, &scanned_config.configuration, run.steps); }; const now = Io.Clock.Timestamp.now(io, .awake); @@ -532,6 +530,7 @@ pub fn main(init: process.Init.Minimal) !void { .watch = watch, .listen_address = listen_address, .base_timestamp = now, + .configuration = &scanned_config.configuration, }); } else null; @@ -546,7 +545,7 @@ pub fn main(init: process.Init.Minimal) !void { }) { if (run.web_server) |*ws| ws.startBuild(); - try run.makeStepNames(step_names, main_progress_node, fuzz); + try run.makeStepNames(step_names.items, main_progress_node, fuzz); if (run.web_server) |*web_server| { if (fuzz) |mode| if (mode != .forever) fatal( @@ -558,12 +557,15 @@ pub fn main(init: process.Init.Minimal) !void { } if (run.web_server) |*ws| { + const c = &scanned_config.configuration; assert(!watch); // fatal error after CLI parsing while (true) switch (try ws.wait()) { .rebuild => { - for (run.step_stack.keys()) |step| { + for (run.step_stack.keys()) |step_index| { + const step = run.stepByIndex(step_index); step.state = .precheck_done; - step.pending_deps = @intCast(step.dependencies.items.len); + const deps = step_index.ptr(c).deps.slice(c); + step.pending_deps = @intCast(deps.len); step.reset(gpa); } continue :rebuild; @@ -583,7 +585,7 @@ pub fn main(init: process.Init.Minimal) !void { // recursive dependants. var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ - w.dir_count, countSubProcesses(run.step_stack.keys()), + w.dir_count, countSubProcesses(run.steps, run.step_stack.keys()), }) catch &caption_buf; var debouncing_node = main_progress_node.start(caption, 0); var in_debounce = false; @@ -591,7 +593,7 @@ pub fn main(init: process.Init.Minimal) !void { .timeout => { assert(in_debounce); debouncing_node.end(); - markFailedStepsDirty(gpa, run.step_stack.keys()); + markFailedStepsDirty(gpa, run.steps, run.step_stack.keys()); continue :rebuild; }, .dirty => if (!in_debounce) { @@ -604,22 +606,29 @@ pub fn main(init: process.Init.Minimal) !void { } } -fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { - for (all_steps) |step| switch (step.state) { - .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), - else => continue, - }; +fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void { + for (all_steps) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + switch (step.state) { + .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), + else => continue, + } + } // Now that all dirty steps have been found, the remaining steps that // succeeded from last run shall be marked "cached". - for (all_steps) |step| switch (step.state) { - .success => step.result_cached = true, - else => continue, - }; + for (all_steps) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + switch (step.state) { + .success => step.result_cached = true, + else => continue, + } + } } -fn countSubProcesses(all_steps: []const *Step) usize { +fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize { var count: usize = 0; - for (all_steps) |s| { + for (all_steps) |step_index| { + const s = &make_steps[@intFromEnum(step_index)]; count += @intFromBool(s.getZigProcess() != null); } return count; @@ -707,7 +716,7 @@ const Run = struct { if (run.skip_oom_steps) { make_step.state = .skipped_oom; for (make_step.dependants.items) |dependant| { - dependant.pending_deps -= 1; + run.stepByIndex(dependant).pending_deps -= 1; } } else { log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ @@ -742,17 +751,19 @@ const Run = struct { const io = graph.io; const step_stack = &run.step_stack; const top_level_steps = &run.scanned_config.top_level_steps; + const c = &run.scanned_config.configuration; { // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking // a step is initial when it actually became ready due to an earlier initial step. - var initial_set: std.ArrayList(*Step) = .empty; + var initial_set: std.ArrayList(Configuration.Step.Index) = .empty; defer initial_set.deinit(gpa); try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); - for (step_stack.keys()) |s| { + for (step_stack.keys()) |step_index| { + const s = run.stepByIndex(step_index); if (s.state == .precheck_done and s.pending_deps == 0) { - initial_set.appendAssumeCapacity(s); + initial_set.appendAssumeCapacity(step_index); } } @@ -762,7 +773,7 @@ const Run = struct { var group: Io.Group = .init; defer group.cancel(io); // Start working on all of the initial steps... - for (initial_set.items) |s| try stepReady(&group, s, step_prog, run); + for (initial_set.items) |step_index| try stepReady(run, &group, step_index, step_prog); // ...and `makeStep` will trigger every other step when their last dependency finishes. try group.await(io); } @@ -786,16 +797,17 @@ const Run = struct { var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); defer cleanup_task.await(io); - for (step_stack.keys()) |s| { - test_pass_count += s.test_results.passCount(); - test_skip_count += s.test_results.skip_count; - test_fail_count += s.test_results.fail_count; - test_crash_count += s.test_results.crash_count; - test_timeout_count += s.test_results.timeout_count; + for (step_stack.keys()) |step_index| { + const make_step = run.stepByIndex(step_index); + test_pass_count += make_step.test_results.passCount(); + test_skip_count += make_step.test_results.skip_count; + test_fail_count += make_step.test_results.fail_count; + test_crash_count += make_step.test_results.crash_count; + test_timeout_count += make_step.test_results.timeout_count; - test_count += s.test_results.test_count; + test_count += make_step.test_results.test_count; - switch (s.state) { + switch (make_step.state) { .precheck_unstarted => unreachable, .precheck_started => unreachable, .precheck_done => unreachable, @@ -804,7 +816,7 @@ const Run = struct { .skipped, .skipped_oom => skipped_count += 1, .failure => { failure_count += 1; - const compile_errors_len = s.result_error_bundle.errorMessageCount(); + const compile_errors_len = make_step.result_error_bundle.errorMessageCount(); if (compile_errors_len > 0) { total_compile_errors += compile_errors_len; } @@ -925,13 +937,17 @@ const Run = struct { var print_node: PrintNode = .{ .parent = null }; if (step_names.len == 0) { print_node.last = true; - printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {}; + printTreeStep(run, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, + }; } else { const last_index = if (run.summary == .all) top_level_steps.count() else blk: { var i: usize = step_names.len; while (i > 0) { i -= 1; - const step = top_level_steps.get(step_names[i]).?.step; + const step_index = top_level_steps.get(step_names[i]).?; + const step = run.stepByIndex(step_index); const found = switch (run.summary) { .all, .line, .none => unreachable, .failures => step.state != .success, @@ -942,9 +958,12 @@ const Run = struct { break :blk top_level_steps.count(); }; for (step_names, 0..) |step_name, i| { - const tls = top_level_steps.get(step_name).?; + const step_index = top_level_steps.get(step_name).?; print_node.last = i + 1 == last_index; - printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; + printTreeStep(run, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, + }; } } w.writeByte('\n') catch {}; @@ -964,120 +983,331 @@ const Run = struct { _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(code); } -}; -const PrintNode = struct { - parent: ?*PrintNode, - last: bool = false, -}; + fn stepReady( + run: *Run, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, + ) Io.Cancelable!void { + const graph = run.graph; + const io = graph.io; + const c = &run.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + if (max_rss != 0) { + try run.max_rss_mutex.lock(io); + defer run.max_rss_mutex.unlock(io); + if (run.available_rss < max_rss) { + // Running this step right now could possibly exceed the allotted RSS. + run.memory_blocked_steps.append(run.gpa, step_index) catch + @panic("TODO eliminate memory allocation here"); + return; + } + run.available_rss -= max_rss; + } + group.async(io, makeStep, .{ run, group, step_index, root_prog_node }); + } + + /// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready + /// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must + /// have already subtracted this value from `run.available_rss`. This function will release the RSS + /// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked + /// steps after "make" completes for `s`. + fn makeStep( + run: *Run, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, + ) Io.Cancelable!void { + const graph = run.graph; + const io = graph.io; + const gpa = run.gpa; + const c = &run.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const step_name = conf_step.name.slice(c); + const deps = conf_step.deps.slice(c); + const make_step = run.stepByIndex(step_index); + + { + const step_prog_node = root_prog_node.start(step_name, 0); + defer step_prog_node.end(); + + if (run.web_server) |*ws| ws.updateStepStatus(step_index, .wip); + + const new_state: Step.State = for (deps) |dep_index| { + const dep_make_step = run.stepByIndex(dep_index); + switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, -fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { - const parent = node.parent orelse return; - const writer = stderr.writer; - if (parent.parent == null) return; - try printPrefix(parent, stderr); - if (parent.last) { - try writer.writeAll(" "); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ - else => "| ", - }); + .failure, + .dependency_failure, + .skipped_oom, + => break .dependency_failure, + + .success, .skipped => {}, + } + } else if (make_step.make(.{ + .progress_node = step_prog_node, + .watch = run.watch, + .web_server = if (run.web_server) |*ws| ws else null, + .unit_test_timeout_ns = run.unit_test_timeout_ns, + .gpa = gpa, + })) state: { + break :state .success; + } else |err| switch (err) { + error.MakeFailed => .failure, + error.MakeSkipped => .skipped, + }; + + @atomicStore(Step.State, &make_step.state, new_state, .monotonic); + + switch (new_state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .failure, + .dependency_failure, + .skipped_oom, + => { + if (run.web_server) |*ws| ws.updateStepStatus(step_index, .failure); + std.Progress.setStatus(.failure_working); + }, + + .success, + .skipped, + => { + if (run.web_server) |*ws| ws.updateStepStatus(step_index, .success); + }, + } + } + + // No matter the result, we want to display error/warning messages. + if (make_step.result_error_bundle.errorMessageCount() > 0 or + make_step.result_error_msgs.items.len > 0 or + make_step.result_stderr.len > 0) + { + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + printErrorMessages(gpa, c, run.steps, step_index, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch |err| switch (err) { + error.Canceled => |e| return e, + error.WriteFailed => switch (stderr.file_writer.err.?) { + error.Canceled => |e| return e, + else => {}, + }, + else => {}, + }; + } + + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss != 0) { + var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty; + defer dispatch_set.deinit(gpa); + + // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` + // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held. + { + try run.max_rss_mutex.lock(io); + defer run.max_rss_mutex.unlock(io); + run.available_rss += max_rss; + dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch + @panic("TODO eliminate memory allocation here"); + while (run.memory_blocked_steps.getLast()) |candidate_index| { + const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes(); + if (run.available_rss < candidate_max_rss) break; + assert(run.memory_blocked_steps.pop() == candidate_index); + dispatch_set.appendAssumeCapacity(candidate_index); + } + } + for (dispatch_set.items) |candidate| { + group.async(io, makeStep, .{ run, group, candidate, root_prog_node }); + } + } + + for (make_step.dependants.items) |dependant_index| { + const dependant = run.stepByIndex(dependant_index); + // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. + if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { + try stepReady(run, group, dependant_index, root_prog_node); + } + } } -} - -fn printChildNodePrefix(stderr: Io.Terminal) !void { - try stderr.writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ - else => "+- ", - }); -} - -fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void { - const writer = stderr.writer; - switch (s.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .dependency_failure => { - try stderr.setColor(.dim); - try writer.writeAll(" transitive failure\n"); - try stderr.setColor(.reset); - }, - - .success => { - try stderr.setColor(.green); - if (s.result_cached) { - try writer.writeAll(" cached"); - } else if (s.test_results.test_count > 0) { - const pass_count = s.test_results.passCount(); - assert(s.test_results.test_count == pass_count + s.test_results.skip_count); - try writer.print(" {d} pass", .{pass_count}); - if (s.test_results.skip_count > 0) { - try stderr.setColor(.reset); - try writer.writeAll(", "); - try stderr.setColor(.yellow); - try writer.print("{d} skip", .{s.test_results.skip_count}); + + fn printTreeStep( + run: *const Run, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + parent_node: *PrintNode, + step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), + ) !void { + const writer = stderr.writer; + const first = step_stack.swapRemove(step_index); + const summary = run.summary; + const c = &run.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const make_step = run.stepByIndex(step_index); + const skip = switch (summary) { + .none, .line => unreachable, + .all => false, + .new => make_step.result_cached, + .failures => make_step.state == .success, + }; + if (skip) return; + try printPrefix(parent_node, stderr); + + if (parent_node.parent != null) { + if (parent_node.last) { + try printChildNodePrefix(stderr); + } else { + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ + else => "+- ", + }); + } + } + + if (!first) try stderr.setColor(.dim); + + // dep_prefix omitted here because it is redundant with the tree. + try writer.writeAll(conf_step.name.slice(c)); + + const deps = conf_step.deps.slice(c); + + if (first) { + try printStepStatus(run, step_index, stderr); + + const last_index = if (summary == .all) deps.len -| 1 else blk: { + var i: usize = deps.len; + while (i > 0) { + i -= 1; + + const dep_index = deps[i]; + const dep = run.stepByIndex(dep_index); + const found = switch (summary) { + .all, .line, .none => unreachable, + .failures => dep.state != .success, + .new => !dep.result_cached, + }; + if (found) break :blk i; } - try stderr.setColor(.reset); - try writer.print(" ({d} total)", .{s.test_results.test_count}); + break :blk deps.len -| 1; + }; + for (deps, 0..) |dep, i| { + var print_node: PrintNode = .{ + .parent = parent_node, + .last = i == last_index, + }; + try printTreeStep(run, dep, stderr, &print_node, step_stack); + } + } else { + if (deps.len == 0) { + try writer.writeAll(" (reused)\n"); } else { - try writer.writeAll(" success"); + try writer.print(" (+{d} more reused dependencies)\n", .{deps.len}); } try stderr.setColor(.reset); - if (s.result_duration_ns) |ns| { + } + } + + fn printStepStatus(run: *const Run, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { + const s = run.stepByIndex(step_index); + const writer = stderr.writer; + switch (s.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .dependency_failure => { try stderr.setColor(.dim); - if (ns >= std.time.ns_per_min) { - try writer.print(" {d}m", .{ns / std.time.ns_per_min}); - } else if (ns >= std.time.ns_per_s) { - try writer.print(" {d}s", .{ns / std.time.ns_per_s}); - } else if (ns >= std.time.ns_per_ms) { - try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); - } else if (ns >= std.time.ns_per_us) { - try writer.print(" {d}us", .{ns / std.time.ns_per_us}); + try writer.writeAll(" transitive failure\n"); + try stderr.setColor(.reset); + }, + + .success => { + try stderr.setColor(.green); + if (s.result_cached) { + try writer.writeAll(" cached"); + } else if (s.test_results.test_count > 0) { + const pass_count = s.test_results.passCount(); + assert(s.test_results.test_count == pass_count + s.test_results.skip_count); + try writer.print(" {d} pass", .{pass_count}); + if (s.test_results.skip_count > 0) { + try stderr.setColor(.reset); + try writer.writeAll(", "); + try stderr.setColor(.yellow); + try writer.print("{d} skip", .{s.test_results.skip_count}); + } + try stderr.setColor(.reset); + try writer.print(" ({d} total)", .{s.test_results.test_count}); } else { - try writer.print(" {d}ns", .{ns}); + try writer.writeAll(" success"); } try stderr.setColor(.reset); - } - if (s.result_peak_rss != 0) { - const rss = s.result_peak_rss; - try stderr.setColor(.dim); - if (rss >= 1000_000_000) { - try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); - } else if (rss >= 1000_000) { - try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); - } else if (rss >= 1000) { - try writer.print(" MaxRSS:{d}K", .{rss / 1000}); - } else { - try writer.print(" MaxRSS:{d}B", .{rss}); + if (s.result_duration_ns) |ns| { + try stderr.setColor(.dim); + if (ns >= std.time.ns_per_min) { + try writer.print(" {d}m", .{ns / std.time.ns_per_min}); + } else if (ns >= std.time.ns_per_s) { + try writer.print(" {d}s", .{ns / std.time.ns_per_s}); + } else if (ns >= std.time.ns_per_ms) { + try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); + } else if (ns >= std.time.ns_per_us) { + try writer.print(" {d}us", .{ns / std.time.ns_per_us}); + } else { + try writer.print(" {d}ns", .{ns}); + } + try stderr.setColor(.reset); } + if (s.result_peak_rss != 0) { + const rss = s.result_peak_rss; + try stderr.setColor(.dim); + if (rss >= 1000_000_000) { + try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); + } else if (rss >= 1000_000) { + try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); + } else if (rss >= 1000) { + try writer.print(" MaxRSS:{d}K", .{rss / 1000}); + } else { + try writer.print(" MaxRSS:{d}B", .{rss}); + } + try stderr.setColor(.reset); + } + try writer.writeAll("\n"); + }, + .skipped => { + try stderr.setColor(.yellow); + try writer.writeAll(" skipped\n"); try stderr.setColor(.reset); - } - try writer.writeAll("\n"); - }, - .skipped => { - try stderr.setColor(.yellow); - try writer.writeAll(" skipped\n"); - try stderr.setColor(.reset); - }, - .skipped_oom => { - try stderr.setColor(.yellow); - try writer.writeAll(" skipped (not enough memory)"); - try stderr.setColor(.dim); - try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ s.max_rss, run.available_rss }); - try stderr.setColor(.reset); - }, - .failure => { - try printStepFailure(s, stderr, false); - try stderr.setColor(.reset); - }, + }, + .skipped_oom => { + const c = &run.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + try stderr.setColor(.yellow); + try writer.writeAll(" skipped (not enough memory)"); + try stderr.setColor(.dim); + try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ + max_rss, run.available_rss, + }); + try stderr.setColor(.reset); + }, + .failure => { + try printStepFailure(run.steps, step_index, stderr, false); + try stderr.setColor(.reset); + }, + } } -} +}; -fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void { +fn printStepFailure( + make_steps: []Step, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + dim: bool, +) !void { const w = stderr.writer; + const s = &make_steps[@intFromEnum(step_index)]; if (s.result_error_bundle.errorMessageCount() > 0) { try stderr.setColor(.red); try w.print(" {d} errors\n", .{ @@ -1160,79 +1390,33 @@ fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void { } } -fn printTreeStep( - graph: *Graph, - s: *Step, - run: *const Run, - stderr: Io.Terminal, - parent_node: *PrintNode, - step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), -) !void { +const PrintNode = struct { + parent: ?*PrintNode, + last: bool = false, +}; + +fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { + const parent = node.parent orelse return; const writer = stderr.writer; - const first = step_stack.swapRemove(s); - const summary = run.summary; - const skip = switch (summary) { - .none, .line => unreachable, - .all => false, - .new => s.result_cached, - .failures => s.state == .success, - }; - if (skip) return; - try printPrefix(parent_node, stderr); - - if (parent_node.parent != null) { - if (parent_node.last) { - try printChildNodePrefix(stderr); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ - else => "+- ", - }); - } - } - - if (!first) try stderr.setColor(.dim); - - // dep_prefix omitted here because it is redundant with the tree. - try writer.writeAll(s.name); - - if (first) { - try printStepStatus(s, stderr, run); - - const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: { - var i: usize = s.dependencies.items.len; - while (i > 0) { - i -= 1; - - const step = s.dependencies.items[i]; - const found = switch (summary) { - .all, .line, .none => unreachable, - .failures => step.state != .success, - .new => !step.result_cached, - }; - if (found) break :blk i; - } - break :blk s.dependencies.items.len -| 1; - }; - for (s.dependencies.items, 0..) |dep, i| { - var print_node: PrintNode = .{ - .parent = parent_node, - .last = i == last_index, - }; - try printTreeStep(graph, dep, run, stderr, &print_node, step_stack); - } + if (parent.parent == null) return; + try printPrefix(parent, stderr); + if (parent.last) { + try writer.writeAll(" "); } else { - if (s.dependencies.items.len == 0) { - try writer.writeAll(" (reused)\n"); - } else { - try writer.print(" (+{d} more reused dependencies)\n", .{ - s.dependencies.items.len, - }); - } - try stderr.setColor(.reset); + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ + else => "| ", + }); } } +fn printChildNodePrefix(stderr: Io.Terminal) !void { + try stderr.writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ + else => "+- ", + }); +} + /// Traverse the dependency graph depth-first and make it undirected by having /// steps know their dependants (they only know dependencies at start). /// Along the way, check that there is no dependency loop, and record the steps @@ -1252,14 +1436,14 @@ fn constructGraphAndCheckForDependencyLoop( step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), rand: std.Random, ) error{ DependencyLoopDetected, OutOfMemory }!void { - const s: *Step = &steps[@intFromEnum(step_index)]; - switch (s.state) { + const make_step: *Step = &steps[@intFromEnum(step_index)]; + switch (make_step.state) { .precheck_started => { log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)}); return error.DependencyLoopDetected; }, .precheck_unstarted => { - s.state = .precheck_started; + make_step.state = .precheck_started; const step = step_index.ptr(c); const dependencies = step.deps.slice(c); @@ -1275,7 +1459,7 @@ fn constructGraphAndCheckForDependencyLoop( for (deps) |dep| { const dep_step: *Step = &steps[@intFromEnum(dep)]; try step_stack.put(gpa, dep, {}); - try dep_step.dependants.append(gpa, s); + try dep_step.dependants.append(gpa, step_index); constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) { error.DependencyLoopDetected => { log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); @@ -1285,8 +1469,8 @@ fn constructGraphAndCheckForDependencyLoop( }; } - s.state = .precheck_done; - s.pending_deps = @intCast(dependencies.len); + make_step.state = .precheck_done; + make_step.pending_deps = @intCast(dependencies.len); }, .precheck_done => {}, @@ -1299,140 +1483,11 @@ fn constructGraphAndCheckForDependencyLoop( } } -/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready -/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must -/// have already subtracted this value from `run.available_rss`. This function will release the RSS -/// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked -/// steps after "make" completes for `s`. -fn makeStep( - graph: *Graph, - group: *Io.Group, - s: *Step, - root_prog_node: std.Progress.Node, - run: *Run, -) Io.Cancelable!void { - const io = graph.io; - const gpa = run.gpa; - - { - const step_prog_node = root_prog_node.start(s.name, 0); - defer step_prog_node.end(); - - if (run.web_server) |*ws| ws.updateStepStatus(s, .wip); - - const new_state: Step.State = for (s.dependencies.items) |dep| { - switch (@atomicLoad(Step.State, &dep.state, .monotonic)) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => break .dependency_failure, - - .success, .skipped => {}, - } - } else if (s.make(.{ - .progress_node = step_prog_node, - .watch = run.watch, - .web_server = if (run.web_server) |*ws| ws else null, - .unit_test_timeout_ns = run.unit_test_timeout_ns, - .gpa = gpa, - })) state: { - break :state .success; - } else |err| switch (err) { - error.MakeFailed => .failure, - error.MakeSkipped => .skipped, - }; - - @atomicStore(Step.State, &s.state, new_state, .monotonic); - - switch (new_state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => { - if (run.web_server) |*ws| ws.updateStepStatus(s, .failure); - std.Progress.setStatus(.failure_working); - }, - - .success, - .skipped, - => { - if (run.web_server) |*ws| ws.updateStepStatus(s, .success); - }, - } - } - - // No matter the result, we want to display error/warning messages. - if (s.result_error_bundle.errorMessageCount() > 0 or - s.result_error_msgs.items.len > 0 or - s.result_stderr.len > 0) - { - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - printErrorMessages(gpa, s, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch {}; - } - - if (s.max_rss != 0) { - var dispatch_set: std.ArrayList(*Step) = .empty; - defer dispatch_set.deinit(gpa); - - // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` - // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held. - { - try run.max_rss_mutex.lock(io); - defer run.max_rss_mutex.unlock(io); - run.available_rss += s.max_rss; - try dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len); - while (run.memory_blocked_steps.getLast()) |candidate| { - if (run.available_rss < candidate.max_rss) break; - assert(run.memory_blocked_steps.pop() == candidate); - dispatch_set.appendAssumeCapacity(candidate); - } - } - for (dispatch_set.items) |candidate| { - group.async(io, makeStep, .{ graph, group, candidate, root_prog_node, run }); - } - } - - for (s.dependants.items) |dependant| { - // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. - if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { - try stepReady(graph, group, dependant, root_prog_node, run); - } - } -} - -fn stepReady( - graph: *Graph, - group: *Io.Group, - s: *Step, - root_prog_node: std.Progress.Node, - run: *Run, -) !void { - const io = graph.io; - if (s.max_rss != 0) { - try run.max_rss_mutex.lock(io); - defer run.max_rss_mutex.unlock(io); - if (run.available_rss < s.max_rss) { - // Running this step right now could possibly exceed the allotted RSS. - try run.memory_blocked_steps.append(run.gpa, s); - return; - } - run.available_rss -= s.max_rss; - } - group.async(io, makeStep, .{ graph, group, s, root_prog_node, run }); -} - pub fn printErrorMessages( gpa: Allocator, - failing_step: *Step, + c: *const Configuration, + make_steps: []Step, + failing_step_index: Configuration.Step.Index, options: std.zig.ErrorBundle.RenderOptions, stderr: Io.Terminal, error_style: ErrorStyle, @@ -1442,26 +1497,28 @@ pub fn printErrorMessages( if (error_style.verboseContext()) { // Provide context for where these error messages are coming from by // printing the corresponding Step subtree. - var step_stack: std.ArrayList(*Step) = .empty; + var step_stack: std.ArrayList(Configuration.Step.Index) = .empty; defer step_stack.deinit(gpa); - try step_stack.append(gpa, failing_step); - while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) { - try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]); + try step_stack.append(gpa, failing_step_index); + while (true) { + const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])]; + if (last_step.dependants.items.len == 0) break; + try step_stack.append(gpa, last_step.dependants.items[0]); } // Now, `step_stack` has the subtree that we want to print, in reverse order. try stderr.setColor(.dim); var indent: usize = 0; - while (step_stack.pop()) |s| : (indent += 1) { + while (step_stack.pop()) |step_index| : (indent += 1) { if (indent > 0) { try writer.splatByteAll(' ', (indent - 1) * 3); try printChildNodePrefix(stderr); } - try writer.writeAll(s.name); + try writer.writeAll(step_index.ptr(c).name.slice(c)); - if (s == failing_step) { - try printStepFailure(s, stderr, true); + if (step_index == failing_step_index) { + try printStepFailure(make_steps, step_index, stderr, true); } else { try writer.writeAll("\n"); } @@ -1470,11 +1527,13 @@ pub fn printErrorMessages( } else { // Just print the failing step itself. try stderr.setColor(.dim); - try writer.writeAll(failing_step.name); - try printStepFailure(failing_step, stderr, true); + try writer.writeAll(failing_step_index.ptr(c).name.slice(c)); + try printStepFailure(make_steps, failing_step_index, stderr, true); try stderr.setColor(.reset); } + const failing_step = &make_steps[@intFromEnum(failing_step_index)]; + if (failing_step.result_stderr.len > 0) { try writer.writeAll(failing_step.result_stderr); if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { @@ -1567,9 +1626,10 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { fatal(f, args); } -fn cleanTmpFiles(io: Io, steps: []const *Step) void { - for (steps) |step| { - const wf = step.cast(Step.WriteFile) orelse continue; +fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { + for (steps) |step_index| { + if (true) @panic("TODO"); + const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; if (wf.mode != .tmp) continue; const path = wf.generated_directory.path orelse continue; Io.Dir.cwd().deleteTree(io, path) catch |err| { diff --git a/lib/compiler/maker/Fuzz.zig b/lib/compiler/maker/Fuzz.zig index b0bcbf8621e05cb5416f4f1240e4960b8f23e76a..433439f2ceef355c4663419c67ba51b7f4507b9b 100644 --- a/lib/compiler/maker/Fuzz.zig +++ b/lib/compiler/maker/Fuzz.zig @@ -1,16 +1,16 @@ const Fuzz = @This(); const std = @import("std"); -const Io = std.Io; +const Allocator = std.mem.Allocator; const Build = std.Build; const Cache = std.Build.Cache; -const Step = std.Build.Step; +const Coverage = std.debug.Coverage; +const Configuration = std.Build.Configuration; +const Io = std.Io; +const abi = std.Build.abi.fuzz; const assert = std.debug.assert; const fatal = std.process.fatal; -const Allocator = std.mem.Allocator; const log = std.log; -const Coverage = std.debug.Coverage; -const abi = std.Build.abi.fuzz; const maker = @import("../maker.zig"); const WebServer = @import("WebServer.zig"); @@ -20,7 +20,7 @@ io: Io, mode: Mode, /// Allocated into `gpa`. -run_steps: []const *Step.Run, +run_steps: []const Configuration.Step.Index, group: Io.Group, root_prog_node: std.Progress.Node, @@ -51,7 +51,7 @@ const Msg = union(enum) { unique: u64, coverage: u64, }, - run: *Step.Run, + run: Configuration.Step.Index, }, entry_point: struct { coverage_id: u64, @@ -78,12 +78,12 @@ const CoverageMap = struct { pub fn init( gpa: Allocator, io: Io, - all_steps: []const *Build.Step, + all_steps: []const Configuration.Step.Index, root_prog_node: std.Progress.Node, mode: Mode, ) error{ OutOfMemory, Canceled }!Fuzz { - const run_steps: []const *Step.Run = steps: { - var steps: std.ArrayList(*Step.Run) = .empty; + const run_steps: []const Configuration.Step.Index = steps: { + var steps: std.ArrayList(Configuration.Step.Index) = .empty; defer steps.deinit(gpa); const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0); defer rebuild_node.end(); @@ -91,7 +91,8 @@ pub fn init( defer rebuild_group.cancel(io); for (all_steps) |step| { - const run = step.cast(Step.Run) orelse continue; + if (true) @panic("TODO"); + const run = step.cast(std.Build.Step.Run) orelse continue; if (run.producer == null) continue; if (run.fuzz_tests.items.len == 0) continue; try steps.append(gpa, run); @@ -100,15 +101,16 @@ pub fn init( if (steps.items.len == 0) fatal("no fuzz tests found", .{}); rebuild_node.setEstimatedTotalItems(steps.items.len); - const run_steps = try gpa.dupe(*Step.Run, steps.items); + const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items); try rebuild_group.await(io); break :steps run_steps; }; errdefer gpa.free(run_steps); - for (run_steps) |run| { - assert(run.fuzz_tests.items.len > 0); - if (run.rebuilt_executable == null) + for (run_steps) |run_step_index| { + if (true) @panic("TODO"); + assert(run_step_index.fuzz_tests.items.len > 0); + if (run_step_index.rebuilt_executable == null) fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); } @@ -138,6 +140,8 @@ pub fn start(fuzz: *Fuzz) void { fatal("unable to spawn coverage task: {t}", .{err}); } + if (true) @panic("TODO"); + for (fuzz.run_steps) |run| { assert(run.rebuilt_executable != null); fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run }); @@ -151,14 +155,14 @@ pub fn deinit(fuzz: *Fuzz) void { fuzz.gpa.free(fuzz.run_steps); } -fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) void { +fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void { rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { const compile = run.producer.?; log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); }; } -fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { +fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { const graph = run.step.owner.graph; const io = graph.io; const compile = run.producer.?; @@ -185,7 +189,7 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); } -fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run) void { +fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { const owner = run.step.owner; const gpa = owner.allocator; const graph = owner.graph; @@ -209,6 +213,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run) void { } pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { + if (true) @panic("TODO"); assert(fuzz.mode == .forever); var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); @@ -354,7 +359,8 @@ fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { fuzz.msg_queue.clearRetainingCapacity(); } } -fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { +fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { + if (true) @panic("TODO"); assert(fuzz.mode == .forever); const ws = fuzz.mode.forever.ws; const gpa = fuzz.gpa; @@ -384,8 +390,8 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO }; errdefer gop.value_ptr.coverage.deinit(gpa); - const rebuilt_exe_path = run_step.rebuilt_executable.?; - const target = run_step.producer.?.rootModuleTarget(); + const rebuilt_exe_path = run_step_index.rebuilt_executable.?; + const target = run_step_index.producer.?.rootModuleTarget(); var debug_info = std.debug.Info.load( gpa, io, @@ -395,19 +401,19 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO target.cpu.arch, ) catch |err| { log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ - run_step.step.name, rebuilt_exe_path, err, + run_step_index.step.name, rebuilt_exe_path, err, }); return error.AlreadyReported; }; defer debug_info.deinit(gpa); const coverage_file_path: Build.Cache.Path = .{ - .root_dir = run_step.step.owner.cache_root, + .root_dir = run_step_index.step.owner.cache_root, .sub_path = "v/" ++ std.fmt.hex(coverage_id), }; var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ - run_step.step.name, coverage_file_path, err, + run_step_index.step.name, coverage_file_path, err, }); return error.AlreadyReported; }; @@ -514,6 +520,7 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte } pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { + if (true) @panic("TODO"); assert(fuzz.mode == .limit); const io = fuzz.io; diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/maker/Step.zig index 9ede48eb2c7be2c9ffd0a8cc8d86feee937f6e0a..845bc1e1f8ecb4191d945a5c3f86b390adc7290e 100644 --- a/lib/compiler/maker/Step.zig +++ b/lib/compiler/maker/Step.zig @@ -10,6 +10,7 @@ const Io = std.Io; const LazyPath = std.Build.Configuration.LazyPath; const Package = std.Build.Configuration.Package; const Path = std.Build.Cache.Path; +const Configuration = std.Build.Configuration; const assert = std.debug.assert; const WebServer = @import("WebServer.zig"); @@ -21,7 +22,7 @@ pub const Run = void; // @import("Step/Run.zig"); _: void align(std.atomic.cache_line) = {}, state: State = .precheck_unstarted, -dependants: std.ArrayList(*Step) = .empty, +dependants: std.ArrayList(Configuration.Step.Index) = .empty, /// Collects the set of files that retrigger this step to run. /// /// This is used by the build system's implementation of `--watch` but it can @@ -143,6 +144,7 @@ pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; /// have already reported the error. Otherwise, we add a simple error report /// here. pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { + if (true) @panic("TODO Step.make"); const arena = s.owner.allocator; const graph = s.owner.graph; const io = graph.io; @@ -182,6 +184,7 @@ pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!voi /// Implementation detail of file watching. Prepares the step for being re-evaluated. /// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. pub fn invalidateResult(step: *Step, gpa: Allocator) bool { + if (true) @panic("TODO Step.invalidateResult"); if (step.state == .precheck_done) return false; assert(step.pending_deps == 0); step.state = .precheck_done; @@ -544,6 +547,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebSer } pub fn getZigProcess(s: *Step) ?*ZigProcess { + if (true) @panic("TODO getZigProcess"); return switch (s.id) { .compile => s.cast(Compile).?.zig_process, else => null, diff --git a/lib/compiler/maker/Watch.zig b/lib/compiler/maker/Watch.zig index a38f4ac601755382ba248c6415e78d0d0d53a757..907e6536a132863603d70e423041b41c5b677a5c 100644 --- a/lib/compiler/maker/Watch.zig +++ b/lib/compiler/maker/Watch.zig @@ -3,11 +3,13 @@ const builtin = @import("builtin"); const std = @import("std"); const Io = std.Io; -const Step = std.Build.Step; const Allocator = std.mem.Allocator; const assert = std.debug.assert; const fatal = std.process.fatal; +const Configuration = std.Build.Configuration; + const FsEvents = @import("Watch/FsEvents.zig"); +const Step = @import("Step.zig"); os: Os, /// The number to show as the number of directories being watched. @@ -16,6 +18,8 @@ dir_count: usize, // They are `undefined` on implementations which do not utilize then. dir_table: DirTable, generation: Generation, +configuration: *const Configuration, +make_steps: []Step, pub const have_impl = Os != void; @@ -27,7 +31,7 @@ const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAda /// Special key of "." means any changes in this directory trigger the steps. const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); -const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation); +const StepSet = std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, Generation); const Generation = u8; @@ -101,7 +105,7 @@ const Os = switch (builtin.os.tag) { }; }; - fn init(cwd_path: []const u8) !Watch { + fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { _ = cwd_path; return .{ .dir_table = .{}, @@ -114,6 +118,8 @@ const Os = switch (builtin.os.tag) { else => {}, }, .generation = 0, + .make_steps = make_steps, + .configuration = configuration, }; } @@ -161,20 +167,21 @@ const Os = switch (builtin.os.tag) { const lfh: FileHandle = .{ .handle = file_handle }; if (w.os.handle_table.getPtr(lfh)) |value| { if (value.reaction_set.getPtr(".")) |glob_set| - any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); + any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty); if (value.reaction_set.getPtr(file_name)) |step_set| - any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty); } }, - else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), + else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}), } } } } - fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { // Add missing marks and note persisted ones. - for (steps) |step| { + for (steps) |step_index| { + const step = &w.make_steps[@intFromEnum(step_index)]; for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { const reaction_set = rs: { const gop = try w.dir_table.getOrPut(gpa, path); @@ -236,7 +243,7 @@ const Os = switch (builtin.os.tag) { for (files.items) |basename| { const gop = try reaction_set.getOrPut(gpa, basename); if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step, w.generation); + try gop.value_ptr.put(gpa, step_index, w.generation); } } } @@ -537,7 +544,7 @@ const Os = switch (builtin.os.tag) { return any_dirty; } - fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { // Add missing marks and note persisted ones. for (steps) |step| { for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { @@ -678,7 +685,7 @@ const Os = switch (builtin.os.tag) { }; } - fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { const handles = &w.os.handles; for (steps) |step| { for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { @@ -856,7 +863,7 @@ const Os = switch (builtin.os.tag) { .generation = undefined, }; } - fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { try w.os.fse.setPaths(gpa, steps); w.dir_count = w.os.fse.watch_roots.len; } @@ -871,8 +878,8 @@ const Os = switch (builtin.os.tag) { else => void, }; -pub fn init(cwd_path: []const u8) !Watch { - return Os.init(cwd_path); +pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { + return Os.init(cwd_path, configuration, make_steps); } pub const Match = struct { @@ -880,20 +887,19 @@ pub const Match = struct { /// match. basename: []const u8, /// The step to re-run when file corresponding to `basename` is changed. - step: *Step, + step_index: Configuration.Step.Index, pub const Context = struct { pub fn hash(self: Context, a: Match) u32 { _ = self; - var hasher = Hash.init(0); - std.hash.autoHash(&hasher, a.step); + var hasher = Hash.init(@intFromEnum(a.step_index)); hasher.update(a.basename); return @truncate(hasher.final()); } pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { _ = self; _ = b_index; - return a.step == b.step and std.mem.eql(u8, a.basename, b.basename); + return a.step_index == b.step_index and std.mem.eql(u8, a.basename, b.basename); } }; }; @@ -908,22 +914,24 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { else => item, }; for (reaction_set.values()) |step_set| { - for (step_set.keys()) |step| { + for (step_set.keys()) |step_index| { + const step = &w.make_steps[@intFromEnum(step_index)]; _ = step.invalidateResult(gpa); } } } } -fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool { +fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool { var this_any_dirty = false; - for (step_set.keys()) |step| { + for (step_set.keys()) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; if (step.invalidateResult(gpa)) this_any_dirty = true; } return any_dirty or this_any_dirty; } -pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { +pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { return Os.update(w, gpa, steps); } diff --git a/lib/compiler/maker/WebServer.zig b/lib/compiler/maker/WebServer.zig index 9a81d6e5712c74104d40bfe0d9e7de909477187f..fd3806e8b2b925f6badad2830ddd8d780abab2b4 100644 --- a/lib/compiler/maker/WebServer.zig +++ b/lib/compiler/maker/WebServer.zig @@ -4,8 +4,8 @@ const builtin = @import("builtin"); const std = @import("std"); const Allocator = std.mem.Allocator; -const Build = std.Build; const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; const Io = std.Io; const abi = std.Build.abi; const assert = std.debug.assert; @@ -15,10 +15,12 @@ const mem = std.mem; const net = std.Io.net; const Fuzz = @import("Fuzz.zig"); +const Graph = @import("Graph.zig"); +const Step = @import("Step.zig"); gpa: Allocator, -graph: *const Build.Graph, -all_steps: []const *Build.Step, +graph: *const Graph, +all_steps: []const Configuration.Step.Index, listen_address: net.IpAddress, root_prog_node: std.Progress.Node, watch: bool, @@ -69,12 +71,13 @@ pub fn notifyUpdate(ws: *WebServer) void { pub const Options = struct { gpa: Allocator, - graph: *const std.Build.Graph, - all_steps: []const *Build.Step, + graph: *const Graph, + all_steps: []const Configuration.Step.Index, root_prog_node: std.Progress.Node, watch: bool, listen_address: net.IpAddress, base_timestamp: Io.Clock.Timestamp, + configuration: *const Configuration, }; pub fn init(opts: Options) WebServer { // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` @@ -83,19 +86,21 @@ pub fn init(opts: Options) WebServer { assert(opts.base_timestamp.clock == base_clock); const all_steps = opts.all_steps; + const c = opts.configuration; const step_names_trailing = opts.gpa.alloc(u8, len: { var name_bytes: usize = 0; - for (all_steps) |step| name_bytes += step.name.len; + for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len; break :len name_bytes + all_steps.len * 4; }) catch @panic("out of memory"); { const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]); var idx: usize = all_steps.len * 4; - for (all_steps, step_name_lens) |step, *name_len| { - name_len.* = @intCast(step.name.len); - @memcpy(step_names_trailing[idx..][0..step.name.len], step.name); - idx += step.name.len; + for (all_steps, step_name_lens) |step_index, *name_len| { + const step_name = step_index.ptr(c).name.slice(c); + name_len.* = @intCast(step_name.len); + @memcpy(step_names_trailing[idx..][0..step_name.len], step_name); + idx += step_name.len; } assert(idx == step_names_trailing.len); } @@ -213,9 +218,14 @@ pub fn startBuild(ws: *WebServer) void { ws.notifyUpdate(); } -pub fn updateStepStatus(ws: *WebServer, step: *Build.Step, new_status: abi.StepUpdate.Status) void { +pub fn updateStepStatus( + ws: *WebServer, + step_index: Configuration.Step.Index, + new_status: abi.StepUpdate.Status, +) void { + // TODO don't do linear search, especially in a hot loop like this const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == step) break @intCast(i); + if (s == step_index) break @intCast(i); } else unreachable; const ptr = &ws.step_status_bits[step_idx / 4]; const bit_offset: u3 = @intCast((step_idx % 4) * 2); @@ -687,7 +697,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim if (code != 0) { log.err( "the following command exited with error code {d}:\n{s}", - .{ code, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, ); return error.WasmCompilationFailed; } @@ -695,7 +705,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim .signal => |sig| { log.err( "the following command terminated with signal {t}:\n{s}", - .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, ); return error.WasmCompilationFailed; }, @@ -709,7 +719,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim .unknown => { log.err( "the following command terminated unexpectedly:\n{s}", - .{try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items)}, + .{try Step.allocPrintCmd(arena, .inherit, null, argv.items)}, ); return error.WasmCompilationFailed; }, @@ -719,14 +729,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim try result_error_bundle.renderToStderr(io, .{}, .auto); log.err("the following command failed with {d} compilation errors:\n{s}", .{ result_error_bundle.errorMessageCount(), - try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), + try Step.allocPrintCmd(arena, .inherit, null, argv.items), }); return error.WasmCompilationFailed; } const base_path = result orelse { log.err("child process failed to report result\n{s}", .{ - try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), + try Step.allocPrintCmd(arena, .inherit, null, argv.items), }); return error.WasmCompilationFailed; }; @@ -750,7 +760,7 @@ fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 } pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { - compile: *Build.Step.Compile, + compile_step: Configuration.Step.Index, use_llvm: bool, stats: abi.time_report.CompileResult.Stats, @@ -766,8 +776,9 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { const gpa = ws.gpa; const io = ws.graph.io; + // TODO don't do linear search const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == &opts.compile.step) break @intCast(i); + if (s == opts.compile_step) break @intCast(i); } else unreachable; const old_buf = old: { @@ -803,12 +814,13 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { ws.notifyUpdate(); } -pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.Duration) void { +pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void { const gpa = ws.gpa; const io = ws.graph.io; + // TODO don't do linear search const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == step) break @intCast(i); + if (s == step_index) break @intCast(i); } else unreachable; const old_buf = old: { @@ -836,15 +848,16 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.D pub fn updateTimeReportRunTest( ws: *WebServer, - run: *Build.Step.Run, - tests: *const Build.Step.Run.CachedTestMetadata, + run_step_index: Configuration.Step.Index, + tests: *const Step.Run.CachedTestMetadata, ns_per_test: []const u64, ) void { const gpa = ws.gpa; const io = ws.graph.io; + // TODO don't do linear search const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == &run.step) break @intCast(i); + if (s == run_step_index) break @intCast(i); } else unreachable; assert(tests.names.len == ns_per_test.len); -- 2.54.0 From ef050483dff9f4fe57107b4c3ddba9a2692bdef4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 18 Feb 2026 14:06:43 -0800 Subject: [PATCH 012/179] build maker: rename Run to Maker --- lib/compiler/maker.zig | 1303 ++++++++++++++++++++-------------------- 1 file changed, 651 insertions(+), 652 deletions(-) diff --git a/lib/compiler/maker.zig b/lib/compiler/maker.zig index a94416c54070b097007a537b11f38fa83ec35b0f..78b472530f5e8408bc07a4c5f1c9f0767a98fd66 100644 --- a/lib/compiler/maker.zig +++ b/lib/compiler/maker.zig @@ -1,3 +1,4 @@ +const Maker = @This(); const builtin = @import("builtin"); const std = @import("std"); @@ -26,6 +27,28 @@ pub const std_options: std.Options = .{ .http_disable_tls = true, }; +gpa: Allocator, +graph: *Graph, +install_paths: InstallPaths, +scanned_config: *const ScannedConfig, +steps: []Step, + +available_rss: usize, +max_rss_is_default: bool, +max_rss_mutex: Io.Mutex, +skip_oom_steps: bool, +unit_test_timeout_ns: ?u64, +watch: bool, +web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, +/// Allocated into `gpa`. +memory_blocked_steps: std.ArrayList(Configuration.Step.Index), +/// Allocated into `gpa`. +step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), + +error_style: ErrorStyle, +multiline_errors: MultilineErrors, +summary: Summary, + pub fn main(init: process.Init.Minimal) !void { // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not // always the case. So, we do need a true gpa for some things. @@ -467,7 +490,7 @@ pub fn main(init: process.Init.Minimal) !void { .sub_path = cwd_relative, } else try install_prefix_path.join(arena, "include"); - var run: Run = .{ + var maker: Maker = .{ .gpa = gpa, .graph = &graph, .scanned_config = &scanned_config, @@ -495,16 +518,16 @@ pub fn main(init: process.Init.Minimal) !void { .summary = summary orelse if (watch or webui_listen != null) .line else .failures, }; defer { - run.memory_blocked_steps.deinit(gpa); - run.step_stack.deinit(gpa); + maker.memory_blocked_steps.deinit(gpa); + maker.step_stack.deinit(gpa); } - if (run.available_rss == 0) { - run.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); - run.max_rss_is_default = true; + if (maker.available_rss == 0) { + maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); + maker.max_rss_is_default = true; } - run.prepare(step_names.items) catch |err| switch (err) { + maker.prepare(step_names.items) catch |err| switch (err) { error.DependencyLoopDetected, error.InsufficientMemory => { _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(1); @@ -515,17 +538,17 @@ pub fn main(init: process.Init.Minimal) !void { var w: Watch = w: { if (!watch) break :w undefined; if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); - break :w try .init(graph.cache.cwd, &scanned_config.configuration, run.steps); + break :w try .init(graph.cache.cwd, &scanned_config.configuration, maker.steps); }; const now = Io.Clock.Timestamp.now(io, .awake); - run.web_server = if (webui_listen) |listen_address| ws: { + maker.web_server = if (webui_listen) |listen_address| ws: { if (builtin.single_threaded) unreachable; // `fatal` above break :ws .init(.{ .gpa = gpa, .graph = &graph, - .all_steps = run.step_stack.keys(), + .all_steps = maker.step_stack.keys(), .root_prog_node = main_progress_node, .watch = watch, .listen_address = listen_address, @@ -534,20 +557,20 @@ pub fn main(init: process.Init.Minimal) !void { }); } else null; - if (run.web_server) |*ws| { + if (maker.web_server) |*ws| { ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); } - rebuild: while (true) : (if (run.error_style.clearOnUpdate()) { + rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); defer io.unlockStderr(); try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H"); }) { - if (run.web_server) |*ws| ws.startBuild(); + if (maker.web_server) |*ws| ws.startBuild(); - try run.makeStepNames(step_names.items, main_progress_node, fuzz); + try maker.makeStepNames(step_names.items, main_progress_node, fuzz); - if (run.web_server) |*web_server| { + if (maker.web_server) |*web_server| { if (fuzz) |mode| if (mode != .forever) fatal( "error: limited fuzzing is not implemented yet for --webui", .{}, @@ -556,13 +579,13 @@ pub fn main(init: process.Init.Minimal) !void { web_server.finishBuild(.{ .fuzz = fuzz != null }); } - if (run.web_server) |*ws| { + if (maker.web_server) |*ws| { const c = &scanned_config.configuration; assert(!watch); // fatal error after CLI parsing while (true) switch (try ws.wait()) { .rebuild => { - for (run.step_stack.keys()) |step_index| { - const step = run.stepByIndex(step_index); + for (maker.step_stack.keys()) |step_index| { + const step = maker.stepByIndex(step_index); step.state = .precheck_done; const deps = step_index.ptr(c).deps.slice(c); step.pending_deps = @intCast(deps.len); @@ -576,7 +599,7 @@ pub fn main(init: process.Init.Minimal) !void { // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. if (!Watch.have_impl) unreachable; - try w.update(gpa, run.step_stack.keys()); + try w.update(gpa, maker.step_stack.keys()); // Wait until a file system notification arrives. Read all such events // until the buffer is empty. Then wait for a debounce interval, resetting @@ -585,7 +608,7 @@ pub fn main(init: process.Init.Minimal) !void { // recursive dependants. var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ - w.dir_count, countSubProcesses(run.steps, run.step_stack.keys()), + w.dir_count, countSubProcesses(maker.steps, maker.step_stack.keys()), }) catch &caption_buf; var debouncing_node = main_progress_node.start(caption, 0); var in_debounce = false; @@ -593,7 +616,7 @@ pub fn main(init: process.Init.Minimal) !void { .timeout => { assert(in_debounce); debouncing_node.end(); - markFailedStepsDirty(gpa, run.steps, run.step_stack.keys()); + markFailedStepsDirty(gpa, maker.steps, maker.step_stack.keys()); continue :rebuild; }, .dirty => if (!in_debounce) { @@ -634,671 +657,647 @@ fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.I return count; } -const Run = struct { - gpa: Allocator, - graph: *Graph, - install_paths: InstallPaths, - scanned_config: *const ScannedConfig, - steps: []Step, - - available_rss: usize, - max_rss_is_default: bool, - max_rss_mutex: Io.Mutex, - skip_oom_steps: bool, - unit_test_timeout_ns: ?u64, - watch: bool, - web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, - /// Allocated into `gpa`. - memory_blocked_steps: std.ArrayList(Configuration.Step.Index), - /// Allocated into `gpa`. - step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), - - error_style: ErrorStyle, - multiline_errors: MultilineErrors, - summary: Summary, - - const InstallPaths = struct { - prefix: Path, - lib: Path, - bin: Path, - include: Path, - }; - - fn stepByIndex(run: *const Run, i: Configuration.Step.Index) *Step { - return &run.steps[@intFromEnum(i)]; +const InstallPaths = struct { + prefix: Path, + lib: Path, + bin: Path, + include: Path, +}; + +fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step { + return &maker.steps[@intFromEnum(i)]; +} + +fn prepare(maker: *Maker, step_names: []const []const u8) !void { + const gpa = maker.gpa; + const graph = maker.graph; + const arena = graph.arena; + const seed: u32 = graph.random_seed; + const step_stack = &maker.step_stack; + const c = &maker.scanned_config.configuration; + + @memset(maker.steps, .{}); + + if (step_names.len == 0) { + try step_stack.put(gpa, c.default_step, {}); + } else { + try step_stack.ensureUnusedCapacity(gpa, step_names.len); + for (0..step_names.len) |i| { + const step_name = step_names[step_names.len - i - 1]; + const s = maker.scanned_config.top_level_steps.get(step_name) orelse { + log.info("to list available steps: zig build -l", .{}); + fatal("no such step: {s}", .{step_name}); + }; + step_stack.putAssumeCapacity(s, {}); + } + } + + const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys()); + + var rng = std.Random.DefaultPrng.init(seed); + const rand = rng.random(); + rand.shuffle(Configuration.Step.Index, starting_steps); + + for (starting_steps) |s| { + try constructGraphAndCheckForDependencyLoop(gpa, c, maker.steps, s, &maker.step_stack, rand); + } + + { + // Check that we have enough memory to complete the build. + var any_problems = false; + var max_needed: usize = 0; + for (step_stack.keys()) |step_index| { + const make_step = maker.stepByIndex(step_index); + const conf_step = step_index.ptr(c); + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss == 0) continue; + max_needed = @max(max_needed, max_rss); + if (max_rss > maker.available_rss) { + if (maker.skip_oom_steps) { + make_step.state = .skipped_oom; + for (make_step.dependants.items) |dependant| { + maker.stepByIndex(dependant).pending_deps -= 1; + } + } else { + log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ + conf_step.owner.depPrefixSlice(c), + conf_step.name.slice(c), + max_rss, + maker.available_rss, + }); + any_problems = true; + } + } + } + if (any_problems) { + if (maker.max_rss_is_default) { + std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ + max_needed, + }); + } + return error.InsufficientMemory; + } } +} + +fn makeStepNames( + maker: *Maker, + step_names: []const []const u8, + parent_prog_node: std.Progress.Node, + fuzz: ?Fuzz.Mode, +) !void { + const graph = maker.graph; + const gpa = maker.gpa; + const io = graph.io; + const step_stack = &maker.step_stack; + const top_level_steps = &maker.scanned_config.top_level_steps; + const c = &maker.scanned_config.configuration; + + { + // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, + // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking + // a step is initial when it actually became ready due to an earlier initial step. + var initial_set: std.ArrayList(Configuration.Step.Index) = .empty; + defer initial_set.deinit(gpa); + try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); + for (step_stack.keys()) |step_index| { + const s = maker.stepByIndex(step_index); + if (s.state == .precheck_done and s.pending_deps == 0) { + initial_set.appendAssumeCapacity(step_index); + } + } + + const step_prog = parent_prog_node.start("steps", step_stack.count()); + defer step_prog.end(); + + var group: Io.Group = .init; + defer group.cancel(io); + // Start working on all of the initial steps... + for (initial_set.items) |step_index| try stepReady(maker, &group, step_index, step_prog); + // ...and `makeStep` will trigger every other step when their last dependency finishes. + try group.await(io); + } + + assert(maker.memory_blocked_steps.items.len == 0); + + var test_pass_count: usize = 0; + var test_skip_count: usize = 0; + var test_fail_count: usize = 0; + var test_crash_count: usize = 0; + var test_timeout_count: usize = 0; + + var test_count: usize = 0; + + var success_count: usize = 0; + var skipped_count: usize = 0; + var failure_count: usize = 0; + var pending_count: usize = 0; + var total_compile_errors: usize = 0; + + var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); + defer cleanup_task.await(io); + + for (step_stack.keys()) |step_index| { + const make_step = maker.stepByIndex(step_index); + test_pass_count += make_step.test_results.passCount(); + test_skip_count += make_step.test_results.skip_count; + test_fail_count += make_step.test_results.fail_count; + test_crash_count += make_step.test_results.crash_count; + test_timeout_count += make_step.test_results.timeout_count; + + test_count += make_step.test_results.test_count; + + switch (make_step.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + .dependency_failure => pending_count += 1, + .success => success_count += 1, + .skipped, .skipped_oom => skipped_count += 1, + .failure => { + failure_count += 1; + const compile_errors_len = make_step.result_error_bundle.errorMessageCount(); + if (compile_errors_len > 0) { + total_compile_errors += compile_errors_len; + } + }, + } + } + + if (fuzz) |mode| blk: { + switch (builtin.os.tag) { + // Current implementation depends on two things that need to be ported to Windows: + // * Memory-mapping to share data between the fuzzer and build runner. + // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving + // many addresses to source locations). + .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), + else => {}, + } + if (@bitSizeOf(usize) != 64) { + // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, + // being compatible with file system's u64 return value. This is not the case + // on 32-bit platforms. + // Affects or affected by issues #5185, #22523, and #22464. + fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); + } + + switch (mode) { + .forever => break :blk, + .limit => {}, + } + + assert(mode == .limit); + var f = Fuzz.init( + gpa, + io, + step_stack.keys(), + parent_prog_node, + mode, + ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); + defer f.deinit(); + + f.start(); + try f.waitAndPrintReport(); + } + + // Every test has a state + assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); + + if (failure_count == 0) { + std.Progress.setStatus(.success); + } else { + std.Progress.setStatus(.failure); + } + + summary: { + switch (maker.summary) { + .all, .new, .line => {}, + .failures => if (failure_count == 0) break :summary, + .none => break :summary, + } + + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + const t = stderr.terminal(); + const w = &stderr.file_writer.interface; + + const total_count = success_count + failure_count + pending_count + skipped_count; + t.setColor(.cyan) catch {}; + t.setColor(.bold) catch {}; + w.writeAll("Build Summary: ") catch {}; + t.setColor(.reset) catch {}; + w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; + { + t.setColor(.dim) catch {}; + var first = true; + if (skipped_count > 0) { + w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; + first = false; + } + if (failure_count > 0) { + w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; + first = false; + } + if (!first) w.writeByte(')') catch {}; + t.setColor(.reset) catch {}; + } + + if (test_count > 0) { + w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; + t.setColor(.dim) catch {}; + var first = true; + if (test_skip_count > 0) { + w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; + first = false; + } + if (test_fail_count > 0) { + w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; + first = false; + } + if (test_crash_count > 0) { + w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; + first = false; + } + if (test_timeout_count > 0) { + w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; + first = false; + } + if (!first) w.writeByte(')') catch {}; + t.setColor(.reset) catch {}; + } + + w.writeAll("\n") catch {}; - fn prepare(run: *Run, step_names: []const []const u8) !void { - const gpa = run.gpa; - const graph = run.graph; - const arena = graph.arena; - const seed: u32 = graph.random_seed; - const step_stack = &run.step_stack; - const c = &run.scanned_config.configuration; + if (maker.summary == .line) break :summary; - @memset(run.steps, .{}); + // Print a fancy tree with build results. + var step_stack_copy = try step_stack.clone(gpa); + defer step_stack_copy.deinit(gpa); + var print_node: PrintNode = .{ .parent = null }; if (step_names.len == 0) { - try step_stack.put(gpa, c.default_step, {}); - } else { - try step_stack.ensureUnusedCapacity(gpa, step_names.len); - for (0..step_names.len) |i| { - const step_name = step_names[step_names.len - i - 1]; - const s = run.scanned_config.top_level_steps.get(step_name) orelse { - log.info("to list available steps: zig build -l", .{}); - fatal("no such step: {s}", .{step_name}); - }; - step_stack.putAssumeCapacity(s, {}); - } - } - - const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys()); - - var rng = std.Random.DefaultPrng.init(seed); - const rand = rng.random(); - rand.shuffle(Configuration.Step.Index, starting_steps); - - for (starting_steps) |s| { - try constructGraphAndCheckForDependencyLoop(gpa, c, run.steps, s, &run.step_stack, rand); - } - - { - // Check that we have enough memory to complete the build. - var any_problems = false; - var max_needed: usize = 0; - for (step_stack.keys()) |step_index| { - const make_step = run.stepByIndex(step_index); - const conf_step = step_index.ptr(c); - const max_rss = conf_step.max_rss.toBytes(); - if (max_rss == 0) continue; - max_needed = @max(max_needed, max_rss); - if (max_rss > run.available_rss) { - if (run.skip_oom_steps) { - make_step.state = .skipped_oom; - for (make_step.dependants.items) |dependant| { - run.stepByIndex(dependant).pending_deps -= 1; - } - } else { - log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ - conf_step.owner.depPrefixSlice(c), - conf_step.name.slice(c), - max_rss, - run.available_rss, - }); - any_problems = true; - } - } - } - if (any_problems) { - if (run.max_rss_is_default) { - std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ - max_needed, - }); - } - return error.InsufficientMemory; - } - } - } - - fn makeStepNames( - run: *Run, - step_names: []const []const u8, - parent_prog_node: std.Progress.Node, - fuzz: ?Fuzz.Mode, - ) !void { - const graph = run.graph; - const gpa = run.gpa; - const io = graph.io; - const step_stack = &run.step_stack; - const top_level_steps = &run.scanned_config.top_level_steps; - const c = &run.scanned_config.configuration; - - { - // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, - // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking - // a step is initial when it actually became ready due to an earlier initial step. - var initial_set: std.ArrayList(Configuration.Step.Index) = .empty; - defer initial_set.deinit(gpa); - try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); - for (step_stack.keys()) |step_index| { - const s = run.stepByIndex(step_index); - if (s.state == .precheck_done and s.pending_deps == 0) { - initial_set.appendAssumeCapacity(step_index); - } - } - - const step_prog = parent_prog_node.start("steps", step_stack.count()); - defer step_prog.end(); - - var group: Io.Group = .init; - defer group.cancel(io); - // Start working on all of the initial steps... - for (initial_set.items) |step_index| try stepReady(run, &group, step_index, step_prog); - // ...and `makeStep` will trigger every other step when their last dependency finishes. - try group.await(io); - } - - assert(run.memory_blocked_steps.items.len == 0); - - var test_pass_count: usize = 0; - var test_skip_count: usize = 0; - var test_fail_count: usize = 0; - var test_crash_count: usize = 0; - var test_timeout_count: usize = 0; - - var test_count: usize = 0; - - var success_count: usize = 0; - var skipped_count: usize = 0; - var failure_count: usize = 0; - var pending_count: usize = 0; - var total_compile_errors: usize = 0; - - var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); - defer cleanup_task.await(io); - - for (step_stack.keys()) |step_index| { - const make_step = run.stepByIndex(step_index); - test_pass_count += make_step.test_results.passCount(); - test_skip_count += make_step.test_results.skip_count; - test_fail_count += make_step.test_results.fail_count; - test_crash_count += make_step.test_results.crash_count; - test_timeout_count += make_step.test_results.timeout_count; - - test_count += make_step.test_results.test_count; - - switch (make_step.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - .dependency_failure => pending_count += 1, - .success => success_count += 1, - .skipped, .skipped_oom => skipped_count += 1, - .failure => { - failure_count += 1; - const compile_errors_len = make_step.result_error_bundle.errorMessageCount(); - if (compile_errors_len > 0) { - total_compile_errors += compile_errors_len; - } - }, - } - } - - if (fuzz) |mode| blk: { - switch (builtin.os.tag) { - // Current implementation depends on two things that need to be ported to Windows: - // * Memory-mapping to share data between the fuzzer and build runner. - // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving - // many addresses to source locations). - .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), - else => {}, - } - if (@bitSizeOf(usize) != 64) { - // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, - // being compatible with file system's u64 return value. This is not the case - // on 32-bit platforms. - // Affects or affected by issues #5185, #22523, and #22464. - fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); - } - - switch (mode) { - .forever => break :blk, - .limit => {}, - } - - assert(mode == .limit); - var f = Fuzz.init( - gpa, - io, - step_stack.keys(), - parent_prog_node, - mode, - ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); - defer f.deinit(); - - f.start(); - try f.waitAndPrintReport(); - } - - // Every test has a state - assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); - - if (failure_count == 0) { - std.Progress.setStatus(.success); - } else { - std.Progress.setStatus(.failure); - } - - summary: { - switch (run.summary) { - .all, .new, .line => {}, - .failures => if (failure_count == 0) break :summary, - .none => break :summary, - } - - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - const t = stderr.terminal(); - const w = &stderr.file_writer.interface; - - const total_count = success_count + failure_count + pending_count + skipped_count; - t.setColor(.cyan) catch {}; - t.setColor(.bold) catch {}; - w.writeAll("Build Summary: ") catch {}; - t.setColor(.reset) catch {}; - w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; - { - t.setColor(.dim) catch {}; - var first = true; - if (skipped_count > 0) { - w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; - first = false; - } - if (failure_count > 0) { - w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; - first = false; - } - if (!first) w.writeByte(')') catch {}; - t.setColor(.reset) catch {}; - } - - if (test_count > 0) { - w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; - t.setColor(.dim) catch {}; - var first = true; - if (test_skip_count > 0) { - w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; - first = false; - } - if (test_fail_count > 0) { - w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; - first = false; - } - if (test_crash_count > 0) { - w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; - first = false; - } - if (test_timeout_count > 0) { - w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; - first = false; - } - if (!first) w.writeByte(')') catch {}; - t.setColor(.reset) catch {}; - } - - w.writeAll("\n") catch {}; - - if (run.summary == .line) break :summary; - - // Print a fancy tree with build results. - var step_stack_copy = try step_stack.clone(gpa); - defer step_stack_copy.deinit(gpa); - - var print_node: PrintNode = .{ .parent = null }; - if (step_names.len == 0) { - print_node.last = true; - printTreeStep(run, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) { - error.Canceled => |e| return e, - else => {}, - }; - } else { - const last_index = if (run.summary == .all) top_level_steps.count() else blk: { - var i: usize = step_names.len; - while (i > 0) { - i -= 1; - const step_index = top_level_steps.get(step_names[i]).?; - const step = run.stepByIndex(step_index); - const found = switch (run.summary) { - .all, .line, .none => unreachable, - .failures => step.state != .success, - .new => !step.result_cached, - }; - if (found) break :blk i; - } - break :blk top_level_steps.count(); - }; - for (step_names, 0..) |step_name, i| { - const step_index = top_level_steps.get(step_name).?; - print_node.last = i + 1 == last_index; - printTreeStep(run, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) { - error.Canceled => |e| return e, - else => {}, - }; - } - } - w.writeByte('\n') catch {}; - } - - if (run.watch or run.web_server != null) return; - - // Perhaps in the future there could be an Advanced Options flag such as - // --debug-build-runner-leaks which would make this code return instead of - // calling exit. - - const code: u8 = code: { - if (failure_count == 0) break :code 0; // success - if (run.error_style.verboseContext()) break :code 1; // failure; print build command - break :code 2; // failure; do not print build command - }; - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(code); - } - - fn stepReady( - run: *Run, - group: *Io.Group, - step_index: Configuration.Step.Index, - root_prog_node: std.Progress.Node, - ) Io.Cancelable!void { - const graph = run.graph; - const io = graph.io; - const c = &run.scanned_config.configuration; - const max_rss = step_index.ptr(c).max_rss.toBytes(); - if (max_rss != 0) { - try run.max_rss_mutex.lock(io); - defer run.max_rss_mutex.unlock(io); - if (run.available_rss < max_rss) { - // Running this step right now could possibly exceed the allotted RSS. - run.memory_blocked_steps.append(run.gpa, step_index) catch - @panic("TODO eliminate memory allocation here"); - return; - } - run.available_rss -= max_rss; - } - group.async(io, makeStep, .{ run, group, step_index, root_prog_node }); - } - - /// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready - /// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must - /// have already subtracted this value from `run.available_rss`. This function will release the RSS - /// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked - /// steps after "make" completes for `s`. - fn makeStep( - run: *Run, - group: *Io.Group, - step_index: Configuration.Step.Index, - root_prog_node: std.Progress.Node, - ) Io.Cancelable!void { - const graph = run.graph; - const io = graph.io; - const gpa = run.gpa; - const c = &run.scanned_config.configuration; - const conf_step = step_index.ptr(c); - const step_name = conf_step.name.slice(c); - const deps = conf_step.deps.slice(c); - const make_step = run.stepByIndex(step_index); - - { - const step_prog_node = root_prog_node.start(step_name, 0); - defer step_prog_node.end(); - - if (run.web_server) |*ws| ws.updateStepStatus(step_index, .wip); - - const new_state: Step.State = for (deps) |dep_index| { - const dep_make_step = run.stepByIndex(dep_index); - switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => break .dependency_failure, - - .success, .skipped => {}, - } - } else if (make_step.make(.{ - .progress_node = step_prog_node, - .watch = run.watch, - .web_server = if (run.web_server) |*ws| ws else null, - .unit_test_timeout_ns = run.unit_test_timeout_ns, - .gpa = gpa, - })) state: { - break :state .success; - } else |err| switch (err) { - error.MakeFailed => .failure, - error.MakeSkipped => .skipped, - }; - - @atomicStore(Step.State, &make_step.state, new_state, .monotonic); - - switch (new_state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => { - if (run.web_server) |*ws| ws.updateStepStatus(step_index, .failure); - std.Progress.setStatus(.failure_working); - }, - - .success, - .skipped, - => { - if (run.web_server) |*ws| ws.updateStepStatus(step_index, .success); - }, - } - } - - // No matter the result, we want to display error/warning messages. - if (make_step.result_error_bundle.errorMessageCount() > 0 or - make_step.result_error_msgs.items.len > 0 or - make_step.result_stderr.len > 0) - { - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - printErrorMessages(gpa, c, run.steps, step_index, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch |err| switch (err) { + print_node.last = true; + printTreeStep(maker, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) { error.Canceled => |e| return e, - error.WriteFailed => switch (stderr.file_writer.err.?) { - error.Canceled => |e| return e, - else => {}, - }, else => {}, }; - } - - const max_rss = conf_step.max_rss.toBytes(); - if (max_rss != 0) { - var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty; - defer dispatch_set.deinit(gpa); - - // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` - // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held. - { - try run.max_rss_mutex.lock(io); - defer run.max_rss_mutex.unlock(io); - run.available_rss += max_rss; - dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch - @panic("TODO eliminate memory allocation here"); - while (run.memory_blocked_steps.getLast()) |candidate_index| { - const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes(); - if (run.available_rss < candidate_max_rss) break; - assert(run.memory_blocked_steps.pop() == candidate_index); - dispatch_set.appendAssumeCapacity(candidate_index); - } - } - for (dispatch_set.items) |candidate| { - group.async(io, makeStep, .{ run, group, candidate, root_prog_node }); - } - } - - for (make_step.dependants.items) |dependant_index| { - const dependant = run.stepByIndex(dependant_index); - // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. - if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { - try stepReady(run, group, dependant_index, root_prog_node); - } - } - } - - fn printTreeStep( - run: *const Run, - step_index: Configuration.Step.Index, - stderr: Io.Terminal, - parent_node: *PrintNode, - step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), - ) !void { - const writer = stderr.writer; - const first = step_stack.swapRemove(step_index); - const summary = run.summary; - const c = &run.scanned_config.configuration; - const conf_step = step_index.ptr(c); - const make_step = run.stepByIndex(step_index); - const skip = switch (summary) { - .none, .line => unreachable, - .all => false, - .new => make_step.result_cached, - .failures => make_step.state == .success, - }; - if (skip) return; - try printPrefix(parent_node, stderr); - - if (parent_node.parent != null) { - if (parent_node.last) { - try printChildNodePrefix(stderr); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ - else => "+- ", - }); - } - } - - if (!first) try stderr.setColor(.dim); - - // dep_prefix omitted here because it is redundant with the tree. - try writer.writeAll(conf_step.name.slice(c)); - - const deps = conf_step.deps.slice(c); - - if (first) { - try printStepStatus(run, step_index, stderr); - - const last_index = if (summary == .all) deps.len -| 1 else blk: { - var i: usize = deps.len; + } else { + const last_index = if (maker.summary == .all) top_level_steps.count() else blk: { + var i: usize = step_names.len; while (i > 0) { i -= 1; - - const dep_index = deps[i]; - const dep = run.stepByIndex(dep_index); - const found = switch (summary) { + const step_index = top_level_steps.get(step_names[i]).?; + const step = maker.stepByIndex(step_index); + const found = switch (maker.summary) { .all, .line, .none => unreachable, - .failures => dep.state != .success, - .new => !dep.result_cached, + .failures => step.state != .success, + .new => !step.result_cached, }; if (found) break :blk i; } - break :blk deps.len -| 1; + break :blk top_level_steps.count(); }; - for (deps, 0..) |dep, i| { - var print_node: PrintNode = .{ - .parent = parent_node, - .last = i == last_index, + for (step_names, 0..) |step_name, i| { + const step_index = top_level_steps.get(step_name).?; + print_node.last = i + 1 == last_index; + printTreeStep(maker, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, }; - try printTreeStep(run, dep, stderr, &print_node, step_stack); } - } else { - if (deps.len == 0) { - try writer.writeAll(" (reused)\n"); - } else { - try writer.print(" (+{d} more reused dependencies)\n", .{deps.len}); - } - try stderr.setColor(.reset); } + w.writeByte('\n') catch {}; + } + + if (maker.watch or maker.web_server != null) return; + + // Perhaps in the future there could be an Advanced Options flag such as + // --debug-build-runner-leaks which would make this code return instead of + // calling exit. + + const code: u8 = code: { + if (failure_count == 0) break :code 0; // success + if (maker.error_style.verboseContext()) break :code 1; // failure; print build command + break :code 2; // failure; do not print build command + }; + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(code); +} + +fn stepReady( + maker: *Maker, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, +) Io.Cancelable!void { + const graph = maker.graph; + const io = graph.io; + const c = &maker.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + if (max_rss != 0) { + try maker.max_rss_mutex.lock(io); + defer maker.max_rss_mutex.unlock(io); + if (maker.available_rss < max_rss) { + // Running this step right now could possibly exceed the allotted RSS. + maker.memory_blocked_steps.append(maker.gpa, step_index) catch + @panic("TODO eliminate memory allocation here"); + return; + } + maker.available_rss -= max_rss; } + group.async(io, makeStep, .{ maker, group, step_index, root_prog_node }); +} + +/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready +/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must +/// have already subtracted this value from `maker.available_rss`. This function will release the RSS +/// claim (i.e. add `s.max_rss` back into `maker.available_rss`) and queue any viable memory-blocked +/// steps after "make" completes for `s`. +fn makeStep( + maker: *Maker, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, +) Io.Cancelable!void { + const graph = maker.graph; + const io = graph.io; + const gpa = maker.gpa; + const c = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const step_name = conf_step.name.slice(c); + const deps = conf_step.deps.slice(c); + const make_step = maker.stepByIndex(step_index); + + { + const step_prog_node = root_prog_node.start(step_name, 0); + defer step_prog_node.end(); + + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip); + + const new_state: Step.State = for (deps) |dep_index| { + const dep_make_step = maker.stepByIndex(dep_index); + switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .failure, + .dependency_failure, + .skipped_oom, + => break .dependency_failure, + + .success, .skipped => {}, + } + } else if (make_step.make(.{ + .progress_node = step_prog_node, + .watch = maker.watch, + .web_server = if (maker.web_server) |*ws| ws else null, + .unit_test_timeout_ns = maker.unit_test_timeout_ns, + .gpa = gpa, + })) state: { + break :state .success; + } else |err| switch (err) { + error.MakeFailed => .failure, + error.MakeSkipped => .skipped, + }; + + @atomicStore(Step.State, &make_step.state, new_state, .monotonic); - fn printStepStatus(run: *const Run, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { - const s = run.stepByIndex(step_index); - const writer = stderr.writer; - switch (s.state) { + switch (new_state) { .precheck_unstarted => unreachable, .precheck_started => unreachable, .precheck_done => unreachable, - .dependency_failure => { - try stderr.setColor(.dim); - try writer.writeAll(" transitive failure\n"); - try stderr.setColor(.reset); + .failure, + .dependency_failure, + .skipped_oom, + => { + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure); + std.Progress.setStatus(.failure_working); + }, + + .success, + .skipped, + => { + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success); + }, + } + } + + // No matter the result, we want to display error/warning messages. + if (make_step.result_error_bundle.errorMessageCount() > 0 or + make_step.result_error_msgs.items.len > 0 or + make_step.result_stderr.len > 0) + { + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + printErrorMessages(gpa, c, maker.steps, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) { + error.Canceled => |e| return e, + error.WriteFailed => switch (stderr.file_writer.err.?) { + error.Canceled => |e| return e, + else => {}, }, + else => {}, + }; + } + + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss != 0) { + var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty; + defer dispatch_set.deinit(gpa); + + // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` + // as a staging buffer to avoid recursing into `makeStep` while `maker.max_rss_mutex` is held. + { + try maker.max_rss_mutex.lock(io); + defer maker.max_rss_mutex.unlock(io); + maker.available_rss += max_rss; + dispatch_set.ensureUnusedCapacity(gpa, maker.memory_blocked_steps.items.len) catch + @panic("TODO eliminate memory allocation here"); + while (maker.memory_blocked_steps.getLast()) |candidate_index| { + const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes(); + if (maker.available_rss < candidate_max_rss) break; + assert(maker.memory_blocked_steps.pop() == candidate_index); + dispatch_set.appendAssumeCapacity(candidate_index); + } + } + for (dispatch_set.items) |candidate| { + group.async(io, makeStep, .{ maker, group, candidate, root_prog_node }); + } + } + + for (make_step.dependants.items) |dependant_index| { + const dependant = maker.stepByIndex(dependant_index); + // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. + if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { + try stepReady(maker, group, dependant_index, root_prog_node); + } + } +} + +fn printTreeStep( + maker: *const Maker, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + parent_node: *PrintNode, + step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), +) !void { + const writer = stderr.writer; + const first = step_stack.swapRemove(step_index); + const summary = maker.summary; + const c = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const make_step = maker.stepByIndex(step_index); + const skip = switch (summary) { + .none, .line => unreachable, + .all => false, + .new => make_step.result_cached, + .failures => make_step.state == .success, + }; + if (skip) return; + try printPrefix(parent_node, stderr); + + if (parent_node.parent != null) { + if (parent_node.last) { + try printChildNodePrefix(stderr); + } else { + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ + else => "+- ", + }); + } + } + + if (!first) try stderr.setColor(.dim); + + // dep_prefix omitted here because it is redundant with the tree. + try writer.writeAll(conf_step.name.slice(c)); - .success => { - try stderr.setColor(.green); - if (s.result_cached) { - try writer.writeAll(" cached"); - } else if (s.test_results.test_count > 0) { - const pass_count = s.test_results.passCount(); - assert(s.test_results.test_count == pass_count + s.test_results.skip_count); - try writer.print(" {d} pass", .{pass_count}); - if (s.test_results.skip_count > 0) { - try stderr.setColor(.reset); - try writer.writeAll(", "); - try stderr.setColor(.yellow); - try writer.print("{d} skip", .{s.test_results.skip_count}); - } + const deps = conf_step.deps.slice(c); + + if (first) { + try printStepStatus(maker, step_index, stderr); + + const last_index = if (summary == .all) deps.len -| 1 else blk: { + var i: usize = deps.len; + while (i > 0) { + i -= 1; + + const dep_index = deps[i]; + const dep = maker.stepByIndex(dep_index); + const found = switch (summary) { + .all, .line, .none => unreachable, + .failures => dep.state != .success, + .new => !dep.result_cached, + }; + if (found) break :blk i; + } + break :blk deps.len -| 1; + }; + for (deps, 0..) |dep, i| { + var print_node: PrintNode = .{ + .parent = parent_node, + .last = i == last_index, + }; + try printTreeStep(maker, dep, stderr, &print_node, step_stack); + } + } else { + if (deps.len == 0) { + try writer.writeAll(" (reused)\n"); + } else { + try writer.print(" (+{d} more reused dependencies)\n", .{deps.len}); + } + try stderr.setColor(.reset); + } +} + +fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { + const s = maker.stepByIndex(step_index); + const writer = stderr.writer; + switch (s.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .dependency_failure => { + try stderr.setColor(.dim); + try writer.writeAll(" transitive failure\n"); + try stderr.setColor(.reset); + }, + + .success => { + try stderr.setColor(.green); + if (s.result_cached) { + try writer.writeAll(" cached"); + } else if (s.test_results.test_count > 0) { + const pass_count = s.test_results.passCount(); + assert(s.test_results.test_count == pass_count + s.test_results.skip_count); + try writer.print(" {d} pass", .{pass_count}); + if (s.test_results.skip_count > 0) { try stderr.setColor(.reset); - try writer.print(" ({d} total)", .{s.test_results.test_count}); - } else { - try writer.writeAll(" success"); + try writer.writeAll(", "); + try stderr.setColor(.yellow); + try writer.print("{d} skip", .{s.test_results.skip_count}); } try stderr.setColor(.reset); - if (s.result_duration_ns) |ns| { - try stderr.setColor(.dim); - if (ns >= std.time.ns_per_min) { - try writer.print(" {d}m", .{ns / std.time.ns_per_min}); - } else if (ns >= std.time.ns_per_s) { - try writer.print(" {d}s", .{ns / std.time.ns_per_s}); - } else if (ns >= std.time.ns_per_ms) { - try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); - } else if (ns >= std.time.ns_per_us) { - try writer.print(" {d}us", .{ns / std.time.ns_per_us}); - } else { - try writer.print(" {d}ns", .{ns}); - } - try stderr.setColor(.reset); - } - if (s.result_peak_rss != 0) { - const rss = s.result_peak_rss; - try stderr.setColor(.dim); - if (rss >= 1000_000_000) { - try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); - } else if (rss >= 1000_000) { - try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); - } else if (rss >= 1000) { - try writer.print(" MaxRSS:{d}K", .{rss / 1000}); - } else { - try writer.print(" MaxRSS:{d}B", .{rss}); - } - try stderr.setColor(.reset); + try writer.print(" ({d} total)", .{s.test_results.test_count}); + } else { + try writer.writeAll(" success"); + } + try stderr.setColor(.reset); + if (s.result_duration_ns) |ns| { + try stderr.setColor(.dim); + if (ns >= std.time.ns_per_min) { + try writer.print(" {d}m", .{ns / std.time.ns_per_min}); + } else if (ns >= std.time.ns_per_s) { + try writer.print(" {d}s", .{ns / std.time.ns_per_s}); + } else if (ns >= std.time.ns_per_ms) { + try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); + } else if (ns >= std.time.ns_per_us) { + try writer.print(" {d}us", .{ns / std.time.ns_per_us}); + } else { + try writer.print(" {d}ns", .{ns}); } - try writer.writeAll("\n"); - }, - .skipped => { - try stderr.setColor(.yellow); - try writer.writeAll(" skipped\n"); try stderr.setColor(.reset); - }, - .skipped_oom => { - const c = &run.scanned_config.configuration; - const max_rss = step_index.ptr(c).max_rss.toBytes(); - try stderr.setColor(.yellow); - try writer.writeAll(" skipped (not enough memory)"); + } + if (s.result_peak_rss != 0) { + const rss = s.result_peak_rss; try stderr.setColor(.dim); - try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ - max_rss, run.available_rss, - }); - try stderr.setColor(.reset); - }, - .failure => { - try printStepFailure(run.steps, step_index, stderr, false); + if (rss >= 1000_000_000) { + try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); + } else if (rss >= 1000_000) { + try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); + } else if (rss >= 1000) { + try writer.print(" MaxRSS:{d}K", .{rss / 1000}); + } else { + try writer.print(" MaxRSS:{d}B", .{rss}); + } try stderr.setColor(.reset); - }, - } + } + try writer.writeAll("\n"); + }, + .skipped => { + try stderr.setColor(.yellow); + try writer.writeAll(" skipped\n"); + try stderr.setColor(.reset); + }, + .skipped_oom => { + const c = &maker.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + try stderr.setColor(.yellow); + try writer.writeAll(" skipped (not enough memory)"); + try stderr.setColor(.dim); + try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ + max_rss, maker.available_rss, + }); + try stderr.setColor(.reset); + }, + .failure => { + try printStepFailure(maker.steps, step_index, stderr, false); + try stderr.setColor(.reset); + }, } -}; +} fn printStepFailure( make_steps: []Step, -- 2.54.0 From b3d162d6bfe82d84d55edd58016347a420732fe2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 18 Feb 2026 14:09:55 -0800 Subject: [PATCH 013/179] build maker: rename files to match type --- lib/compiler/{maker.zig => Maker.zig} | 10 +++++----- lib/compiler/{maker => Maker}/Fuzz.zig | 6 +++--- lib/compiler/{maker => Maker}/Graph.zig | 0 lib/compiler/{maker => Maker}/Step.zig | 0 lib/compiler/{maker => Maker}/Step/Compile.zig | 0 lib/compiler/{maker => Maker}/Step/InstallArtifact.zig | 0 lib/compiler/{maker => Maker}/Step/Run.zig | 0 lib/compiler/{maker => Maker}/Step/WriteFile.zig | 0 lib/compiler/{maker => Maker}/Watch.zig | 0 lib/compiler/{maker => Maker}/Watch/FsEvents.zig | 0 lib/compiler/{maker => Maker}/WebServer.zig | 0 src/main.zig | 2 +- 12 files changed, 9 insertions(+), 9 deletions(-) rename lib/compiler/{maker.zig => Maker.zig} (99%) rename lib/compiler/{maker => Maker}/Fuzz.zig (99%) rename lib/compiler/{maker => Maker}/Graph.zig (100%) rename lib/compiler/{maker => Maker}/Step.zig (100%) rename lib/compiler/{maker => Maker}/Step/Compile.zig (100%) rename lib/compiler/{maker => Maker}/Step/InstallArtifact.zig (100%) rename lib/compiler/{maker => Maker}/Step/Run.zig (100%) rename lib/compiler/{maker => Maker}/Step/WriteFile.zig (100%) rename lib/compiler/{maker => Maker}/Watch.zig (100%) rename lib/compiler/{maker => Maker}/Watch/FsEvents.zig (100%) rename lib/compiler/{maker => Maker}/WebServer.zig (100%) diff --git a/lib/compiler/maker.zig b/lib/compiler/Maker.zig similarity index 99% rename from lib/compiler/maker.zig rename to lib/compiler/Maker.zig index 78b472530f5e8408bc07a4c5f1c9f0767a98fd66..15c2799185bb5610206e2dfda5ab61ee215898da 100644 --- a/lib/compiler/maker.zig +++ b/lib/compiler/Maker.zig @@ -16,11 +16,11 @@ const log = std.log; const mem = std.mem; const process = std.process; -const Fuzz = @import("maker/Fuzz.zig"); -const Graph = @import("maker/Graph.zig"); -const Step = @import("maker/Step.zig"); -const Watch = @import("maker/Watch.zig"); -const WebServer = @import("maker/WebServer.zig"); +const Fuzz = @import("Maker/Fuzz.zig"); +const Graph = @import("Maker/Graph.zig"); +const Step = @import("Maker/Step.zig"); +const Watch = @import("Maker/Watch.zig"); +const WebServer = @import("Maker/WebServer.zig"); pub const std_options: std.Options = .{ .side_channels_mitigations = .none, diff --git a/lib/compiler/maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig similarity index 99% rename from lib/compiler/maker/Fuzz.zig rename to lib/compiler/Maker/Fuzz.zig index 433439f2ceef355c4663419c67ba51b7f4507b9b..d3066d24afc31632e211e2883843adb497c83de7 100644 --- a/lib/compiler/maker/Fuzz.zig +++ b/lib/compiler/Maker/Fuzz.zig @@ -12,7 +12,7 @@ const assert = std.debug.assert; const fatal = std.process.fatal; const log = std.log; -const maker = @import("../maker.zig"); +const Maker = @import("../Maker.zig"); const WebServer = @import("WebServer.zig"); gpa: Allocator, @@ -179,7 +179,7 @@ fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, var buf: [256]u8 = undefined; const stderr = try io.lockStderr(&buf, graph.stderr_mode); defer io.unlockStderr(); - maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + Maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; } const rebuilt_bin_path = result catch |err| switch (err) { @@ -202,7 +202,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { error.Canceled => return, }; defer io.unlockStderr(); - maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + Maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; return; }, else => { diff --git a/lib/compiler/maker/Graph.zig b/lib/compiler/Maker/Graph.zig similarity index 100% rename from lib/compiler/maker/Graph.zig rename to lib/compiler/Maker/Graph.zig diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/Maker/Step.zig similarity index 100% rename from lib/compiler/maker/Step.zig rename to lib/compiler/Maker/Step.zig diff --git a/lib/compiler/maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig similarity index 100% rename from lib/compiler/maker/Step/Compile.zig rename to lib/compiler/Maker/Step/Compile.zig diff --git a/lib/compiler/maker/Step/InstallArtifact.zig b/lib/compiler/Maker/Step/InstallArtifact.zig similarity index 100% rename from lib/compiler/maker/Step/InstallArtifact.zig rename to lib/compiler/Maker/Step/InstallArtifact.zig diff --git a/lib/compiler/maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig similarity index 100% rename from lib/compiler/maker/Step/Run.zig rename to lib/compiler/Maker/Step/Run.zig diff --git a/lib/compiler/maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig similarity index 100% rename from lib/compiler/maker/Step/WriteFile.zig rename to lib/compiler/Maker/Step/WriteFile.zig diff --git a/lib/compiler/maker/Watch.zig b/lib/compiler/Maker/Watch.zig similarity index 100% rename from lib/compiler/maker/Watch.zig rename to lib/compiler/Maker/Watch.zig diff --git a/lib/compiler/maker/Watch/FsEvents.zig b/lib/compiler/Maker/Watch/FsEvents.zig similarity index 100% rename from lib/compiler/maker/Watch/FsEvents.zig rename to lib/compiler/Maker/Watch/FsEvents.zig diff --git a/lib/compiler/maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig similarity index 100% rename from lib/compiler/maker/WebServer.zig rename to lib/compiler/Maker/WebServer.zig diff --git a/src/main.zig b/src/main.zig index a24acdc14ec48af7be44fe003704912c12533649..1366deef2bc3646540643588915af21a1c571008 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5799,7 +5799,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn const main_mod_paths: Package.Module.CreateOptions.Paths = .{ .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"), - .root_src_path = "maker.zig", + .root_src_path = "Maker.zig", }; const config = try Compilation.Config.resolve(.{ -- 2.54.0 From 6b7ce1fa22b301ac06d3bfa0a8938546966f684c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 18 Feb 2026 17:30:54 -0800 Subject: [PATCH 014/179] massage Step code into compiling --- lib/compiler/Maker.zig | 117 ++++++++------ lib/compiler/Maker/Fuzz.zig | 58 ++++--- lib/compiler/Maker/Step.zig | 173 ++++++++++++++------ lib/compiler/Maker/Step/Compile.zig | 163 ++++++++++++++----- lib/compiler/Maker/Step/Run.zig | 243 ++++++++++++++-------------- lib/compiler/Maker/Watch.zig | 83 +++++----- lib/compiler/Maker/WebServer.zig | 165 +++++++++++-------- lib/std/Build.zig | 13 -- lib/std/Build/Step/Compile.zig | 40 ----- lib/std/zig/Configuration.zig | 26 +-- 10 files changed, 629 insertions(+), 452 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 15c2799185bb5610206e2dfda5ab61ee215898da..8101449cb4ea5f1d2442ab1bdcf0c5c36dbbbeb3 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -419,7 +419,7 @@ pub fn main(init: process.Init.Minimal) !void { var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; for (configuration.steps, 0..) |*conf_step, step_index_usize| { const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); - const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]); + const flags = conf_step.flags(&configuration); if (flags.tag == .top_level) { const name = step_index.ptr(&configuration).name.slice(&configuration); try top_level_steps.put(arena, name, step_index); @@ -538,7 +538,7 @@ pub fn main(init: process.Init.Minimal) !void { var w: Watch = w: { if (!watch) break :w undefined; if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); - break :w try .init(graph.cache.cwd, &scanned_config.configuration, maker.steps); + break :w try .init(&maker); }; const now = Io.Clock.Timestamp.now(io, .awake); @@ -546,14 +546,10 @@ pub fn main(init: process.Init.Minimal) !void { maker.web_server = if (webui_listen) |listen_address| ws: { if (builtin.single_threaded) unreachable; // `fatal` above break :ws .init(.{ - .gpa = gpa, - .graph = &graph, - .all_steps = maker.step_stack.keys(), + .maker = &maker, .root_prog_node = main_progress_node, - .watch = watch, .listen_address = listen_address, .base_timestamp = now, - .configuration = &scanned_config.configuration, }); } else null; @@ -564,7 +560,9 @@ pub fn main(init: process.Init.Minimal) !void { rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); defer io.unlockStderr(); - try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H"); + stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H") catch |err| switch (err) { + error.WriteFailed => return stderr.file_writer.err.?, + }; }) { if (maker.web_server) |*ws| ws.startBuild(); @@ -608,15 +606,15 @@ pub fn main(init: process.Init.Minimal) !void { // recursive dependants. var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ - w.dir_count, countSubProcesses(maker.steps, maker.step_stack.keys()), + w.dir_count, countSubProcesses(&maker), }) catch &caption_buf; var debouncing_node = main_progress_node.start(caption, 0); var in_debounce = false; - while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { + while (true) switch (try w.wait(if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { .timeout => { assert(in_debounce); debouncing_node.end(); - markFailedStepsDirty(gpa, maker.steps, maker.step_stack.keys()); + markFailedStepsDirty(&maker); continue :rebuild; }, .dirty => if (!in_debounce) { @@ -629,18 +627,20 @@ pub fn main(init: process.Init.Minimal) !void { } } -fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void { +fn markFailedStepsDirty(maker: *Maker) void { + const all_steps = maker.step_stack.keys(); + for (all_steps) |step_index| { - const step = &make_steps[@intFromEnum(step_index)]; + const step = maker.stepByIndex(step_index); switch (step.state) { - .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), + .dependency_failure, .failure, .skipped => _ = maker.invalidateResult(step), else => continue, } } // Now that all dirty steps have been found, the remaining steps that // succeeded from last run shall be marked "cached". for (all_steps) |step_index| { - const step = &make_steps[@intFromEnum(step_index)]; + const step = maker.stepByIndex(step_index); switch (step.state) { .success => step.result_cached = true, else => continue, @@ -648,10 +648,11 @@ fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const C } } -fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize { +fn countSubProcesses(maker: *Maker) usize { + const all_steps = maker.step_stack.keys(); var count: usize = 0; for (all_steps) |step_index| { - const s = &make_steps[@intFromEnum(step_index)]; + const s = maker.stepByIndex(step_index); count += @intFromBool(s.getZigProcess() != null); } return count; @@ -664,7 +665,7 @@ const InstallPaths = struct { include: Path, }; -fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step { +pub fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step { return &maker.steps[@intFromEnum(i)]; } @@ -676,7 +677,10 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void { const step_stack = &maker.step_stack; const c = &maker.scanned_config.configuration; - @memset(maker.steps, .{}); + for (maker.steps, 0..) |*step, step_index_usize| { + const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); + step.* = .{ .extended = .init(step_index.ptr(c).flags(c).tag) }; + } if (step_names.len == 0) { try step_stack.put(gpa, c.default_step, {}); @@ -699,7 +703,7 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void { rand.shuffle(Configuration.Step.Index, starting_steps); for (starting_steps) |s| { - try constructGraphAndCheckForDependencyLoop(gpa, c, maker.steps, s, &maker.step_stack, rand); + try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand); } { @@ -847,13 +851,8 @@ fn makeStepNames( } assert(mode == .limit); - var f = Fuzz.init( - gpa, - io, - step_stack.keys(), - parent_prog_node, - mode, - ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); + var f = Fuzz.init(maker, step_stack.keys(), parent_prog_node, mode) catch |err| + fatal("failed to start fuzzer: {t}", .{err}); defer f.deinit(); f.start(); @@ -1048,13 +1047,7 @@ fn makeStep( .success, .skipped => {}, } - } else if (make_step.make(.{ - .progress_node = step_prog_node, - .watch = maker.watch, - .web_server = if (maker.web_server) |*ws| ws else null, - .unit_test_timeout_ns = maker.unit_test_timeout_ns, - .gpa = gpa, - })) state: { + } else if (Step.make(step_index, maker, step_prog_node)) state: { break :state .success; } else |err| switch (err) { error.MakeFailed => .failure, @@ -1091,7 +1084,7 @@ fn makeStep( { const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); defer io.unlockStderr(); - printErrorMessages(gpa, c, maker.steps, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) { + printErrorMessages(maker, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) { error.Canceled => |e| return e, error.WriteFailed => switch (stderr.file_writer.err.?) { error.Canceled => |e| return e, @@ -1136,7 +1129,7 @@ fn makeStep( } fn printTreeStep( - maker: *const Maker, + maker: *Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal, parent_node: *PrintNode, @@ -1211,7 +1204,7 @@ fn printTreeStep( } } -fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { +fn printStepStatus(maker: *Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { const s = maker.stepByIndex(step_index); const writer = stderr.writer; switch (s.state) { @@ -1293,20 +1286,20 @@ fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, st try stderr.setColor(.reset); }, .failure => { - try printStepFailure(maker.steps, step_index, stderr, false); + try printStepFailure(maker, step_index, stderr, false); try stderr.setColor(.reset); }, } } fn printStepFailure( - make_steps: []Step, + maker: *Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal, dim: bool, ) !void { const w = stderr.writer; - const s = &make_steps[@intFromEnum(step_index)]; + const s = maker.stepByIndex(step_index); if (s.result_error_bundle.errorMessageCount() > 0) { try stderr.setColor(.red); try w.print(" {d} errors\n", .{ @@ -1428,14 +1421,14 @@ fn printChildNodePrefix(stderr: Io.Terminal) !void { /// when it finishes executing in `makeStep`, it spawns next steps to run in /// random order fn constructGraphAndCheckForDependencyLoop( - gpa: Allocator, - c: *const Configuration, - steps: []Step, + maker: *Maker, step_index: Configuration.Step.Index, step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), rand: std.Random, ) error{ DependencyLoopDetected, OutOfMemory }!void { - const make_step: *Step = &steps[@intFromEnum(step_index)]; + const c = &maker.scanned_config.configuration; + const gpa = maker.gpa; + const make_step = maker.stepByIndex(step_index); switch (make_step.state) { .precheck_started => { log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)}); @@ -1456,10 +1449,10 @@ fn constructGraphAndCheckForDependencyLoop( rand.shuffle(Configuration.Step.Index, deps); for (deps) |dep| { - const dep_step: *Step = &steps[@intFromEnum(dep)]; + const dep_step = maker.stepByIndex(dep); try step_stack.put(gpa, dep, {}); try dep_step.dependants.append(gpa, step_index); - constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) { + constructGraphAndCheckForDependencyLoop(maker, dep, step_stack, rand) catch |err| switch (err) { error.DependencyLoopDetected => { log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); return err; @@ -1482,16 +1475,34 @@ fn constructGraphAndCheckForDependencyLoop( } } +/// When file watching, prepares the step for being re-evaluated. Returns +/// `true` if the step was newly invalidated, `false` if it was already +/// invalidated. +pub fn invalidateResult(maker: *Maker, step: *Step) bool { + if (step.state == .precheck_done) return false; + const gpa = maker.gpa; + assert(step.pending_deps == 0); + step.state = .precheck_done; + step.reset(gpa); + for (step.dependants.items) |dependant_index| { + const dependant = maker.stepByIndex(dependant_index); + _ = invalidateResult(maker, dependant); + dependant.pending_deps += 1; + } + return true; +} + pub fn printErrorMessages( - gpa: Allocator, - c: *const Configuration, - make_steps: []Step, + maker: *Maker, failing_step_index: Configuration.Step.Index, options: std.zig.ErrorBundle.RenderOptions, stderr: Io.Terminal, error_style: ErrorStyle, multiline_errors: MultilineErrors, ) !void { + const c = &maker.scanned_config.configuration; + const gpa = maker.gpa; + log.err("TODO also report if result_oom flag is set", .{}); const writer = stderr.writer; if (error_style.verboseContext()) { // Provide context for where these error messages are coming from by @@ -1500,7 +1511,7 @@ pub fn printErrorMessages( defer step_stack.deinit(gpa); try step_stack.append(gpa, failing_step_index); while (true) { - const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])]; + const last_step = maker.stepByIndex(step_stack.items[step_stack.items.len - 1]); if (last_step.dependants.items.len == 0) break; try step_stack.append(gpa, last_step.dependants.items[0]); } @@ -1517,7 +1528,7 @@ pub fn printErrorMessages( try writer.writeAll(step_index.ptr(c).name.slice(c)); if (step_index == failing_step_index) { - try printStepFailure(make_steps, step_index, stderr, true); + try printStepFailure(maker, step_index, stderr, true); } else { try writer.writeAll("\n"); } @@ -1527,11 +1538,11 @@ pub fn printErrorMessages( // Just print the failing step itself. try stderr.setColor(.dim); try writer.writeAll(failing_step_index.ptr(c).name.slice(c)); - try printStepFailure(make_steps, failing_step_index, stderr, true); + try printStepFailure(maker, failing_step_index, stderr, true); try stderr.setColor(.reset); } - const failing_step = &make_steps[@intFromEnum(failing_step_index)]; + const failing_step = maker.stepByIndex(failing_step_index); if (failing_step.result_stderr.len > 0) { try writer.writeAll(failing_step.result_stderr); diff --git a/lib/compiler/Maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig index d3066d24afc31632e211e2883843adb497c83de7..77fab2b2df6dfe80bb4c4ae4dea33871992782fd 100644 --- a/lib/compiler/Maker/Fuzz.zig +++ b/lib/compiler/Maker/Fuzz.zig @@ -15,8 +15,7 @@ const log = std.log; const Maker = @import("../Maker.zig"); const WebServer = @import("WebServer.zig"); -gpa: Allocator, -io: Io, +maker: *Maker, mode: Mode, /// Allocated into `gpa`. @@ -76,12 +75,15 @@ const CoverageMap = struct { }; pub fn init( - gpa: Allocator, - io: Io, + maker: *Maker, all_steps: []const Configuration.Step.Index, root_prog_node: std.Progress.Node, mode: Mode, ) error{ OutOfMemory, Canceled }!Fuzz { + const graph = maker.graph; + const gpa = graph.cache.gpa; + const io = graph.io; + const run_steps: []const Configuration.Step.Index = steps: { var steps: std.ArrayList(Configuration.Step.Index) = .empty; defer steps.deinit(gpa); @@ -115,8 +117,7 @@ pub fn init( } return .{ - .gpa = gpa, - .io = io, + .maker = maker, .mode = mode, .run_steps = run_steps, .group = .init, @@ -131,7 +132,10 @@ pub fn init( } pub fn start(fuzz: *Fuzz) void { - const io = fuzz.io; + const maker = fuzz.maker; + const graph = maker.graph; + const io = graph.io; + fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0); if (fuzz.mode == .forever) { @@ -149,10 +153,14 @@ pub fn start(fuzz: *Fuzz) void { } pub fn deinit(fuzz: *Fuzz) void { - const io = fuzz.io; + const maker = fuzz.maker; + const graph = maker.graph; + const io = graph.io; + const gpa = maker.gpa; + fuzz.group.cancel(io); fuzz.prog_node.end(); - fuzz.gpa.free(fuzz.run_steps); + gpa.free(fuzz.run_steps); } fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void { @@ -215,19 +223,20 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { if (true) @panic("TODO"); assert(fuzz.mode == .forever); + const gpa = fuzz.maker.gpa; - var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); + var arena_state: std.heap.ArenaAllocator = .init(gpa); defer arena_state.deinit(); const arena = arena_state.allocator(); const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); var dedup_table: DedupTable = .empty; - defer dedup_table.deinit(fuzz.gpa); + defer dedup_table.deinit(gpa); for (fuzz.run_steps) |run_step| { const compile_inputs = run_step.producer.?.step.inputs.table; for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { - try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len); + try dedup_table.ensureUnusedCapacity(gpa, file_list.items.len); for (file_list.items) |sub_path| { if (!std.mem.endsWith(u8, sub_path, ".zig")) continue; const joined_path = try dir_path.join(arena, sub_path); @@ -266,7 +275,9 @@ pub fn sendUpdate( socket: *std.http.Server.WebSocket, prev: *Previous, ) !void { - const io = fuzz.io; + const maker = fuzz.maker; + const graph = maker.graph; + const io = graph.io; try fuzz.coverage_mutex.lock(io); defer fuzz.coverage_mutex.unlock(io); @@ -337,7 +348,9 @@ fn coverageRun(fuzz: *Fuzz) void { } fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { - const io = fuzz.io; + const maker = fuzz.maker; + const graph = maker.graph; + const io = graph.io; try fuzz.queue_mutex.lock(io); defer fuzz.queue_mutex.unlock(io); @@ -363,8 +376,10 @@ fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage if (true) @panic("TODO"); assert(fuzz.mode == .forever); const ws = fuzz.mode.forever.ws; - const gpa = fuzz.gpa; - const io = fuzz.io; + const maker = fuzz.maker; + const graph = maker.graph; + const io = graph.io; + const gpa = maker.gpa; try fuzz.coverage_mutex.lock(io); defer fuzz.coverage_mutex.unlock(io); @@ -470,7 +485,10 @@ fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage } fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void { - const io = fuzz.io; + const maker = fuzz.maker; + const graph = maker.graph; + const io = graph.io; + const gpa = maker.gpa; try fuzz.coverage_mutex.lock(io); defer fuzz.coverage_mutex.unlock(io); @@ -516,13 +534,15 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte }); } } - try coverage_map.entry_points.append(fuzz.gpa, @intCast(index)); + try coverage_map.entry_points.append(gpa, @intCast(index)); } pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { if (true) @panic("TODO"); assert(fuzz.mode == .limit); - const io = fuzz.io; + const maker = fuzz.maker; + const graph = maker.graph; + const io = graph.io; try fuzz.group.await(io); fuzz.group = .init; diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 845bc1e1f8ecb4191d945a5c3f86b390adc7290e..c43b502bc5e17b1dace1236ac5f5a8426fa72217 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -1,4 +1,5 @@ -//! The state that maker needs in order to process a step. +//! The *mutable* state that `Maker` needs in order to process one node from +//! the build graph. const Step = @This(); const builtin = @import("builtin"); @@ -14,14 +15,20 @@ const Configuration = std.Build.Configuration; const assert = std.debug.assert; const WebServer = @import("WebServer.zig"); +const Maker = @import("../Maker.zig"); -pub const Compile = void; // @import("Step/Compile.zig"); -pub const Run = void; // @import("Step/Run.zig"); +const Compile = @import("Step/Compile.zig"); +const Run = @import("Step/Run.zig"); /// Avoid false sharing. _: void align(std.atomic.cache_line) = {}, +/// Extra data for specific types of steps. +extended: Extended, + +/// This field is atomically accessed multi-threaded. state: State = .precheck_unstarted, + dependants: std.ArrayList(Configuration.Step.Index) = .empty, /// Collects the set of files that retrigger this step to run. /// @@ -38,6 +45,8 @@ result_error_msgs: std.ArrayList([]const u8) = .empty, result_error_bundle: std.zig.ErrorBundle = .empty, result_stderr: []const u8 = "", result_cached: bool = false, +/// Indicates error information is missing due to allocation failure. +result_oom: bool = false, result_duration_ns: ?u64 = null, /// 0 means unavailable or not reported. result_peak_rss: usize = 0, @@ -46,6 +55,70 @@ result_peak_rss: usize = 0, result_failed_command: ?[]const u8 = null, test_results: TestResults = .{}, +comptime { + // Common cache line size is 128. This check prevents accidentally crossing + // an additional cache line. In the future it might be nice to try to fit + // this struct in 128 bytes or less. + assert(@sizeOf(@This()) <= 128 * 3); +} + +pub const Extended = union(enum) { + check_file: Todo, + check_object: Todo, + compile: Compile, + config_header: Todo, + fail: Todo, + fmt: Todo, + install_artifact: Todo, + install_dir: Todo, + install_file: Todo, + objcopy: Todo, + options: Todo, + remove_dir: Todo, + run: Run, + top_level: Todo, + translate_c: Todo, + update_source_files: Todo, + write_file: Todo, + + pub fn init(tag: Configuration.Step.Tag) Extended { + return switch (tag) { + .check_file => .{ .check_file = .{} }, + .check_object => .{ .check_object = .{} }, + .compile => .{ .compile = .{} }, + .config_header => .{ .config_header = .{} }, + .fail => .{ .fail = .{} }, + .fmt => .{ .fmt = .{} }, + .install_artifact => .{ .install_artifact = .{} }, + .install_dir => .{ .install_dir = .{} }, + .install_file => .{ .install_file = .{} }, + .objcopy => .{ .objcopy = .{} }, + .options => .{ .options = .{} }, + .remove_dir => .{ .remove_dir = .{} }, + .run => .{ .run = .{} }, + .top_level => .{ .top_level = .{} }, + .translate_c => .{ .translate_c = .{} }, + .update_source_files => .{ .update_source_files = .{} }, + .write_file => .{ .write_file = .{} }, + }; + } + + pub const Todo = struct { + pub fn make( + todo: *Todo, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, + ) Step.ExtendedMakeError!void { + _ = todo; + _ = step_index; + _ = maker; + _ = progress_node; + @panic("TODO implement another step type"); + } + }; +}; + pub const State = enum { precheck_unstarted, precheck_started, @@ -128,43 +201,51 @@ pub const TestResults = struct { } }; -pub const MakeOptions = struct { - progress_node: std.Progress.Node, - watch: bool, - web_server: ?*WebServer, - /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. - unit_test_timeout_ns: ?u64, - /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. - gpa: Allocator, +pub const MakeError = error{ + /// Indicates the error is already reported. + MakeFailed, + MakeSkipped, }; -pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; +pub const ExtendedMakeError = MakeError || Allocator.Error; -/// If the Step's `make` function reports `error.MakeFailed`, it indicates they -/// have already reported the error. Otherwise, we add a simple error report -/// here. -pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { - if (true) @panic("TODO Step.make"); - const arena = s.owner.allocator; - const graph = s.owner.graph; +pub fn make( + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) MakeError!void { + const graph = maker.graph; + const process_arena = graph.arena; // TODO don't leak into the process arena const io = graph.io; + const c = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const s = maker.stepByIndex(step_index); var start_ts: ?Io.Timestamp = t: { if (!graph.time_report) break :t null; - if (s.id == .compile) break :t null; - if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; + const flags = conf_step.flags(c); + switch (flags.tag) { + .compile => break :t null, + .run => { + const run_flags: Configuration.Step.Run.Flags = @bitCast(flags); + if (run_flags.stdio == .zig_test) break :t null; + }, + else => {}, + } break :t Io.Clock.awake.now(io); }; - const make_result = s.makeFn(s, options); + const make_result = switch (s.extended) { + inline else => |*extended| extended.make(step_index, maker, progress_node), + }; if (start_ts) |*ts| { const duration = ts.untilNow(io, .awake); - options.web_server.?.updateTimeReportGeneric(s, duration); + maker.web_server.?.updateTimeReportGeneric(step_index, duration); } make_result catch |err| switch (err) { error.MakeFailed, error.MakeSkipped => |e| return e, - else => { - s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); + error.OutOfMemory => { + s.result_oom = true; return error.MakeFailed; }, }; @@ -173,30 +254,19 @@ pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!voi return error.MakeFailed; } - if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { - const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ - s.result_peak_rss, s.max_rss, - }) catch @panic("OOM"); - s.result_error_msgs.append(arena, msg) catch @panic("OOM"); + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss != 0 and s.result_peak_rss > max_rss) { + if (std.fmt.allocPrint( + process_arena, + "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", + .{ s.result_peak_rss, max_rss }, + )) |msg| { + s.oomWrap(s.result_error_msgs.append(process_arena, msg)); + } else |_| s.result_oom = true; } } -/// Implementation detail of file watching. Prepares the step for being re-evaluated. -/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. -pub fn invalidateResult(step: *Step, gpa: Allocator) bool { - if (true) @panic("TODO Step.invalidateResult"); - if (step.state == .precheck_done) return false; - assert(step.pending_deps == 0); - step.state = .precheck_done; - step.reset(gpa); - for (step.dependants.items) |dependant| { - _ = dependant.invalidateResult(gpa); - dependant.pending_deps += 1; - } - return true; -} - -/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. +/// Prepares the step for being re-evaluated. pub fn reset(step: *Step, gpa: Allocator) void { assert(step.state == .precheck_done); @@ -547,9 +617,8 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebSer } pub fn getZigProcess(s: *Step) ?*ZigProcess { - if (true) @panic("TODO getZigProcess"); - return switch (s.id) { - .compile => s.cast(Compile).?.zig_process, + return switch (s.extended) { + .compile => |*compile| compile.zig_process, else => null, }; } @@ -838,3 +907,9 @@ pub fn allocPrintCmd( } return aw.toOwnedSlice(); } + +fn oomWrap(s: *Step, result: error{OutOfMemory}!void) void { + result catch { + s.result_oom = true; + }; +} diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 5ffbc23cfc15d6d6de5df4f4585585364052da6d..5ef2ec9dacc021e2287af597107de953b88cb846 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -1,19 +1,40 @@ +const Compile = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Configuration = std.Build.Configuration; +const Dir = std.Io.Dir; +const Path = std.Build.Cache.Path; +const Module = std.Build.Configuration.Module; +const Io = std.Io; +const Sha256 = std.crypto.hash.sha2.Sha256; +const assert = std.debug.assert; +const mem = std.mem; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + /// Populated during the make phase when there is a long-lived compiler process. /// Managed by the build runner, not user build script. -zig_process: ?*Step.ZigProcess, +zig_process: ?*Step.ZigProcess = null, -fn make(step: *Step, options: Step.MakeOptions) !void { - const b = step.owner; - const compile: *Compile = @fieldParentPtr("step", step); - - const zig_args = try getZigArgs(compile, false); +pub fn make( + compile: *Compile, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + if (true) @panic("TODO implement compile.make()"); + const graph = maker.graph; + const step = maker.stepByIndex(step_index); + const zig_args = try getZigArgs(compile, maker, false); + const process_arena = graph.arena; // TODO don't leak into the process_arena const maybe_output_dir = step.evalZigProcess( zig_args, - options.progress_node, - (b.graph.incremental == true) and (options.watch or options.web_server != null), - options.web_server, - options.gpa, + progress_node, + (graph.incremental == true) and (maker.watch or maker.web_server != null), + maker, ) catch |err| switch (err) { error.NeedCompileErrorCheck => { assert(compile.expect_errors != null); @@ -26,7 +47,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { // Update generated files if (maybe_output_dir) |output_dir| { if (compile.emit_directory) |lp| { - lp.path = b.fmt("{f}", .{output_dir}); + lp.path = try std.fmt.allocPrint(process_arena, "{f}", .{output_dir}); } // zig fmt: off @@ -49,22 +70,23 @@ fn make(step: *Step, options: Step.MakeOptions) !void { { try doAtomicSymLinks( step, - compile.getEmittedBin().getPath2(b, step), + compile.getEmittedBin().getPath2(step.owner, step), compile.major_only_filename.?, compile.name_only_filename.?, ); } } -fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { +fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { const step = &compile.step; const b = step.owner; - const arena = b.allocator; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena var zig_args = std.array_list.Managed([]const u8).init(arena); defer zig_args.deinit(); - try zig_args.append(b.graph.zig_exe); + try zig_args.append(graph.zig_exe); const cmd = switch (compile.kind) { .lib => "build-lib", @@ -78,7 +100,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { if (b.reference_trace) |some| { try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); } - try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); + try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse graph.allow_so_scripts); try addFlag(&zig_args, "llvm", compile.use_llvm); try addFlag(&zig_args, "lld", compile.use_lld); @@ -118,7 +140,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { // module, along with any arguments that need to be passed to the // compiler for each module individually. var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; - var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; + var frameworks: std.StringArrayHashMapUnmanaged(Module.FrameworkFlags) = .empty; var prev_has_cflags = false; var prev_has_rcflags = false; @@ -130,7 +152,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { // Fully recursive iteration including dynamic libraries to detect // libc and libc++ linkage. - for (compile.getCompileDependencies(true)) |some_compile| { + for (getCompileDependencies(true)) |some_compile| { for (some_compile.root_module.getGraph().modules) |mod| { if (mod.link_libc == true) compile.is_linking_libc = true; if (mod.link_libcpp == true) compile.is_linking_libcpp = true; @@ -141,7 +163,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { // For this loop, don't chase dynamic libraries because their link // objects are already linked. - for (compile.getCompileDependencies(false)) |dep_compile| { + for (getCompileDependencies(false)) |dep_compile| { for (dep_compile.root_module.getGraph().modules) |mod| { // While walking transitive dependencies, if a given link object is // already included in a library, it should not redundantly be @@ -207,7 +229,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { switch (system_lib.use_pkg_config) { .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), .yes, .force => { - if (compile.runPkgConfig(system_lib.name)) |result| { + if (compile.runPkgConfig(maker, system_lib.name)) |result| { try zig_args.appendSlice(result.cflags); try zig_args.appendSlice(result.libs); try seen_system_libs.put(arena, system_lib.name, result.cflags); @@ -227,7 +249,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { })); }, .force => { - panic("pkg-config failed for library {s}", .{system_lib.name}); + return step.fail("pkg-config failed for library {s}", .{system_lib.name}); }, .no => unreachable, }, @@ -272,7 +294,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { if (other.linkage == .dynamic and compile.rootModuleTarget().os.tag != .windows) { - if (fs.path.dirname(full_path_lib)) |dirname| { + if (Dir.path.dirname(full_path_lib)) |dirname| { try zig_args.append("-rpath"); try zig_args.append(dirname); } @@ -479,7 +501,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); - if (b.graph.time_report) try zig_args.append("--time-report"); + if (graph.time_report) try zig_args.append("--time-report"); if (compile.generated_asm != null) try zig_args.append("-femit-asm"); if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); @@ -555,9 +577,9 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { try zig_args.append(b.cache_root.path orelse "."); try zig_args.append("--global-cache-dir"); - try zig_args.append(b.graph.global_cache_root.path orelse "."); + try zig_args.append(graph.global_cache_root.path orelse "."); - if (b.graph.debug_compiler_runtime_libs) |mode| + if (graph.debug_compiler_runtime_libs) |mode| try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); try zig_args.append("--name"); @@ -681,7 +703,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { // -I and -L arguments that appear after the last --mod argument apply to all modules. const cwd: Io.Dir = .cwd(); - const io = b.graph.io; + const io = graph.io; for (b.search_prefixes.items) |search_prefix| { var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { @@ -734,8 +756,8 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| dir.getPath2(b, step) - else if (b.graph.zig_lib_directory.path) |_| - b.fmt("{f}", .{b.graph.zig_lib_directory}) + else if (graph.zig_lib_directory.path) |_| + b.fmt("{f}", .{graph.zig_lib_directory}) else null; @@ -769,7 +791,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { "--error-limit", b.fmt("{d}", .{err_limit}), }); - try addFlag(&zig_args, "incremental", b.graph.incremental); + try addFlag(&zig_args, "incremental", graph.incremental); try zig_args.append("--listen=-"); @@ -814,7 +836,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); - const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; + const args_file = "args" ++ Dir.path.sep_str ++ args_hex_hash; if (b.cache_root.handle.access(io, args_file, .{})) |_| { // The args file is already present from a previous run. } else |err| switch (err) { @@ -859,7 +881,9 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { return try zig_args.toOwnedSlice(); } -pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { +pub fn rebuildInFuzzMode(c: *Compile, maker: *Maker, progress_node: std.Progress.Node) !Path { + const gpa = maker.graph.gpa; + c.step.result_error_msgs.clearRetainingCapacity(); c.step.result_stderr = ""; @@ -871,21 +895,23 @@ pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progres c.step.result_failed_command = null; } - const zig_args = try getZigArgs(c, true); + const zig_args = try getZigArgs(c, maker, true); const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); return maybe_output_bin_path.?; } pub fn doAtomicSymLinks( step: *Step, + maker: *Maker, output_path: []const u8, filename_major_only: []const u8, filename_name_only: []const u8, ) !void { const b = step.owner; - const io = b.graph.io; - const out_dir = fs.path.dirname(output_path) orelse "."; - const out_basename = fs.path.basename(output_path); + const graph = maker.graph; + const io = graph.io; + const out_dir = Dir.path.dirname(output_path) orelse "."; + const out_basename = Dir.path.basename(output_path); // sym link for libfoo.so.1 to libfoo.so.1.2.3 const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); const cwd: Io.Dir = .cwd(); @@ -903,10 +929,24 @@ pub fn doAtomicSymLinks( }; } -fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { - const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); - var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); +pub const PkgConfigError = error{ + PkgConfigCrashed, + PkgConfigFailed, + PkgConfigNotInstalled, + PkgConfigInvalidOutput, +}; + +pub const PkgConfigPkg = struct { + name: []const u8, + desc: []const u8, +}; + +fn execPkgConfigList(maker: *Maker, out_code: *u8) (PkgConfigError || Maker.RunError)![]const PkgConfigPkg { + const graph = maker.graph; + const process_arena = graph.arena; // TODO don't leak into process arena + const pkg_config_exe = graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = try maker.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); + var list = std.array_list.Managed(PkgConfigPkg).init(process_arena); errdefer list.deinit(); var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); while (line_it.next()) |line| { @@ -960,7 +1000,8 @@ const PkgConfigResult = struct { /// Run pkg-config for the given library name and parse the output, returning the arguments /// that should be passed to zig to link the given library. -fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult { +fn runPkgConfig(compile: *Compile, maker: *Maker, lib_name: []const u8) !PkgConfigResult { + const graph = maker.graph; const wl_rpath_prefix = "-Wl,-rpath,"; const b = compile.step.owner; @@ -1013,7 +1054,7 @@ fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult { }; var code: u8 = undefined; - const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; + const pkg_config_exe = graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; const stdout = if (b.runAllowFail(&[_][]const u8{ pkg_config_exe, pkg_name, @@ -1198,3 +1239,43 @@ fn moduleNeedsCliArg(mod: *const Module) bool { } else false; } +const CliNamedModules = struct { + modules: std.AutoArrayHashMapUnmanaged(*Module, void), + names: std.StringArrayHashMapUnmanaged(void), + + /// Traverse the whole dependency graph and give every module a unique + /// name, ideally one named after what it's called somewhere in the graph. + /// It will help here to have both a mapping from module to name and a set + /// of all the currently-used names. + fn init(arena: Allocator, root_module: *Module) Allocator.Error!CliNamedModules { + var compile: CliNamedModules = .{ + .modules = .{}, + .names = .{}, + }; + const graph = root_module.getGraph(); + { + assert(graph.modules[0] == root_module); + try compile.modules.put(arena, root_module, {}); + try compile.names.put(arena, "root", {}); + } + for (graph.modules[1..], graph.names[1..]) |mod, orig_name| { + var name = orig_name; + var n: usize = 0; + while (true) { + const gop = try compile.names.getOrPut(arena, name); + if (!gop.found_existing) { + try compile.modules.putNoClobber(arena, mod, {}); + break; + } + name = try std.fmt.allocPrint(arena, "{s}{d}", .{ orig_name, n }); + n += 1; + } + } + return compile; + } +}; + +fn getCompileDependencies(chase_dynamic: bool) void { + _ = chase_dynamic; + @panic("TODO"); +} diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 4ba04092e02221fffc55729a64b44aa41a5b9d91..122c394e0d93d6b293545e8677d16d142c9dd27c 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -3,38 +3,46 @@ const Run = @This(); const builtin = @import("builtin"); const std = @import("std"); -const Io = std.Io; +const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; const Dir = std.Io.Dir; -const mem = std.mem; -const process = std.process; const EnvMap = std.process.Environ.Map; -const assert = std.debug.assert; -const Cache = std.Build.Cache; +const Io = std.Io; const Path = std.Build.Cache.Path; +const assert = std.debug.assert; +const mem = std.mem; +const process = std.process; const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); /// If this is a Zig unit test binary, this tracks the names of the unit /// tests that are also fuzz tests. Indexes cannot be used as they may /// change between reruns. -fuzz_tests: std.ArrayList([]const u8), +fuzz_tests: std.ArrayList([]const u8) = .empty, cached_test_metadata: ?CachedTestMetadata = null, /// Populated during the fuzz phase if this run step corresponds to a unit test /// executable that contains fuzz tests. -rebuilt_executable: ?Path, +rebuilt_executable: ?Path = null, -fn make(step: *Step, options: Step.MakeOptions) !void { - const b = step.owner; - const io = b.graph.io; - const arena = b.allocator; - const run: *Run = @fieldParentPtr("step", step); +pub fn make( + run: *Run, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + if (true) @panic("TODO implement run.make()"); + const graph = maker.graph; + const step = maker.stepByIndex(step_index); + const io = graph.io; + const arena = graph.arena; // TODO don't leak into the process arena const has_side_effects = run.hasSideEffects(); var argv_list = std.array_list.Managed([]const u8).init(arena); var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); - var man = b.graph.cache.obtain(); + var man = graph.cache.obtain(); defer man.deinit(); if (run.environ_map) |environ_map| { @@ -54,19 +62,19 @@ fn make(step: *Step, options: Step.MakeOptions) !void { man.hash.addBytes(bytes); }, .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); + const file_path = file.lazy_path.getPath3(graph, step); + try argv_list.append(graph.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(maker, file_path) })); man.hash.addBytes(file.prefix); _ = try man.addFilePath(file_path, null); }, .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); + const file_path = dd.lazy_path.getPath3(graph, step); + const resolved_arg = graph.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(maker, file_path), dd.suffix }); try argv_list.append(resolved_arg); man.hash.addBytes(resolved_arg); }, .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); + const file_path = file_plp.lazy_path.getPath3(graph, step); var result: std.Io.Writer.Allocating = .init(arena); errdefer result.deinit(); @@ -99,13 +107,13 @@ fn make(step: *Step, options: Step.MakeOptions) !void { if (artifact.rootModuleTarget().os.tag == .windows) { // On Windows we don't have rpaths so we have to add .dll search paths to PATH - run.addPathForDynLibs(artifact); + addPathForDynLibs(artifact); } const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; - try argv_list.append(b.fmt("{s}{s}", .{ + try argv_list.append(graph.fmt("{s}{s}", .{ pa.prefix, - run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), + run.convertPathArg(maker, .{ .root_dir = .cwd(), .sub_path = file_path }), })); _ = try man.addFile(file_path, null); @@ -131,7 +139,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { man.hash.addBytes(bytes); }, .lazy_path => |lazy_path| { - const file_path = lazy_path.getPath2(b, step); + const file_path = lazy_path.getPath2(graph, step); _ = try man.addFile(file_path, null); }, .none => {}, @@ -147,14 +155,15 @@ fn make(step: *Step, options: Step.MakeOptions) !void { man.hash.add(captured.trim_whitespace); } - hashStdIo(&man.hash, run.stdio); + std.log.err("TODO hashStdIo", .{}); + //hashStdIo(&man.hash, run.stdio); for (run.file_inputs.items) |lazy_path| { - _ = try man.addFile(lazy_path.getPath2(b, step), null); + _ = try man.addFile(lazy_path.getPath2(graph, step), null); } if (run.cwd) |cwd| { - const cwd_path = cwd.getPath3(b, step); + const cwd_path = cwd.getPath3(graph, step); _ = man.hash.addBytes(try cwd_path.toString(arena)); } @@ -165,9 +174,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { try populateGeneratedPaths( arena, output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, + graph.cache_root, &digest, ); @@ -185,36 +192,34 @@ fn make(step: *Step, options: Step.MakeOptions) !void { try populateGeneratedPaths( arena, output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, + graph.cache_root, &digest, ); const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; for (output_placeholders.items) |placeholder| { - const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); + const output_sub_path = graph.pathJoin(&.{ output_dir_path, placeholder.output.basename }); const output_sub_dir_path = switch (placeholder.tag) { .output_file => Dir.path.dirname(output_sub_path).?, .output_directory => output_sub_path, else => unreachable, }; - b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, output_sub_dir_path, @errorName(err), + graph.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + return step.fail("unable to make path '{f}{s}': {t}", .{ + graph.cache_root, output_sub_dir_path, err, }); }; - const arg_output_path = run.convertPathArg(.{ + const arg_output_path = run.convertPathArg(maker, .{ .root_dir = .cwd(), .sub_path = placeholder.output.generated_file.getPath(), }); argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) arg_output_path else - b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); + graph.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); } - try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); + try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); if (!has_side_effects) try step.writeManifestAndWatch(&man); return; }; @@ -226,32 +231,32 @@ fn make(step: *Step, options: Step.MakeOptions) !void { for (output_placeholders.items) |placeholder| { const output_components = .{ tmp_dir_path, placeholder.output.basename }; - const output_sub_path = b.pathJoin(&output_components); + const output_sub_path = graph.pathJoin(&output_components); const output_sub_dir_path = switch (placeholder.tag) { .output_file => Dir.path.dirname(output_sub_path).?, .output_directory => output_sub_path, else => unreachable, }; - b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, output_sub_dir_path, @errorName(err), + graph.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + return step.fail("unable to make path '{f}{s}': {t}", .{ + graph.cache_root, output_sub_dir_path, err, }); }; - const raw_output_path: Cache.Path = .{ - .root_dir = b.cache_root, - .sub_path = b.pathJoin(&output_components), + const raw_output_path: Path = .{ + .root_dir = graph.cache_root, + .sub_path = graph.pathJoin(&output_components), }; - placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); - argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ + placeholder.output.generated_file.path = raw_output_path.toString(arena) catch @panic("OOM"); + argv_list.items[placeholder.index] = graph.fmt("{s}{s}", .{ placeholder.output.prefix, - run.convertPathArg(raw_output_path), + run.convertPathArg(maker, raw_output_path), }); } - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); + try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null); const dep_file_dir = Dir.cwd(); - const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); + const dep_file_basename = dep_output_file.generated_file.getPath2(graph, step); if (has_side_effects) try man.addDepFile(dep_file_dir, dep_file_basename) else @@ -269,21 +274,21 @@ fn make(step: *Step, options: Step.MakeOptions) !void { if (any_output) { const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; - b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { + graph.cache_root.handle.rename(tmp_dir_path, graph.cache_root.handle, o_sub_path, io) catch |err| switch (err) { Dir.RenameError.DirNotEmpty => { - b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { + graph.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { return step.fail("unable to remove dir '{f}'{s}: {t}", .{ - b.cache_root, tmp_dir_path, del_err, + graph.cache_root, tmp_dir_path, del_err, }); }; - b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { + graph.cache_root.handle.rename(tmp_dir_path, graph.cache_root.handle, o_sub_path, io) catch |retry_err| { return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, + graph.cache_root, tmp_dir_path, graph.cache_root, o_sub_path, retry_err, }); }; }, else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, + graph.cache_root, tmp_dir_path, graph.cache_root, o_sub_path, err, }), }; } @@ -293,9 +298,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { try populateGeneratedPaths( arena, output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, + graph.cache_root, &digest, ); } @@ -347,7 +350,6 @@ fn waitZigTest( // start and it acknowledging the test starting, we terminate the child and raise an error. This // *should* never happen, but could in theory be caused by some very unlucky IB in a test. const response_timeout: Io.Clock.Duration = t: { - if (fuzz_context != null) break :t null; // don't timeout fuzz tests const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; }; @@ -773,8 +775,8 @@ const FuzzTestRunner = struct { try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); f.pending_broadcasts.appendSliceAssumeCapacity(body); f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); - } - }, + } + }, else => {}, // ignore other messages } @@ -863,7 +865,7 @@ const FuzzTestRunner = struct { if (f.coverage_id == null) return; // Search for the input file corresponding to the instance - const InputHeader = Build.abi.fuzz.MmapInputHeader; + const InputHeader = std.Build.abi.fuzz.MmapInputHeader; var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; var in_r: Io.File.Reader = undefined; var in_f: Io.File = undefined; @@ -1299,11 +1301,11 @@ fn sendRunFuzzTestMessage( } } -fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { - const b = run.step.owner; - const io = b.graph.io; - const arena = b.allocator; - const gpa = b.allocator; +fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !EvalGenericResult { + const graph = maker.graph; + const io = graph.io; + const arena = graph.allocator; // TODO don't leak into the process arena + const gpa = maker.gpa; var child = try process.spawn(io, spawn_options); defer child.kill(io); @@ -1317,7 +1319,7 @@ fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResul child.stdin = null; }, .lazy_path => |lazy_path| { - const path = lazy_path.getPath3(b, &run.step); + const path = lazy_path.getPath3(graph, &run.step); const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { return run.step.fail("unable to open stdin file: {t}", .{err}); }; @@ -1417,18 +1419,22 @@ fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResul const IndexedOutput = struct { index: usize, - tag: @typeInfo(Arg).@"union".tag_type.?, + tag: Configuration.Step.Run.Arg.Tag, output: *Output, }; +const Output = void; // TODO + pub fn rerunInFuzzMode( run: *Run, fuzz: *std.Build.Fuzz, prog_node: std.Progress.Node, ) !void { + const maker = fuzz.maker; + const graph = maker.graph; const step = &run.step; const b = step.owner; - const io = b.graph.io; + const io = graph.io; const arena = b.allocator; var argv_list: std.ArrayList([]const u8) = .empty; for (run.argv.items) |arg| { @@ -1438,11 +1444,11 @@ pub fn rerunInFuzzMode( }, .lazy_path => |file| { const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(maker, file_path) })); }, .decorated_directory => |dd| { const file_path = dd.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); + try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(maker, file_path), dd.suffix })); }, .file_content => |file_plp| { const file_path = file_plp.lazy_path.getPath3(b, step); @@ -1471,7 +1477,7 @@ pub fn rerunInFuzzMode( }; try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, - run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), + run.convertPathArg(maker, .{ .root_dir = .cwd(), .sub_path = file_path }), })); }, .output_file, .output_directory => unreachable, @@ -1487,17 +1493,13 @@ pub fn rerunInFuzzMode( var rand_int: u64 = undefined; io.random(@ptrCast(&rand_int)); const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ - .progress_node = prog_node, - .watch = undefined, // not used by `runCommand` - .web_server = null, // only needed for time reports - .unit_test_timeout_ns = null, // don't time out fuzz tests for now - .gpa = fuzz.gpa, - }, .{ + try runCommand(run, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{ .fuzz = fuzz, }); } +const CapturedStdIo = void; // TODO get it from Configuration + fn populateGeneratedPaths( arena: std.mem.Allocator, output_placeholders: []const IndexedOutput, @@ -1545,17 +1547,19 @@ const FuzzContext = struct { fn runCommand( run: *Run, + maker: *Maker, + progress_node: std.Progress.Node, argv: []const []const u8, has_side_effects: bool, output_dir_path: []const u8, - options: Step.MakeOptions, fuzz_context: ?FuzzContext, ) !void { + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into process arena + const gpa = maker.gpa; const step = &run.step; const b = step.owner; - const arena = b.allocator; - const gpa = options.gpa; - const io = b.graph.io; + const io = graph.io; const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; @@ -1571,12 +1575,12 @@ fn runCommand( defer interp_argv.deinit(); var environ_map: EnvMap = env: { - const orig = run.environ_map orelse &b.graph.environ_map; + const orig = run.environ_map orelse &graph.environ_map; break :env try orig.clone(gpa); }; defer environ_map.deinit(); - const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { + const opt_generic_result = spawnChildAndCollect(run, maker, progress_node, argv, &environ_map, has_side_effects, fuzz_context) catch |err| term: { // InvalidExe: cpu arch mismatch // FileNotFound: can happen with a wrong dynamic linker path if (err == error.InvalidExe or err == error.FileNotFound) interpret: { @@ -1597,7 +1601,7 @@ fn runCommand( const need_cross_libc = exe.is_linking_libc and (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); const other_target = exe.root_module.resolved_target.?.result; - switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ + switch (std.zig.system.getExternalExecutor(io, &graph.host.result, &other_target, .{ .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, .link_libc = exe.is_linking_libc, })) { @@ -1669,7 +1673,7 @@ fn runCommand( .bad_dl => |foreign_dl| { if (allow_skip) return error.MakeSkipped; - const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; + const host_dl = graph.host.result.dynamic_linker.get() orelse "(none)"; return step.fail( \\the host system is unable to execute binaries from the target @@ -1681,7 +1685,7 @@ fn runCommand( .bad_os_or_cpu => { if (allow_skip) return error.MakeSkipped; - const host_name = try b.graph.host.result.zigTriple(b.allocator); + const host_name = try graph.host.result.zigTriple(b.allocator); const foreign_name = try root_target.zigTriple(b.allocator); return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ @@ -1692,14 +1696,14 @@ fn runCommand( if (root_target.os.tag == .windows) { // On Windows we don't have rpaths so we have to add .dll search paths to PATH - run.addPathForDynLibs(exe); + addPathForDynLibs(exe); } gpa.free(step.result_failed_command.?); step.result_failed_command = null; try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); - break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { + break :term spawnChildAndCollect(run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; if (e == error.MakeFailed) return error.MakeFailed; // error already reported return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); @@ -1851,14 +1855,16 @@ const EvalGenericResult = struct { fn spawnChildAndCollect( run: *Run, + maker: *Maker, + progress_node: std.Progress.Node, argv: []const []const u8, environ_map: *EnvMap, has_side_effects: bool, - options: Step.MakeOptions, fuzz_context: ?FuzzContext, ) !?EvalGenericResult { const b = run.step.owner; - const graph = b.graph; + const graph = maker.graph; + const gpa = maker.gpa; const io = graph.io; if (fuzz_context != null) { @@ -1870,7 +1876,7 @@ fn spawnChildAndCollect( // If an error occurs, it's caused by this command: assert(run.step.result_failed_command == null); - run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ + run.step.result_failed_command = try Step.allocPrintCmd(gpa, child_cwd, .{ .child = environ_map, .parent = &graph.environ_map, }, argv); @@ -1905,7 +1911,7 @@ fn spawnChildAndCollect( if (run.stdio == .zig_test) { const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { + const result = evalZigTest(run, maker, progress_node, spawn_options, fuzz_context) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| e, }; @@ -1915,7 +1921,7 @@ fn spawnChildAndCollect( } else { const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; if (!run.disable_zig_progress and !inherit) { - spawn_options.progress_node = options.progress_node; + spawn_options.progress_node = progress_node; } const terminal_mode: Io.Terminal.Mode = if (inherit) m: { const stderr = try io.lockStderr(&.{}, graph.stderr_mode); @@ -1925,7 +1931,7 @@ fn spawnChildAndCollect( try setColorEnvironmentVariables(run, environ_map, terminal_mode); const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalGeneric(run, spawn_options) catch |err| switch (err) { + const result = evalGeneric(run, maker, spawn_options) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| e, }; @@ -1934,11 +1940,11 @@ fn spawnChildAndCollect( } } -fn hashStdIo(hh: *Cache.HashHelper, stdio: StdIo) void { +fn hashStdIo(hh: *Cache.HashHelper, stdio: void) void { switch (stdio) { .infer_from_args, .inherit, .zig_test => {}, .check => |checks| for (checks.items) |check| { - hh.add(@as(std.meta.Tag(StdIo.Check), check)); + hh.add(@as(std.meta.Tag(@This().StdIo.Check), check)); switch (check) { .expect_stderr_exact, .expect_stderr_match, @@ -2010,7 +2016,7 @@ fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: } } -fn checksContainStdout(checks: []const StdIo.Check) bool { +fn checksContainStdout(checks: []const @This().StdIo.Check) bool { for (checks) |check| switch (check) { .expect_stderr_exact, .expect_stderr_match, @@ -2024,7 +2030,7 @@ fn checksContainStdout(checks: []const StdIo.Check) bool { return false; } -fn checksContainStderr(checks: []const StdIo.Check) bool { +fn checksContainStderr(checks: []const @This().StdIo.Check) bool { for (checks) |check| switch (check) { .expect_stdout_exact, .expect_stdout_match, @@ -2063,9 +2069,9 @@ fn hasAnyOutputArgs(run: Run) bool { /// /// Whenever a path is included in the argv of a child, it should be put through this function first /// to make sure the child doesn't see paths relative to a cwd other than its own. -fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { +fn convertPathArg(run: *Run, maker: *Maker, path: Path) []const u8 { const b = run.step.owner; - const graph = b.graph; + const graph = maker.graph; const arena = graph.arena; const path_str = path.toString(arena) catch @panic("OOM"); @@ -2091,40 +2097,43 @@ fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); } -fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { - const b = run.step.owner; - const compiles = artifact.getCompileDependencies(true); - for (compiles) |compile| { +fn addPathForDynLibs(artifact: *Step.Compile) void { + if (true) @panic("TODO"); + for (artifact.getCompileDependencies(true)) |compile| { if (compile.root_module.resolved_target.?.result.os.tag == .windows and compile.isDynamicLibrary()) { - addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); + @panic("TODO"); + //addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); } } } fn failForeign( run: *Run, + maker: *Maker, + step_index: Configuration.Step.Index, suggested_flag: []const u8, argv0: []const u8, exe: *Step.Compile, -) error{ MakeFailed, MakeSkipped, OutOfMemory } { +) Step.ExtendedMakeError { + const step = maker.stepByIndex(step_index); switch (run.stdio) { .check, .zig_test => { - if (run.skip_foreign_checks) - return error.MakeSkipped; + if (run.skip_foreign_checks) return error.MakeSkipped; - const b = run.step.owner; - const host_name = try b.graph.host.result.zigTriple(b.allocator); - const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); + const graph = maker.graph; + const process_arena = graph.arena; // TODO don't leak into process arena + const host_name = try graph.host.result.zigTriple(process_arena); + const foreign_name = try exe.rootModuleTarget().zigTriple(process_arena); - return run.step.fail( + return step.fail( \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) \\ consider using {s} or enabling skip_foreign_checks in the Run step , .{ argv0, foreign_name, host_name, suggested_flag }); }, else => { - return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); + return step.fail("unable to spawn foreign binary '{s}'", .{argv0}); }, } } diff --git a/lib/compiler/Maker/Watch.zig b/lib/compiler/Maker/Watch.zig index 907e6536a132863603d70e423041b41c5b677a5c..b59f616d995c1b7c77893714cdf441563ea7c1c6 100644 --- a/lib/compiler/Maker/Watch.zig +++ b/lib/compiler/Maker/Watch.zig @@ -10,6 +10,7 @@ const Configuration = std.Build.Configuration; const FsEvents = @import("Watch/FsEvents.zig"); const Step = @import("Step.zig"); +const Maker = @import("../Maker.zig"); os: Os, /// The number to show as the number of directories being watched. @@ -18,8 +19,7 @@ dir_count: usize, // They are `undefined` on implementations which do not utilize then. dir_table: DirTable, generation: Generation, -configuration: *const Configuration, -make_steps: []Step, +maker: *Maker, pub const have_impl = Os != void; @@ -105,8 +105,7 @@ const Os = switch (builtin.os.tag) { }; }; - fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { - _ = cwd_path; + fn init(maker: *Maker) !Watch { return .{ .dir_table = .{}, .dir_count = 0, @@ -118,8 +117,7 @@ const Os = switch (builtin.os.tag) { else => {}, }, .generation = 0, - .make_steps = make_steps, - .configuration = configuration, + .maker = maker, }; } @@ -136,7 +134,8 @@ const Os = switch (builtin.os.tag) { return stack_lfh.clone(gpa); } - fn markDirtySteps(w: *Watch, gpa: Allocator, fan_fd: posix.fd_t) !bool { + fn markDirtySteps(w: *Watch, fan_fd: posix.fd_t) !bool { + const maker = w.maker; const fanotify = std.os.linux.fanotify; const M = fanotify.event_metadata; var events_buf: [256 + 4096]u8 = undefined; @@ -155,7 +154,7 @@ const Os = switch (builtin.os.tag) { if (meta[0].mask.Q_OVERFLOW) { any_dirty = true; std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); - markAllFilesDirty(w, gpa); + markAllFilesDirty(w); return true; } const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); @@ -167,9 +166,9 @@ const Os = switch (builtin.os.tag) { const lfh: FileHandle = .{ .handle = file_handle }; if (w.os.handle_table.getPtr(lfh)) |value| { if (value.reaction_set.getPtr(".")) |glob_set| - any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty); + any_dirty = markStepSetDirty(maker, glob_set, any_dirty); if (value.reaction_set.getPtr(file_name)) |step_set| - any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty); + any_dirty = markStepSetDirty(maker, step_set, any_dirty); } }, else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}), @@ -179,9 +178,11 @@ const Os = switch (builtin.os.tag) { } fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + const maker = w.maker; + // Add missing marks and note persisted ones. for (steps) |step_index| { - const step = &w.make_steps[@intFromEnum(step_index)]; + const step = maker.stepByIndex(step_index); for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { const reaction_set = rs: { const gop = try w.dir_table.getOrPut(gpa, path); @@ -298,13 +299,12 @@ const Os = switch (builtin.os.tag) { w.dir_count = w.dir_table.count(); } - fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - _ = io; + fn wait(w: *Watch, timeout: Timeout) !WaitResult { const events_len = try std.posix.poll(w.os.poll_fds.values(), timeout.to_i32_ms()); if (events_len == 0) return .timeout; for (w.os.poll_fds.values()) |poll_fd| { - if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try markDirtySteps(w, gpa, poll_fd.fd)) + if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try markDirtySteps(w, poll_fd.fd)) return .dirty; } return .clean; @@ -515,12 +515,14 @@ const Os = switch (builtin.os.tag) { return file_id; } - fn markDirtySteps(w: *Watch, gpa: Allocator, dir: *Directory) !bool { + fn markDirtySteps(w: *Watch, dir: *Directory) !bool { + const maker = w.maker; + var any_dirty = false; const bytes_returned = dir.iosb.Information; if (bytes_returned == 0) { std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); - markAllFilesDirty(w, gpa); + markAllFilesDirty(w); try dir.startListening(w); return true; } @@ -530,9 +532,9 @@ const Os = switch (builtin.os.tag) { const notify: *windows.FILE.NOTIFY.INFORMATION = @ptrCast(@alignCast(&dir.buffer[offset])); const file_name = file_name_buf[0..std.unicode.wtf16LeToWtf8(&file_name_buf, notify.fileName())]; if (dir.reaction_set.getPtr(".")) |glob_set| - any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); + any_dirty = markStepSetDirty(maker, glob_set, any_dirty); if (dir.reaction_set.getPtr(file_name)) |step_set| - any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + any_dirty = markStepSetDirty(maker, step_set, any_dirty); if (notify.NextEntryOffset == 0) break; @@ -619,14 +621,17 @@ const Os = switch (builtin.os.tag) { w.dir_count = w.dir_table.count(); } - fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { + fn wait(w: *Watch, timeout: Timeout) !WaitResult { + const maker = w.maker; + const io = maker.graph.io; + for (0..2) |attempt| { while (w.os.ready_dirs.popFirst()) |ready_node| { const dir: *Directory = @fieldParentPtr("ready_node", ready_node); assert(dir.state == .ready); dir.state = .idle; switch (dir.iosb.u.Status) { - .SUCCESS => return if (try markDirtySteps(w, gpa, dir)) .dirty else .clean, + .SUCCESS => return if (try markDirtySteps(w, dir)) .dirty else .clean, .PENDING => unreachable, .CANCELLED => {}, else => |status| return windows.unexpectedStatus(status), @@ -810,25 +815,25 @@ const Os = switch (builtin.os.tag) { w.dir_count = w.dir_table.count(); } - fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - _ = io; + fn wait(w: *Watch, timeout: Timeout) !WaitResult { + const maker = w.maker; var timespec_buffer: posix.timespec = undefined; var event_buffer: [100]posix.Kevent = undefined; var n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, timeout.toTimespec(×pec_buffer)); if (n == 0) return .timeout; const reaction_sets = w.os.handles.items(.rs); - var any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], false); + var any_dirty = markDirtySteps(maker, reaction_sets, event_buffer[0..n], false); timespec_buffer = .{ .sec = 0, .nsec = 0 }; while (n == event_buffer.len) { n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, ×pec_buffer); if (n == 0) break; - any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], any_dirty); + any_dirty = markDirtySteps(maker, reaction_sets, event_buffer[0..n], any_dirty); } return if (any_dirty) .dirty else .clean; } fn markDirtySteps( - gpa: Allocator, + maker: *Maker, reaction_sets: []ReactionSet, events: []const std.c.Kevent, start_any_dirty: bool, @@ -840,13 +845,13 @@ const Os = switch (builtin.os.tag) { // If we knew the basename of the changed file, here we would // mark only the step set dirty, and possibly the glob set: //if (reaction_set.getPtr(".")) |glob_set| - // any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); + // any_dirty = markStepSetDirty(maker, glob_set, any_dirty); //if (reaction_set.getPtr(file_name)) |step_set| - // any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + // any_dirty = markStepSetDirty(maker, step_set, any_dirty); // However we don't know the file name so just mark all the // sets dirty for this directory. for (reaction_set.values()) |*step_set| { - any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + any_dirty = markStepSetDirty(maker, step_set, any_dirty); } } return any_dirty; @@ -878,8 +883,8 @@ const Os = switch (builtin.os.tag) { else => void, }; -pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { - return Os.init(cwd_path, configuration, make_steps); +pub fn init(maker: *Maker) !Watch { + return Os.init(maker); } pub const Match = struct { @@ -904,7 +909,9 @@ pub const Match = struct { }; }; -fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { +fn markAllFilesDirty(w: *Watch) void { + const maker = w.maker; + for (switch (builtin.os.tag) { .windows => w.os.handle_table.keys(), else => w.os.handle_table.values(), @@ -915,18 +922,18 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { }; for (reaction_set.values()) |step_set| { for (step_set.keys()) |step_index| { - const step = &w.make_steps[@intFromEnum(step_index)]; - _ = step.invalidateResult(gpa); + const step = maker.stepByIndex(step_index); + _ = maker.invalidateResult(step); } } } } -fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool { +fn markStepSetDirty(maker: *Maker, step_set: *StepSet, any_dirty: bool) bool { var this_any_dirty = false; for (step_set.keys()) |step_index| { - const step = &make_steps[@intFromEnum(step_index)]; - if (step.invalidateResult(gpa)) this_any_dirty = true; + const step = maker.stepByIndex(step_index); + if (maker.invalidateResult(step)) this_any_dirty = true; } return any_dirty or this_any_dirty; } @@ -971,6 +978,6 @@ pub const WaitResult = enum { clean, }; -pub fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - return Os.wait(w, gpa, io, timeout); +pub fn wait(w: *Watch, timeout: Timeout) !WaitResult { + return Os.wait(w, timeout); } diff --git a/lib/compiler/Maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig index fd3806e8b2b925f6badad2830ddd8d780abab2b4..1ea44e0ba0a47a2f2c607ca7f8d4c42bdba08c99 100644 --- a/lib/compiler/Maker/WebServer.zig +++ b/lib/compiler/Maker/WebServer.zig @@ -14,16 +14,14 @@ const log = std.log.scoped(.web_server); const mem = std.mem; const net = std.Io.net; +const Maker = @import("../Maker.zig"); const Fuzz = @import("Fuzz.zig"); const Graph = @import("Graph.zig"); const Step = @import("Step.zig"); -gpa: Allocator, -graph: *const Graph, -all_steps: []const Configuration.Step.Index, +maker: *Maker, listen_address: net.IpAddress, root_prog_node: std.Progress.Node, -watch: bool, tcp_server: ?net.Server, serve_task: ?Io.Future(Io.Cancelable!void), @@ -65,19 +63,16 @@ pub const base_clock: Io.Clock = .awake; /// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`. pub fn notifyUpdate(ws: *WebServer) void { + const io = ws.maker.graph.io; _ = ws.update_id.rmw(.Add, 1, .release); - ws.graph.io.futexWake(u32, &ws.update_id.raw, 16); + io.futexWake(u32, &ws.update_id.raw, 16); } pub const Options = struct { - gpa: Allocator, - graph: *const Graph, - all_steps: []const Configuration.Step.Index, + maker: *Maker, root_prog_node: std.Progress.Node, - watch: bool, listen_address: net.IpAddress, base_timestamp: Io.Clock.Timestamp, - configuration: *const Configuration, }; pub fn init(opts: Options) WebServer { // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` @@ -85,10 +80,13 @@ pub fn init(opts: Options) WebServer { comptime assert(!builtin.single_threaded); assert(opts.base_timestamp.clock == base_clock); - const all_steps = opts.all_steps; - const c = opts.configuration; + const maker = opts.maker; + const all_steps = maker.step_stack.keys(); + const c = &maker.scanned_config.configuration; + const gpa = maker.gpa; + const graph = maker.graph; - const step_names_trailing = opts.gpa.alloc(u8, len: { + const step_names_trailing = gpa.alloc(u8, len: { var name_bytes: usize = 0; for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len; break :len name_bytes + all_steps.len * 4; @@ -105,25 +103,22 @@ pub fn init(opts: Options) WebServer { assert(idx == step_names_trailing.len); } - const step_status_bits = opts.gpa.alloc( + const step_status_bits = gpa.alloc( u8, std.math.divCeil(usize, all_steps.len, 4) catch unreachable, ) catch @panic("out of memory"); @memset(step_status_bits, 0); - const time_reports_len: usize = if (opts.graph.time_report) all_steps.len else 0; - const time_report_msgs = opts.gpa.alloc([]u8, time_reports_len) catch @panic("out of memory"); - const time_report_update_times = opts.gpa.alloc(i64, time_reports_len) catch @panic("out of memory"); + const time_reports_len: usize = if (graph.time_report) all_steps.len else 0; + const time_report_msgs = gpa.alloc([]u8, time_reports_len) catch @panic("out of memory"); + const time_report_update_times = gpa.alloc(i64, time_reports_len) catch @panic("out of memory"); @memset(time_report_msgs, &.{}); @memset(time_report_update_times, std.math.minInt(i64)); return .{ - .gpa = opts.gpa, - .graph = opts.graph, - .all_steps = all_steps, + .maker = maker, .listen_address = opts.listen_address, .root_prog_node = opts.root_prog_node, - .watch = opts.watch, .tcp_server = null, .serve_task = null, @@ -148,8 +143,9 @@ pub fn init(opts: Options) WebServer { }; } pub fn deinit(ws: *WebServer) void { - const gpa = ws.gpa; - const io = ws.graph.io; + const maker = ws.maker; + const gpa = maker.gpa; + const io = maker.graph.io; gpa.free(ws.step_names_trailing); gpa.free(ws.step_status_bits); @@ -170,7 +166,8 @@ pub fn deinit(ws: *WebServer) void { pub fn start(ws: *WebServer) error{AlreadyReported}!void { assert(ws.tcp_server == null); assert(ws.serve_task == null); - const io = ws.graph.io; + const maker = ws.maker; + const io = maker.graph.io; ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| { log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err }); @@ -189,9 +186,12 @@ pub fn start(ws: *WebServer) error{AlreadyReported}!void { } } fn serve(ws: *WebServer) Io.Cancelable!void { - const io = ws.graph.io; + const maker = ws.maker; + const io = maker.graph.io; + var group: Io.Group = .init; defer group.cancel(io); + while (true) { var stream = ws.tcp_server.?.accept(io) catch |err| switch (err) { error.Canceled => |e| return e, @@ -223,8 +223,10 @@ pub fn updateStepStatus( step_index: Configuration.Step.Index, new_status: abi.StepUpdate.Status, ) void { + const maker = ws.maker; + const all_steps = maker.step_stack.keys(); // TODO don't do linear search, especially in a hot loop like this - const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == step_index) break @intCast(i); } else unreachable; const ptr = &ws.step_status_bits[step_idx / 4]; @@ -238,13 +240,16 @@ pub fn updateStepStatus( pub fn finishBuild(ws: *WebServer, opts: struct { fuzz: bool, }) void { + const maker = ws.maker; + const all_steps = maker.step_stack.keys(); + if (opts.fuzz) { switch (builtin.os.tag) { // Current implementation depends on two things that need to be ported to Windows: // * Memory-mapping to share data between the fuzzer and build runner. // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving // many addresses to source locations). - .windows => std.process.fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), + .windows => std.process.fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), else => {}, } if (@bitSizeOf(usize) != 64) { @@ -260,28 +265,26 @@ pub fn finishBuild(ws: *WebServer, opts: struct { ws.build_status.store(.fuzz_init, .monotonic); ws.notifyUpdate(); - ws.fuzz = Fuzz.init( - ws.gpa, - ws.graph.io, - ws.all_steps, - ws.root_prog_node, - .{ .forever = .{ .ws = ws } }, - ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)}); + ws.fuzz = Fuzz.init(maker, all_steps, ws.root_prog_node, .{ .forever = .{ .ws = ws } }) catch |err| + std.process.fatal("failed to start fuzzer: {t}", .{err}); ws.fuzz.?.start(); } - ws.build_status.store(if (ws.watch) .watching else .idle, .monotonic); + ws.build_status.store(if (maker.watch) .watching else .idle, .monotonic); ws.notifyUpdate(); } -pub fn now(s: *const WebServer) i64 { - const io = s.graph.io; +pub fn now(ws: *const WebServer) i64 { + const maker = ws.maker; + const io = maker.graph.io; const ts = base_clock.now(io); - return @intCast(s.base_timestamp.durationTo(ts).toNanoseconds()); + return @intCast(ws.base_timestamp.durationTo(ts).toNanoseconds()); } fn accept(ws: *WebServer, stream: net.Stream) void { - const io = ws.graph.io; + const maker = ws.maker; + const io = maker.graph.io; + defer { // `net.Stream.close` wants to helpfully overwrite `stream` with // `undefined`, but it cannot do so since it is an immutable parameter. @@ -326,12 +329,16 @@ fn accept(ws: *WebServer, stream: net.Stream) void { } fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { - const io = ws.graph.io; + const maker = ws.maker; + const gpa = maker.gpa; + const graph = maker.graph; + const io = graph.io; + const all_steps = maker.step_stack.keys(); var prev_build_status = ws.build_status.load(.monotonic); - const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len); - defer ws.gpa.free(prev_step_status_bits); + const prev_step_status_bits = try gpa.alloc(u8, ws.step_status_bits.len); + defer gpa.free(prev_step_status_bits); for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| { copy.* = @atomicLoad(u8, shared, .monotonic); } @@ -343,10 +350,10 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { const hello_header: abi.Hello = .{ .status = prev_build_status, .flags = .{ - .time_report = ws.graph.time_report, + .time_report = graph.time_report, }, .timestamp = ws.now(), - .steps_len = @intCast(ws.all_steps.len), + .steps_len = @intCast(all_steps.len), }; var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits }; try sock.writeMessageVec(&bufs, .binary); @@ -369,8 +376,8 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { if (update_time <= prev_time) continue; // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so // that we don't hold up the build system on the client accepting this packet. - const owned_msg = try ws.gpa.dupe(u8, msg); - defer ws.gpa.free(owned_msg); + const owned_msg = try gpa.dupe(u8, msg); + defer gpa.free(owned_msg); // Temporarily unlock, then re-lock after the message is sent. ws.time_report_mutex.unlock(io); defer ws.time_report_mutex.lockUncancelable(io); @@ -427,7 +434,8 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { } } fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void { - const io = ws.graph.io; + const maker = ws.maker; + const io = maker.graph.io; while (true) { const msg = sock.readSmallMessage() catch return; @@ -485,8 +493,11 @@ fn serveLibFile( sub_path: []const u8, content_type: []const u8, ) !void { + const maker = ws.maker; + const graph = maker.graph; + return serveFile(ws, request, .{ - .root_dir = ws.graph.zig_lib_directory, + .root_dir = graph.zig_lib_directory, .sub_path = sub_path, }, content_type); } @@ -495,7 +506,9 @@ fn serveClientWasm( req: *http.Server.Request, optimize_mode: std.builtin.OptimizeMode, ) !void { - var arena_state: std.heap.ArenaAllocator = .init(ws.gpa); + const gpa = ws.maker.gpa; + + var arena_state: std.heap.ArenaAllocator = .init(gpa); defer arena_state.deinit(); const arena = arena_state.allocator(); @@ -510,8 +523,10 @@ pub fn serveFile( path: Cache.Path, content_type: []const u8, ) !void { - const gpa = ws.gpa; - const io = ws.graph.io; + const maker = ws.maker; + const gpa = ws.maker.gpa; + const io = maker.graph.io; + // The desired API is actually sendfile, which will require enhancing http.Server. // We load the file with every request so that the user can make changes to the file // and refresh the HTML page without restarting this server. @@ -528,7 +543,8 @@ pub fn serveFile( }); } pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void { - const graph = ws.graph; + const maker = ws.maker; + const graph = maker.graph; const io = graph.io; var send_buffer: [0x4000]u8 = undefined; @@ -576,8 +592,9 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim const arch_os_abi = "wasm32-freestanding"; const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; - const gpa = ws.gpa; - const graph = ws.graph; + const maker = ws.maker; + const gpa = maker.gpa; + const graph = maker.graph; const io = graph.io; const main_src_path: Cache.Path = .{ @@ -697,7 +714,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim if (code != 0) { log.err( "the following command exited with error code {d}:\n{s}", - .{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ code, try std.zig.allocPrintCmd(arena, .inherit, null, argv.items) }, ); return error.WasmCompilationFailed; } @@ -705,21 +722,21 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim .signal => |sig| { log.err( "the following command terminated with signal {t}:\n{s}", - .{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ sig, try std.zig.allocPrintCmd(arena, .inherit, null, argv.items) }, ); return error.WasmCompilationFailed; }, .stopped => |sig| { log.err( "the following command stopped unexpectedly with signal {t}:\n{s}", - .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ sig, try std.zig.allocPrintCmd(arena, .inherit, null, argv.items) }, ); return error.WasmCompilationFailed; }, .unknown => { log.err( "the following command terminated unexpectedly:\n{s}", - .{try Step.allocPrintCmd(arena, .inherit, null, argv.items)}, + .{try std.zig.allocPrintCmd(arena, .inherit, null, argv.items)}, ); return error.WasmCompilationFailed; }, @@ -729,14 +746,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim try result_error_bundle.renderToStderr(io, .{}, .auto); log.err("the following command failed with {d} compilation errors:\n{s}", .{ result_error_bundle.errorMessageCount(), - try Step.allocPrintCmd(arena, .inherit, null, argv.items), + try std.zig.allocPrintCmd(arena, .inherit, null, argv.items), }); return error.WasmCompilationFailed; } const base_path = result orelse { log.err("child process failed to report result\n{s}", .{ - try Step.allocPrintCmd(arena, .inherit, null, argv.items), + try std.zig.allocPrintCmd(arena, .inherit, null, argv.items), }); return error.WasmCompilationFailed; }; @@ -773,11 +790,13 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { /// The trailing data of `abi.time_report.CompileResult`, except the step name. trailing: []const u8, }) void { - const gpa = ws.gpa; - const io = ws.graph.io; + const maker = ws.maker; + const gpa = maker.gpa; + const io = maker.graph.io; + const all_steps = maker.step_stack.keys(); // TODO don't do linear search - const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == opts.compile_step) break @intCast(i); } else unreachable; @@ -815,11 +834,13 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { } pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void { - const gpa = ws.gpa; - const io = ws.graph.io; + const maker = ws.maker; + const gpa = maker.gpa; + const io = maker.graph.io; + const all_steps = maker.step_stack.keys(); // TODO don't do linear search - const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == step_index) break @intCast(i); } else unreachable; @@ -852,11 +873,13 @@ pub fn updateTimeReportRunTest( tests: *const Step.Run.CachedTestMetadata, ns_per_test: []const u64, ) void { - const gpa = ws.gpa; - const io = ws.graph.io; + const maker = ws.maker; + const gpa = maker.gpa; + const io = maker.graph.io; + const all_steps = maker.step_stack.keys(); // TODO don't do linear search - const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == run_step_index) break @intCast(i); } else unreachable; @@ -910,7 +933,7 @@ const RunnerRequest = union(enum) { rebuild, }; pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { - const io = ws.graph.io; + const io = ws.maker.graph.io; ws.runner_request_mutex.lock(io) catch return; defer ws.runner_request_mutex.unlock(io); if (ws.runner_request) |req| { @@ -921,7 +944,7 @@ pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { return null; } pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest { - const io = ws.graph.io; + const io = ws.maker.graph.io; try ws.runner_request_mutex.lock(io); defer ws.runner_request_mutex.unlock(io); while (true) { diff --git a/lib/std/Build.zig b/lib/std/Build.zig index a6668c5c1daa5fde7ea2c52ff4433023e74c60eb..e7e523e0c85a9e71e450608228e9a829a87ff503 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -45,7 +45,6 @@ install_prefix: []const u8, /// Path to the directory containing build.zig. build_root: Cache.Directory, cache_root: Cache.Directory, -pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, debug_log_scopes: []const []const u8 = &.{}, debug_compile_errors: bool = false, debug_incremental: bool = false, @@ -176,18 +175,6 @@ pub const RunError = error{ ExecNotSupported, } || std.process.SpawnError; -pub const PkgConfigError = error{ - PkgConfigCrashed, - PkgConfigFailed, - PkgConfigNotInstalled, - PkgConfigInvalidOutput, -}; - -pub const PkgConfigPkg = struct { - name: []const u8, - desc: []const u8, -}; - const UserInputOptionsMap = StringHashMap(UserInputOption); const AvailableOptionsMap = StringHashMap(AvailableOption); diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 8b580f3a5da9a11903540524f7b98a445b2d9573..46b9cd0a52beab8c20c94802fae244a4c906e78c 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -8,13 +8,9 @@ const fs = std.fs; const assert = std.debug.assert; const panic = std.debug.panic; const StringHashMap = std.StringHashMap; -const Sha256 = std.crypto.hash.sha2.Sha256; const Allocator = std.mem.Allocator; const Step = std.Build.Step; const LazyPath = std.Build.LazyPath; -const PkgConfigPkg = std.Build.PkgConfigPkg; -const PkgConfigError = std.Build.PkgConfigError; -const RunError = std.Build.RunError; const Module = std.Build.Module; const InstallDir = std.Build.InstallDir; const GeneratedFile = std.Build.GeneratedFile; @@ -777,42 +773,6 @@ pub fn setExecCmd(compile: *Compile, args: []const ?[]const u8) void { compile.exec_cmd_args = duped_args; } -const CliNamedModules = struct { - modules: std.AutoArrayHashMapUnmanaged(*Module, void), - names: std.StringArrayHashMapUnmanaged(void), - - /// Traverse the whole dependency graph and give every module a unique - /// name, ideally one named after what it's called somewhere in the graph. - /// It will help here to have both a mapping from module to name and a set - /// of all the currently-used names. - fn init(arena: Allocator, root_module: *Module) Allocator.Error!CliNamedModules { - var compile: CliNamedModules = .{ - .modules = .{}, - .names = .{}, - }; - const graph = root_module.getGraph(); - { - assert(graph.modules[0] == root_module); - try compile.modules.put(arena, root_module, {}); - try compile.names.put(arena, "root", {}); - } - for (graph.modules[1..], graph.names[1..]) |mod, orig_name| { - var name = orig_name; - var n: usize = 0; - while (true) { - const gop = try compile.names.getOrPut(arena, name); - if (!gop.found_existing) { - try compile.modules.putNoClobber(arena, mod, {}); - break; - } - name = try std.fmt.allocPrint(arena, "{s}{d}", .{ orig_name, n }); - n += 1; - } - } - return compile; - } -}; - fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking_step: ?*Step) ![]const u8 { const step = &compile.step; const b = step.owner; diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 69061bdb136f607c9186951e9ad61b00b51571bc..dffc69c70e597f2ea75efd2a70b7f218d6f6be21 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -436,23 +436,23 @@ pub const Step = extern struct { }; pub const Tag = enum(u5) { - top_level, + check_file, + check_object, compile, + config_header, + fail, + fmt, install_artifact, - install_file, install_dir, + install_file, + objcopy, + options, remove_dir, - fail, - fmt, + run, + top_level, translate_c, - write_file, update_source_files, - run, - check_file, - check_object, - config_header, - objcopy, - options, + write_file, }; pub const TopLevel = struct { @@ -808,6 +808,10 @@ pub const Step = extern struct { _: u23 = 0, }; }; + + pub fn flags(s: *const Step, c: *const Configuration) Flags { + return @bitCast(c.extra[s.extra_index]); + } }; pub const MaxRss = enum(u32) { -- 2.54.0 From 959103c3fd544abe06f32b279a6ebd1a3cd1f61b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 13:50:56 -0800 Subject: [PATCH 015/179] Maker.Step.Compile: progress towards lowering zig args --- lib/compiler/Maker.zig | 89 ++--- lib/compiler/Maker/Graph.zig | 20 ++ lib/compiler/Maker/Step.zig | 94 +----- lib/compiler/Maker/Step/Compile.zig | 493 +++++++++++++--------------- lib/compiler/Maker/Step/Run.zig | 4 +- lib/compiler/configure_runner.zig | 25 +- lib/std/Build.zig | 184 +---------- lib/std/Build/Step/Compile.zig | 4 + lib/std/zig.zig | 73 ++++ 9 files changed, 410 insertions(+), 576 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 8101449cb4ea5f1d2442ab1bdcf0c5c36dbbbeb3..48da4098138a92ef289196699ac49d8fc75d9324 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -124,7 +124,6 @@ pub fn main(init: process.Init.Minimal) !void { graph.cache.hash.addBytes(builtin.zig_version_string); var step_names: std.ArrayList([]const u8) = .empty; - var debug_log_scopes: std.ArrayList([]const u8) = .empty; var help_menu = false; var steps_menu = false; var print_configuration = false; @@ -143,10 +142,6 @@ pub fn main(init: process.Init.Minimal) !void { var fuzz: ?Fuzz.Mode = null; var debounce_interval_ms: u16 = 50; var webui_listen: ?Io.net.IpAddress = null; - var verbose = false; - var sysroot: ?[]const u8 = null; - var search_prefixes: std.ArrayList([]const u8) = .empty; - var libc_file: ?[]const u8 = null; var debug_pkg_config: bool = false; // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, // this will be the directory $glibc-build-dir/install/glibcs @@ -159,7 +154,6 @@ pub fn main(init: process.Init.Minimal) !void { var enable_wasmtime = false; var enable_darling = false; var enable_rosetta = false; - var reference_trace: ?u32 = null; var run_args: ?[]const []const u8 = null; if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { @@ -182,8 +176,6 @@ pub fn main(init: process.Init.Minimal) !void { steps_menu = true; } else if (mem.eql(u8, arg, "--print-configuration")) { print_configuration = true; - } else if (mem.eql(u8, arg, "--verbose")) { - verbose = true; } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { override_install_prefix = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { @@ -193,11 +185,12 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--prefix-include-dir")) { override_include_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--sysroot")) { - sysroot = nextArgOrFatal(args, &arg_idx); + graph.sysroot = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--maxrss")) { + // TODO refactor and reuse the fuzz number parsing here const max_rss_text = nextArgOrFatal(args, &arg_idx); max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| - fatal("invalid byte size: '{s}': {t}", .{ max_rss_text, err }); + fatal("invalid byte size {q}: {t}", .{ max_rss_text, err }); } else if (mem.eql(u8, arg, "--skip-oom-steps")) { skip_oom_steps = true; } else if (mem.eql(u8, arg, "--test-timeout")) { @@ -217,7 +210,7 @@ pub fn main(init: process.Init.Minimal) !void { }; const timeout_str = nextArgOrFatal(args, &arg_idx); const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( - "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)", + "invalid timeout {q}: expected unit (ns, us, ms, s, m, h)", .{timeout_str}, ); const num_str = timeout_str[0 .. num_end_idx + 1]; @@ -227,57 +220,63 @@ pub fn main(init: process.Init.Minimal) !void { break @floatFromInt(unit_and_factor[1]); } } else fatal( - "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)", + "invalid timeout {q}: invalid unit {q} (expected ns, us, ms, s, m, h)", .{ timeout_str, unit_str }, ); const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal( - "invalid timeout '{s}': invalid number '{s}' ({t})", + "invalid timeout {q}: invalid number {q} ({t})", .{ timeout_str, num_str, err }, ); test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); } else if (mem.eql(u8, arg, "--search-prefix")) { - try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); + try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); } else if (mem.eql(u8, arg, "--libc")) { - libc_file = nextArgOrFatal(args, &arg_idx); + graph.libc_file = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--color")) { const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); + fatalWithHint("expected [auto|on|off] after {q}", .{arg}); color = std.meta.stringToEnum(Color, next_arg) orelse { - fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ + fatalWithHint("expected [auto|on|off] after {q}, found {q}", .{ arg, next_arg, }); }; } else if (mem.eql(u8, arg, "--error-style")) { const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected style after '{s}'", .{arg}); + fatalWithHint("expected style after {q}", .{arg}); error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { - fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--multiline-errors")) { const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected style after '{s}'", .{arg}); + fatalWithHint("expected style after {q}", .{arg}); multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { - fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--summary")) { const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg}); + fatalWithHint("expected [all|new|failures|line|none] after {q}", .{arg}); summary = std.meta.stringToEnum(Summary, next_arg) orelse { - fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{ + fatalWithHint("expected [all|new|failures|line|none] after {q}, found {q}", .{ arg, next_arg, }); }; } else if (mem.eql(u8, arg, "--seed")) { const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u32 after '{s}'", .{arg}); + fatalWithHint("expected u32 after {q}", .{arg}); graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {t}", .{ next_arg, err }); + fatal("unable to parse seed {q} as unsigned 32-bit integer: {t}", .{ next_arg, err }); }; + } else if (mem.eql(u8, arg, "--build-id")) { + graph.build_id = .fast; + } else if (mem.cutPrefix(u8, arg, "--build-id=")) |style| { + graph.build_id = std.zig.BuildId.parse(style) catch |err| + fatal("unable to parse --build-id style {q}: {t}", .{ style, err }); } else if (mem.eql(u8, arg, "--debounce")) { + // TODO refactor and reuse the timeout parsing code also here const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u16 after '{s}'", .{arg}); + fatalWithHint("expected u16 after {q}", .{arg}); debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { - fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{ + fatal("unable to parse debounce interval {q} as unsigned 16-bit integer: {t}", .{ next_arg, err, }); }; @@ -287,11 +286,15 @@ pub fn main(init: process.Init.Minimal) !void { const addr_str = arg["--webui=".len..]; if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { - fatal("invalid web UI address '{s}': {t}", .{ addr_str, err }); + fatal("invalid web UI address {q}: {t}", .{ addr_str, err }); }; } else if (mem.eql(u8, arg, "--debug-log")) { const next_arg = nextArgOrFatal(args, &arg_idx); - try debug_log_scopes.append(arena, next_arg); + try graph.debug_log_scopes.append(arena, next_arg); + } else if (mem.eql(u8, arg, "--debug-compile-errors")) { + graph.debug_compile_errors = true; + } else if (mem.eql(u8, arg, "--debug-incremental")) { + graph.debug_incremental = true; } else if (mem.eql(u8, arg, "--debug-pkg-config")) { debug_pkg_config = true; } else if (mem.eql(u8, arg, "--debug-rt")) { @@ -302,6 +305,14 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--verbose")) { + graph.verbose = true; + } else if (mem.eql(u8, arg, "--verbose-air")) { + graph.verbose_air = true; + } else if (mem.eql(u8, arg, "--verbose-cc")) { + graph.verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { + graph.verbose_llvm_ir = true; } else if (mem.eql(u8, arg, "--watch")) { watch = true; } else if (mem.eql(u8, arg, "--time-report")) { @@ -373,18 +384,15 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { graph.allow_so_scripts = false; } else if (mem.eql(u8, arg, "-freference-trace")) { - reference_trace = 256; - } else if (mem.startsWith(u8, arg, "-freference-trace=")) { - const num = arg["-freference-trace=".len..]; - reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - std.debug.print("unable to parse reference_trace count '{s}': {t}", .{ num, err }); - process.exit(1); - }; + graph.reference_trace = 256; + } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { + graph.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| + fatal("unable to parse reference_trace count {q}: {t}", .{ num, err }); } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - reference_trace = null; + graph.reference_trace = null; } else if (mem.cutPrefix(u8, arg, "-j")) |text| { const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| - fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); + fatal("unable to parse jobs count {q}: {t}", .{ text, err }); if (n < 1) fatal("number of jobs must be at least 1", .{}); threaded.setAsyncLimit(.limited(n)); graph.max_jobs = n; @@ -392,7 +400,7 @@ pub fn main(init: process.Init.Minimal) !void { run_args = argsRest(args, arg_idx); break; } else { - fatalWithHint("unrecognized argument: '{s}'", .{arg}); + fatalWithHint("unrecognized argument: {s}", .{arg}); } } else { try step_names.append(arena, arg); @@ -1848,8 +1856,7 @@ const ScannedConfig = struct { \\ --debug-rt Debug compiler runtime libraries \\ --verbose-link Enable compiler debug output for linking \\ --verbose-air Enable compiler debug output for Zig AIR - \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR - \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC + \\ --verbose-llvm-ir Enable compiler debug output for LLVM IR \\ --verbose-cimport Enable compiler debug output for C imports \\ --verbose-cc Enable compiler debug output for C compilation \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index ba9cad0ee17b713cd1731aff04e22a132d94afdd..c62e49a9a5d1840e3a3c3be5b5ea530ed2953ee5 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -23,3 +23,23 @@ time_report: bool = false, /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also /// respects the '--color' flag. stderr_mode: ?Io.Terminal.Mode = null, +reference_trace: ?u32 = null, +debug_log_scopes: std.ArrayList([]const u8) = .empty, +debug_compile_errors: bool = false, +debug_incremental: bool = false, +verbose: bool = false, +verbose_air: bool = false, +verbose_cc: bool = false, +verbose_link: bool = false, +verbose_llvm_cpu_features: bool = false, +verbose_llvm_ir: bool = false, +libc_file: ?[]const u8 = null, +/// What does this do? Nobody bothered to document it, and I think it's a +/// smelly option. So unless somebody deletes these passive aggressive comments +/// and replaces them with actual documentation, I'm going to delete this +/// option from the build system in a future release. In other words, this is +/// deprecated due to lack of test coverage, lack of documentation, and a hunch +/// that it's a bad option that should be avoided. +sysroot: ?[]const u8 = null, +search_prefixes: std.ArrayList([]const u8) = .empty, +build_id: ?std.zig.BuildId = null, diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index c43b502bc5e17b1dace1236ac5f5a8426fa72217..8eda9e1e042ac03a84d66735d775dacb234bded0 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -298,7 +298,7 @@ pub fn captureChildProcess( // If an error occurs, it's happened in this command: assert(s.result_failed_command == null); - s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); + s.result_failed_command = try std.zig.allocPrintCmd(gpa, .inherit, null, argv); try handleChildProcUnsupported(s); try handleVerbose(s, .inherit, argv); @@ -354,15 +354,15 @@ pub fn evalZigProcess( argv: []const []const u8, prog_node: std.Progress.Node, watch: bool, - web_server: ?*WebServer, - gpa: Allocator, + maker: *Maker, ) !?Cache.Path { + const gpa = maker.gpa; const b = s.owner; const io = b.graph.io; // If an error occurs, it's happened in this command: assert(s.result_failed_command == null); - s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); + s.result_failed_command = try std.zig.allocPrintCmd(gpa, .inherit, null, argv); if (s.getZigProcess()) |zp| update: { assert(watch); @@ -374,7 +374,7 @@ pub fn evalZigProcess( zp.deinit(io); gpa.destroy(zp); } else zp.saveState(prog_node); - const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { + const result = zigProcessUpdate(s, zp, watch, maker) catch |err| switch (err) { error.BrokenPipe, error.EndOfStream => |reason| { std.log.info("{s} restart required: {t}", .{ argv[0], reason }); // Process restart required. @@ -431,7 +431,7 @@ pub fn evalZigProcess( const result = result: { defer if (watch) zp.saveState(prog_node); - break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); + break :result try zigProcessUpdate(s, zp, watch, maker); }; if (!watch) { @@ -485,7 +485,8 @@ pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); } -fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebServer, gpa: Allocator) !?Path { +fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, maker: *Maker) !?Path { + const gpa = maker.gpa; const b = s.owner; const arena = b.allocator; const io = b.graph.io; @@ -586,7 +587,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebSer } } }, - .time_report => if (web_server) |ws| { + .time_report => if (maker.web_server) |ws| { const TimeReport = std.zig.Server.Message.TimeReport; const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); ws.updateTimeReportCompile(.{ @@ -641,11 +642,11 @@ pub fn handleVerbose( opt_env: ?*const std.process.Environ.Map, argv: []const []const u8, ) error{OutOfMemory}!void { - if (!s.verbose) return; const graph = s.graph; + if (!graph.verbose) return; // Intention of verbose is to print all sub-process command lines to // stderr before spawning them. - const text = try allocPrintCmd(arena, cwd, if (opt_env) |env| .{ + const text = try std.zig.allocPrintCmd(arena, cwd, if (opt_env) |env| .{ .child = env, .parent = &graph.environ_map, } else null, argv); @@ -835,79 +836,6 @@ fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !v try gop.value_ptr.append(gpa, basename); } -pub fn allocPrintCmd( - gpa: Allocator, - cwd: std.process.Child.Cwd, - opt_env: ?struct { - child: *const std.process.Environ.Map, - parent: *const std.process.Environ.Map, - }, - argv: []const []const u8, -) Allocator.Error![]u8 { - const shell = struct { - fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { - for (string) |c| { - if (switch (c) { - else => true, - '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, - '=' => is_argv0, - }) break; - } else return writer.writeAll(string); - - try writer.writeByte('"'); - for (string) |c| { - if (switch (c) { - std.ascii.control_code.nul => break, - '!', '"', '$', '\\', '`' => true, - else => !std.ascii.isPrint(c), - }) try writer.writeByte('\\'); - switch (c) { - std.ascii.control_code.nul => unreachable, - std.ascii.control_code.bel => try writer.writeByte('a'), - std.ascii.control_code.bs => try writer.writeByte('b'), - std.ascii.control_code.ht => try writer.writeByte('t'), - std.ascii.control_code.lf => try writer.writeByte('n'), - std.ascii.control_code.vt => try writer.writeByte('v'), - std.ascii.control_code.ff => try writer.writeByte('f'), - std.ascii.control_code.cr => try writer.writeByte('r'), - std.ascii.control_code.esc => try writer.writeByte('E'), - ' '...'~' => try writer.writeByte(c), - else => try writer.print("{o:0>3}", .{c}), - } - } - try writer.writeByte('"'); - } - }; - - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - const writer = &aw.writer; - switch (cwd) { - .inherit => {}, - .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, - .dir => @panic("TODO"), - } - if (opt_env) |env| { - var it = env.child.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; - if (env.parent.get(key)) |process_value| { - if (std.mem.eql(u8, value, process_value)) continue; - } - writer.print("{s}=", .{key}) catch return error.OutOfMemory; - shell.escape(writer, value, false) catch return error.OutOfMemory; - writer.writeByte(' ') catch return error.OutOfMemory; - } - } - shell.escape(writer, argv[0], true) catch return error.OutOfMemory; - for (argv[1..]) |arg| { - writer.writeByte(' ') catch return error.OutOfMemory; - shell.escape(writer, arg, false) catch return error.OutOfMemory; - } - return aw.toOwnedSlice(); -} - fn oomWrap(s: *Step, result: error{OutOfMemory}!void) void { result catch { s.result_oom = true; diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 5ef2ec9dacc021e2287af597107de953b88cb846..1deb6d128934e21ab3e2210c011a101b135afac2 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -10,6 +10,7 @@ const Io = std.Io; const Sha256 = std.crypto.hash.sha2.Sha256; const assert = std.debug.assert; const mem = std.mem; +const allocPrint = std.fmt.allocPrint; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); @@ -17,6 +18,8 @@ const Maker = @import("../../Maker.zig"); /// Populated during the make phase when there is a long-lived compiler process. /// Managed by the build runner, not user build script. zig_process: ?*Step.ZigProcess = null, +/// Persisted to reuse memory on subsequent make. +zig_args: std.ArrayList([]const u8) = .empty, pub fn make( compile: *Compile, @@ -24,14 +27,15 @@ pub fn make( maker: *Maker, progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { + const graph = maker.graph; + const step = maker.stepByIndex(step_index); + compile.zig_args.clearRetainingCapacity(); if (true) @panic("TODO implement compile.make()"); - const graph = maker.graph; - const step = maker.stepByIndex(step_index); - const zig_args = try getZigArgs(compile, maker, false); + try lowerZigArgs(compile, step_index, maker, &compile.zig_args, false); const process_arena = graph.arena; // TODO don't leak into the process_arena const maybe_output_dir = step.evalZigProcess( - zig_args, + compile.zig_args.items, progress_node, (graph.incremental == true) and (maker.watch or maker.web_server != null), maker, @@ -47,7 +51,7 @@ pub fn make( // Update generated files if (maybe_output_dir) |output_dir| { if (compile.emit_directory) |lp| { - lp.path = try std.fmt.allocPrint(process_arena, "{f}", .{output_dir}); + lp.path = try allocPrint(process_arena, "{f}", .{output_dir}); } // zig fmt: off @@ -70,23 +74,26 @@ pub fn make( { try doAtomicSymLinks( step, - compile.getEmittedBin().getPath2(step.owner, step), + compile.getEmittedBin().getPath2(step), compile.major_only_filename.?, compile.name_only_filename.?, ); } } -fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { - const step = &compile.step; - const b = step.owner; +fn lowerZigArgs( + compile: *Compile, + step_index: Configuration.Step.Index, + maker: *Maker, + zig_args: *std.ArrayList([]const u8), + fuzz: bool, +) Allocator.Error!void { + const step = maker.stepByIndex(step_index); const graph = maker.graph; const arena = graph.arena; // TODO don't leak into the process arena + const gpa = maker.gpa; - var zig_args = std.array_list.Managed([]const u8).init(arena); - defer zig_args.deinit(); - - try zig_args.append(graph.zig_exe); + try zig_args.append(gpa, graph.zig_exe); const cmd = switch (compile.kind) { .lib => "build-lib", @@ -95,10 +102,10 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { .@"test" => "test", .test_obj => "test-obj", }; - try zig_args.append(cmd); + try zig_args.append(gpa, cmd); - if (b.reference_trace) |some| { - try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); + if (graph.reference_trace) |some| { + try zig_args.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{some})); } try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse graph.allow_so_scripts); @@ -107,33 +114,31 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { try addFlag(&zig_args, "new-linker", compile.use_new_linker); if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { - try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); + try zig_args.append(gpa, try allocPrint(arena, "-ofmt={t}", .{ofmt})); } switch (compile.entry) { .default => {}, - .disabled => try zig_args.append("-fno-entry"), - .enabled => try zig_args.append("-fentry"), + .disabled => try zig_args.append(gpa, "-fno-entry"), + .enabled => try zig_args.append(gpa, "-fentry"), .symbol_name => |entry_name| { - try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); + try zig_args.append(gpa, try allocPrint(arena, "-fentry={s}", .{entry_name})); }, } { for (compile.force_undefined_symbols.keys()) |symbol_name| { - try zig_args.append("--force_undefined"); - try zig_args.append(symbol_name.*); + try zig_args.append(gpa, "--force_undefined"); + try zig_args.append(gpa, symbol_name.*); } } if (compile.stack_size) |stack_size| { - try zig_args.append("--stack"); - try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); + try zig_args.append(gpa, "--stack"); + try zig_args.append(gpa, try allocPrint(arena, "{}", .{stack_size})); } - if (fuzz) { - try zig_args.append("-ffuzz"); - } + try addBool(gpa, zig_args, fuzz, "-ffuzz"); { // Stores system libraries that have already been seen for at least one @@ -183,14 +188,14 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { switch (link_object) { .static_path => |static_path| { if (my_responsibility) { - try zig_args.append(static_path.getPath2(mod.owner, step)); + try zig_args.append(gpa, static_path.getPath2(step)); total_linker_objects += 1; } }, .system_lib => |system_lib| { const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); if (system_lib_gop.found_existing) { - try zig_args.appendSlice(system_lib_gop.value_ptr.*); + try zig_args.appendSlice(gpa, system_lib_gop.value_ptr.*); continue; } else { system_lib_gop.value_ptr.* = &.{}; @@ -205,16 +210,16 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { { switch (system_lib.search_strategy) { .no_fallback => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_dylibs_only"), - .static => try zig_args.append("-search_static_only"), + .dynamic => try zig_args.append(gpa, "-search_dylibs_only"), + .static => try zig_args.append(gpa, "-search_static_only"), }, .paths_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_paths_first"), - .static => try zig_args.append("-search_paths_first_static"), + .dynamic => try zig_args.append(gpa, "-search_paths_first"), + .static => try zig_args.append(gpa, "-search_paths_first_static"), }, .mode_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_dylibs_first"), - .static => try zig_args.append("-search_static_first"), + .dynamic => try zig_args.append(gpa, "-search_dylibs_first"), + .static => try zig_args.append(gpa, "-search_static_first"), }, } prev_search_strategy = system_lib.search_strategy; @@ -227,11 +232,11 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { break :prefix "-l"; }; switch (system_lib.use_pkg_config) { - .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), + .no => try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ prefix, system_lib.name })), .yes, .force => { if (compile.runPkgConfig(maker, system_lib.name)) |result| { - try zig_args.appendSlice(result.cflags); - try zig_args.appendSlice(result.libs); + try zig_args.appendSlice(gpa, result.cflags); + try zig_args.appendSlice(gpa, result.libs); try seen_system_libs.put(arena, system_lib.name, result.cflags); } else |err| switch (err) { error.PkgConfigInvalidOutput, @@ -243,7 +248,7 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { .yes => { // pkg-config failed, so fall back to linking the library // by name directly. - try zig_args.append(b.fmt("{s}{s}", .{ + try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ prefix, system_lib.name, })); @@ -267,7 +272,7 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { const included_in_lib_or_obj = !my_responsibility and (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); if (!already_linked and !included_in_lib_or_obj) { - try zig_args.append(other.getEmittedBin().getPath2(b, step)); + try zig_args.append(gpa, other.getEmittedBin().getPath2(step)); total_linker_objects += 1; } }, @@ -288,15 +293,15 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { else try other.getGeneratedFilePath("generated_bin", &compile.step); - try zig_args.append(full_path_lib); + try zig_args.append(gpa, full_path_lib); total_linker_objects += 1; if (other.linkage == .dynamic and compile.rootModuleTarget().os.tag != .windows) { if (Dir.path.dirname(full_path_lib)) |dirname| { - try zig_args.append("-rpath"); - try zig_args.append(dirname); + try zig_args.append(gpa, "-rpath"); + try zig_args.append(gpa, dirname); } } }, @@ -306,11 +311,11 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { if (!my_responsibility) break :l; if (prev_has_cflags) { - try zig_args.append("-cflags"); - try zig_args.append("--"); + try zig_args.append(gpa, "-cflags"); + try zig_args.append(gpa, "--"); prev_has_cflags = false; } - try zig_args.append(asm_file.getPath2(mod.owner, step)); + try zig_args.append(gpa, asm_file.getPath2(mod.owner, step)); total_linker_objects += 1; }, @@ -318,24 +323,24 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { if (!my_responsibility) break :l; if (prev_has_cflags or c_source_file.flags.len != 0) { - try zig_args.append("-cflags"); + try zig_args.append(gpa, "-cflags"); for (c_source_file.flags) |arg| { - try zig_args.append(arg); + try zig_args.append(gpa, arg); } - try zig_args.append("--"); + try zig_args.append(gpa, "--"); } prev_has_cflags = (c_source_file.flags.len != 0); if (c_source_file.language) |lang| { - try zig_args.append("-x"); - try zig_args.append(lang.internalIdentifier()); + try zig_args.append(gpa, "-x"); + try zig_args.append(gpa, lang.internalIdentifier()); } - try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); + try zig_args.append(gpa, c_source_file.file.getPath2(mod.owner, step)); if (c_source_file.language != null) { - try zig_args.append("-x"); - try zig_args.append("none"); + try zig_args.append(gpa, "-x"); + try zig_args.append(gpa, "none"); } total_linker_objects += 1; }, @@ -344,27 +349,27 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { if (!my_responsibility) break :l; if (prev_has_cflags or c_source_files.flags.len != 0) { - try zig_args.append("-cflags"); + try zig_args.append(gpa, "-cflags"); for (c_source_files.flags) |arg| { - try zig_args.append(arg); + try zig_args.append(gpa, arg); } - try zig_args.append("--"); + try zig_args.append(gpa, "--"); } prev_has_cflags = (c_source_files.flags.len != 0); if (c_source_files.language) |lang| { - try zig_args.append("-x"); - try zig_args.append(lang.internalIdentifier()); + try zig_args.append(gpa, "-x"); + try zig_args.append(gpa, lang.internalIdentifier()); } const root_path = c_source_files.root.getPath2(mod.owner, step); for (c_source_files.files) |file| { - try zig_args.append(b.pathJoin(&.{ root_path, file })); + try zig_args.append(gpa, try Dir.path.join(arena, &.{ root_path, file })); } if (c_source_files.language != null) { - try zig_args.append("-x"); - try zig_args.append("none"); + try zig_args.append(gpa, "-x"); + try zig_args.append(gpa, "none"); } total_linker_objects += c_source_files.files.len; @@ -375,23 +380,23 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { if (prev_has_rcflags) { - try zig_args.append("-rcflags"); - try zig_args.append("--"); + try zig_args.append(gpa, "-rcflags"); + try zig_args.append(gpa, "--"); prev_has_rcflags = false; } } else { - try zig_args.append("-rcflags"); + try zig_args.append(gpa, "-rcflags"); for (rc_source_file.flags) |arg| { - try zig_args.append(arg); + try zig_args.append(gpa, arg); } for (rc_source_file.include_paths) |include_path| { - try zig_args.append("/I"); - try zig_args.append(include_path.getPath2(mod.owner, step)); + try zig_args.append(gpa, "/I"); + try zig_args.append(gpa, include_path.getPath2(mod.owner, step)); } - try zig_args.append("--"); + try zig_args.append(gpa, "--"); prev_has_rcflags = true; } - try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); + try zig_args.append(gpa, rc_source_file.file.getPath2(mod.owner, step)); total_linker_objects += 1; }, } @@ -414,7 +419,7 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { if (std.mem.eql(u8, import_cli_name, name)) { zig_args.appendAssumeCapacity(import_cli_name); } else { - zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); + zig_args.appendAssumeCapacity(try allocPrint(arena, "{s}={s}", .{ name, import_cli_name })); } } @@ -427,9 +432,9 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { // files must have a module parent. if (mod.root_source_file) |lp| { const src = lp.getPath2(mod.owner, step); - try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); + try zig_args.append(gpa, try allocPrint(arena, "-M{s}={s}", .{ module_cli_name, src })); } else if (moduleNeedsCliArg(mod)) { - try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); + try zig_args.append(gpa, try allocPrint(arena, "-M{s}", .{module_cli_name})); } } } @@ -441,275 +446,248 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { for (frameworks.keys(), frameworks.values()) |name, info| { if (info.needed) { - try zig_args.append("-needed_framework"); + try zig_args.append(gpa, "-needed_framework"); } else if (info.weak) { - try zig_args.append("-weak_framework"); + try zig_args.append(gpa, "-weak_framework"); } else { - try zig_args.append("-framework"); + try zig_args.append(gpa, "-framework"); } - try zig_args.append(name); + try zig_args.append(gpa, name); } if (compile.is_linking_libcpp) { - try zig_args.append("-lc++"); + try zig_args.append(gpa, "-lc++"); } if (compile.is_linking_libc) { - try zig_args.append("-lc"); + try zig_args.append(gpa, "-lc"); } } if (compile.win32_manifest) |manifest_file| { - try zig_args.append(manifest_file.getPath2(b, step)); + try zig_args.append(gpa, manifest_file.getPath2(step)); } if (compile.win32_module_definition) |module_file| { - try zig_args.append(module_file.getPath2(b, step)); + try zig_args.append(gpa, module_file.getPath2(step)); } if (compile.image_base) |image_base| { - try zig_args.append("--image-base"); - try zig_args.append(b.fmt("0x{x}", .{image_base})); + try zig_args.appendSlice(gpa, &.{ + "--image-base", try allocPrint(arena, "0x{x}", .{image_base}), + }); } for (compile.filters) |filter| { - try zig_args.append("--test-filter"); - try zig_args.append(filter); + try zig_args.appendSlice(gpa, &.{ "--test-filter", filter }); } if (compile.test_runner) |test_runner| { - try zig_args.append("--test-runner"); - try zig_args.append(test_runner.path.getPath2(b, step)); + try zig_args.appendSlice(gpa, &.{ "--test-runner", test_runner.path.getPath2(step) }); } - for (b.debug_log_scopes) |log_scope| { - try zig_args.append("--debug-log"); - try zig_args.append(log_scope); + for (graph.debug_log_scopes) |log_scope| { + try zig_args.appendSlice(gpa, &.{ "--debug-log", log_scope }); } - if (b.debug_compile_errors) { - try zig_args.append("--debug-compile-errors"); - } - - if (b.debug_incremental) { - try zig_args.append("--debug-incremental"); - } - - if (b.verbose_air) try zig_args.append("--verbose-air"); - if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); - if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); - if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); - if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); - if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); - if (graph.time_report) try zig_args.append("--time-report"); + try addBool(gpa, zig_args, graph.debug_compile_errors, "--debug-compile-errors"); + try addBool(gpa, zig_args, graph.debug_incremental, "--debug-incremental"); + try addBool(gpa, zig_args, graph.verbose_air, "--verbose-air"); + try addBool(gpa, zig_args, graph.verbose_llvm_ir, "--verbose-llvm-ir"); + try addBool(gpa, zig_args, graph.verbose_link or compile.verbose_link, "--verbose-link"); + try addBool(gpa, zig_args, graph.verbose_cc or compile.verbose_cc, "--verbose-cc"); + try addBool(gpa, zig_args, graph.verbose_llvm_cpu_features, "--verbose-llvm-cpu-features"); + try addBool(gpa, zig_args, graph.time_report, "--time-report"); - if (compile.generated_asm != null) try zig_args.append("-femit-asm"); - if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); - if (compile.generated_docs != null) try zig_args.append("-femit-docs"); - if (compile.generated_implib != null) try zig_args.append("-femit-implib"); - if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); - if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); - if (compile.generated_h != null) try zig_args.append("-femit-h"); + if (compile.generated_asm != null) try zig_args.append(gpa, "-femit-asm"); + if (compile.generated_bin == null) try zig_args.append(gpa, "-fno-emit-bin"); + if (compile.generated_docs != null) try zig_args.append(gpa, "-femit-docs"); + if (compile.generated_implib != null) try zig_args.append(gpa, "-femit-implib"); + if (compile.generated_llvm_bc != null) try zig_args.append(gpa, "-femit-llvm-bc"); + if (compile.generated_llvm_ir != null) try zig_args.append(gpa, "-femit-llvm-ir"); + if (compile.generated_h != null) try zig_args.append(gpa, "-femit-h"); try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); switch (compile.compress_debug_sections) { .none => {}, - .zlib => try zig_args.append("--compress-debug-sections=zlib"), - .zstd => try zig_args.append("--compress-debug-sections=zstd"), + .zlib => try zig_args.append(gpa, "--compress-debug-sections=zlib"), + .zstd => try zig_args.append(gpa, "--compress-debug-sections=zstd"), } if (compile.link_eh_frame_hdr) { - try zig_args.append("--eh-frame-hdr"); + try zig_args.append(gpa, "--eh-frame-hdr"); } if (compile.link_emit_relocs) { - try zig_args.append("--emit-relocs"); + try zig_args.append(gpa, "--emit-relocs"); } if (compile.link_function_sections) { - try zig_args.append("-ffunction-sections"); + try zig_args.append(gpa, "-ffunction-sections"); } if (compile.link_data_sections) { - try zig_args.append("-fdata-sections"); + try zig_args.append(gpa, "-fdata-sections"); } if (compile.link_gc_sections) |x| { - try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); + try zig_args.append(gpa, if (x) "--gc-sections" else "--no-gc-sections"); } if (!compile.linker_dynamicbase) { - try zig_args.append("--no-dynamicbase"); + try zig_args.append(gpa, "--no-dynamicbase"); } if (compile.linker_allow_shlib_undefined) |x| { - try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); - } - if (compile.link_z_notext) { - try zig_args.append("-z"); - try zig_args.append("notext"); - } - if (!compile.link_z_relro) { - try zig_args.append("-z"); - try zig_args.append("norelro"); - } - if (compile.link_z_lazy) { - try zig_args.append("-z"); - try zig_args.append("lazy"); - } - if (compile.link_z_common_page_size) |size| { - try zig_args.append("-z"); - try zig_args.append(b.fmt("common-page-size={d}", .{size})); - } - if (compile.link_z_max_page_size) |size| { - try zig_args.append("-z"); - try zig_args.append(b.fmt("max-page-size={d}", .{size})); - } - if (compile.link_z_defs) { - try zig_args.append("-z"); - try zig_args.append("defs"); - } + try zig_args.append(gpa, if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); + } + if (compile.link_z_notext) try zig_args.appendSlice(gpa, &.{ "-z", "notext" }); + if (!compile.link_z_relro) try zig_args.appendSlice(gpa, &.{ "-z", "norelro" }); + if (compile.link_z_lazy) try zig_args.appendSlice(gpa, &.{ "-z", "lazy" }); + if (compile.link_z_common_page_size) |size| try zig_args.appendSlice(gpa, &.{ + "-z", + try allocPrint(arena, "common-page-size={d}", .{size}), + }); + if (compile.link_z_max_page_size) |size| try zig_args.appendSlice(gpa, &.{ + "-z", + try allocPrint(arena, "max-page-size={d}", .{size}), + }); + if (compile.link_z_defs) try zig_args.appendSlice(gpa, &.{ "-z", "defs" }); if (compile.libc_file) |libc_file| { - try zig_args.append("--libc"); - try zig_args.append(libc_file.getPath2(b, step)); - } else if (b.libc_file) |libc_file| { - try zig_args.append("--libc"); - try zig_args.append(libc_file); + try zig_args.appendSlice(gpa, &.{ "--libc", libc_file.getPath2(step) }); + } else if (graph.libc_file) |libc_file| { + try zig_args.appendSlice(gpa, &.{ "--libc", libc_file }); } - try zig_args.append("--cache-dir"); - try zig_args.append(b.cache_root.path orelse "."); + try zig_args.append(gpa, "--cache-dir"); + try zig_args.append(gpa, graph.cache_root.path orelse "."); - try zig_args.append("--global-cache-dir"); - try zig_args.append(graph.global_cache_root.path orelse "."); + try zig_args.append(gpa, "--global-cache-dir"); + try zig_args.append(gpa, graph.global_cache_root.path orelse "."); if (graph.debug_compiler_runtime_libs) |mode| - try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); + try zig_args.append(gpa, try allocPrint(arena, "--debug-rt={t}", .{mode})); - try zig_args.append("--name"); - try zig_args.append(compile.name); + try zig_args.append(gpa, "--name"); + try zig_args.append(gpa, compile.name); if (compile.linkage) |some| switch (some) { - .dynamic => try zig_args.append("-dynamic"), - .static => try zig_args.append("-static"), + .dynamic => try zig_args.append(gpa, "-dynamic"), + .static => try zig_args.append(gpa, "-static"), }; if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { if (compile.version) |version| { - try zig_args.append("--version"); - try zig_args.append(b.fmt("{f}", .{version})); + try zig_args.append(gpa, "--version"); + try zig_args.append(gpa, try allocPrint(arena, "{f}", .{version})); } if (compile.rootModuleTarget().os.tag.isDarwin()) { - const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ + const install_name = compile.install_name orelse try allocPrint(arena, "@rpath/{s}{s}{s}", .{ compile.rootModuleTarget().libPrefix(), compile.name, compile.rootModuleTarget().dynamicLibSuffix(), }); - try zig_args.append("-install_name"); - try zig_args.append(install_name); + try zig_args.append(gpa, "-install_name"); + try zig_args.append(gpa, install_name); } } if (compile.entitlements) |entitlements| { - try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); + try zig_args.appendSlice(gpa, &[_][]const u8{ "--entitlements", entitlements }); } if (compile.pagezero_size) |pagezero_size| { - const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); - try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); + const size = try allocPrint(arena, "{x}", .{pagezero_size}); + try zig_args.appendSlice(gpa, &[_][]const u8{ "-pagezero_size", size }); } if (compile.headerpad_size) |headerpad_size| { - const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); - try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); + const size = try allocPrint(arena, "{x}", .{headerpad_size}); + try zig_args.appendSlice(gpa, &[_][]const u8{ "-headerpad", size }); } if (compile.headerpad_max_install_names) { - try zig_args.append("-headerpad_max_install_names"); + try zig_args.append(gpa, "-headerpad_max_install_names"); } if (compile.dead_strip_dylibs) { - try zig_args.append("-dead_strip_dylibs"); + try zig_args.append(gpa, "-dead_strip_dylibs"); } if (compile.force_load_objc) { - try zig_args.append("-ObjC"); + try zig_args.append(gpa, "-ObjC"); } if (compile.discard_local_symbols) { - try zig_args.append("--discard-all"); + try zig_args.append(gpa, "--discard-all"); } try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); if (compile.rdynamic) { - try zig_args.append("-rdynamic"); + try zig_args.append(gpa, "-rdynamic"); } if (compile.import_memory) { - try zig_args.append("--import-memory"); + try zig_args.append(gpa, "--import-memory"); } if (compile.export_memory) { - try zig_args.append("--export-memory"); + try zig_args.append(gpa, "--export-memory"); } if (compile.import_symbols) { - try zig_args.append("--import-symbols"); + try zig_args.append(gpa, "--import-symbols"); } if (compile.import_table) { - try zig_args.append("--import-table"); + try zig_args.append(gpa, "--import-table"); } if (compile.export_table) { - try zig_args.append("--export-table"); + try zig_args.append(gpa, "--export-table"); } if (compile.initial_memory) |initial_memory| { - try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); + try zig_args.append(gpa, try allocPrint(arena, "--initial-memory={d}", .{initial_memory})); } if (compile.max_memory) |max_memory| { - try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); + try zig_args.append(gpa, try allocPrint(arena, "--max-memory={d}", .{max_memory})); } if (compile.shared_memory) { - try zig_args.append("--shared-memory"); + try zig_args.append(gpa, "--shared-memory"); } if (compile.global_base) |global_base| { - try zig_args.append(b.fmt("--global-base={d}", .{global_base})); + try zig_args.append(gpa, try allocPrint(arena, "--global-base={d}", .{global_base})); } if (compile.wasi_exec_model) |model| { - try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); + try zig_args.append(gpa, try allocPrint(arena, "-mexec-model={t}", .{model})); } if (compile.linker_script) |linker_script| { - try zig_args.append("--script"); - try zig_args.append(linker_script.getPath2(b, step)); + try zig_args.append(gpa, "--script"); + try zig_args.append(gpa, linker_script.getPath2(step)); } if (compile.version_script) |version_script| { - try zig_args.append("--version-script"); - try zig_args.append(version_script.getPath2(b, step)); + try zig_args.append(gpa, "--version-script"); + try zig_args.append(gpa, version_script.getPath2(step)); } if (compile.linker_allow_undefined_version) |x| { - try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); + try zig_args.append(gpa, if (x) "--undefined-version" else "--no-undefined-version"); } if (compile.linker_enable_new_dtags) |enabled| { - try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); + try zig_args.append(gpa, if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); } if (compile.kind == .@"test") { if (compile.exec_cmd_args) |exec_cmd_args| { for (exec_cmd_args) |cmd_arg| { if (cmd_arg) |arg| { - try zig_args.append("--test-cmd"); - try zig_args.append(arg); + try zig_args.append(gpa, "--test-cmd"); + try zig_args.append(gpa, arg); } else { - try zig_args.append("--test-cmd-bin"); + try zig_args.append(gpa, "--test-cmd-bin"); } } } } - if (b.sysroot) |sysroot| { - try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); - } + if (graph.sysroot) |sysroot| try zig_args.appendSlice(gpa, &.{ "--sysroot", sysroot }); // -I and -L arguments that appear after the last --mod argument apply to all modules. const cwd: Io.Dir = .cwd(); const io = graph.io; - for (b.search_prefixes.items) |search_prefix| { + for (graph.search_prefixes.items) |search_prefix| { var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { - return step.fail("unable to open prefix directory '{s}': {s}", .{ - search_prefix, @errorName(err), - }); + return step.fail("unable to open prefix directory '{s}': {t}", .{ search_prefix, err }); }; defer prefix_dir.close(io); @@ -718,58 +696,53 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { // CLI parsing code, when the linker sees an -L directory that does not exist. if (prefix_dir.access(io, "lib", .{})) |_| { - try zig_args.appendSlice(&.{ - "-L", b.pathJoin(&.{ search_prefix, "lib" }), + try zig_args.appendSlice(gpa, &.{ + "-L", try Dir.path.join(arena, &.{ search_prefix, "lib" }), }); } else |err| switch (err) { error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ - search_prefix, @errorName(e), - }), + else => |e| return step.fail("unable to access '{s}/lib' directory: {t}", .{ search_prefix, e }), } if (prefix_dir.access(io, "include", .{})) |_| { - try zig_args.appendSlice(&.{ - "-I", b.pathJoin(&.{ search_prefix, "include" }), + try zig_args.appendSlice(gpa, &.{ + "-I", try Dir.path.join(arena, &.{ search_prefix, "include" }), }); } else |err| switch (err) { error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ - search_prefix, @errorName(e), - }), + else => |e| return step.fail("unable to access '{s}/include' directory: {t}", .{ search_prefix, e }), } } if (compile.rc_includes != .any) { - try zig_args.append("-rcincludes"); - try zig_args.append(@tagName(compile.rc_includes)); + try zig_args.appendSlice(gpa, &.{ "-rcincludes", @tagName(compile.rc_includes) }); } try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); - if (compile.build_id orelse b.build_id) |build_id| { - try zig_args.append(switch (build_id) { - .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), - .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), + if (compile.build_id orelse graph.build_id) |build_id| { + try zig_args.append(gpa, switch (build_id) { + .hexstring => |hs| try allocPrint(arena, "--build-id=0x{x}", .{hs.toSlice()}), + .none, .fast, .uuid, .sha1, .md5 => try allocPrint(arena, "--build-id={t}", .{build_id}), }); } const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| - dir.getPath2(b, step) + dir.getPath2(step) else if (graph.zig_lib_directory.path) |_| - b.fmt("{f}", .{graph.zig_lib_directory}) + try allocPrint(arena, "{f}", .{graph.zig_lib_directory}) else null; if (opt_zig_lib_dir) |zig_lib_dir| { - try zig_args.append("--zig-lib-dir"); - try zig_args.append(zig_lib_dir); + try zig_args.append(gpa, "--zig-lib-dir"); + try zig_args.append(gpa, zig_lib_dir); } try addFlag(&zig_args, "PIE", compile.pie); if (compile.lto) |lto| { - try zig_args.append(switch (lto) { + try zig_args.append(gpa, switch (lto) { .full => "-flto=full", .thin => "-flto=thin", .none => "-fno-lto", @@ -779,21 +752,20 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); if (compile.subsystem) |subsystem| { - try zig_args.append("--subsystem"); - try zig_args.append(@tagName(subsystem)); + try zig_args.appendSlice(gpa, &.{ "--subsystem", @tagName(subsystem) }); } if (compile.mingw_unicode_entry_point) { - try zig_args.append("-municode"); + try zig_args.append(gpa, "-municode"); } - if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ - "--error-limit", b.fmt("{d}", .{err_limit}), + if (compile.error_limit) |err_limit| try zig_args.appendSlice(gpa, &.{ + "--error-limit", try allocPrint(arena, "{d}", .{err_limit}), }); try addFlag(&zig_args, "incremental", graph.incremental); - try zig_args.append("--listen=-"); + try zig_args.append(gpa, "--listen=-"); // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and @@ -804,7 +776,7 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { args_length += arg.len + 1; // +1 to account for null terminator } if (args_length >= 30 * 1024) { - try b.cache_root.handle.createDirPath(io, "args"); + try graph.cache_root.handle.createDirPath(io, "args"); const args_to_escape = zig_args.items[2..]; var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); @@ -837,21 +809,21 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); const args_file = "args" ++ Dir.path.sep_str ++ args_hex_hash; - if (b.cache_root.handle.access(io, args_file, .{})) |_| { + if (graph.cache_root.handle.access(io, args_file, .{})) |_| { // The args file is already present from a previous run. } else |err| switch (err) { error.FileNotFound => { - var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ + var af = graph.cache_root.handle.createFileAtomic(io, args_file, .{ .replace = false, .make_path = true, }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ - b.cache_root, args_file, e, + graph.cache_root, args_file, e, }); defer af.deinit(io); af.file.writeStreamingAll(io, args) catch |e| { return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ - b.cache_root, args_file, e, + graph.cache_root, args_file, e, }); }; // Note we can't clean up this file, not even after build @@ -862,7 +834,7 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { // The args file was created by another concurrent build process. }, else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ - b.cache_root, args_file, other_err, + graph.cache_root, args_file, other_err, }), }; }, @@ -871,32 +843,34 @@ fn getZigArgs(compile: *Compile, maker: *Maker, fuzz: bool) ![][]const u8 { const resolved_args_file = try mem.concat(arena, u8, &.{ "@", - try b.cache_root.join(arena, &.{args_file}), + try graph.cache_root.join(arena, &.{args_file}), }); zig_args.shrinkRetainingCapacity(2); - try zig_args.append(resolved_args_file); + try zig_args.append(gpa, resolved_args_file); } return try zig_args.toOwnedSlice(); } -pub fn rebuildInFuzzMode(c: *Compile, maker: *Maker, progress_node: std.Progress.Node) !Path { +pub fn rebuildInFuzzMode(compile: *Compile, maker: *Maker, progress_node: std.Progress.Node) !Path { const gpa = maker.graph.gpa; - c.step.result_error_msgs.clearRetainingCapacity(); - c.step.result_stderr = ""; + compile.step.result_error_msgs.clearRetainingCapacity(); + compile.step.result_stderr = ""; - c.step.result_error_bundle.deinit(gpa); - c.step.result_error_bundle = std.zig.ErrorBundle.empty; + compile.step.result_error_bundle.deinit(gpa); + compile.step.result_error_bundle = std.zig.ErrorBundle.empty; - if (c.step.result_failed_command) |cmd| { + if (compile.step.result_failed_command) |cmd| { gpa.free(cmd); - c.step.result_failed_command = null; + compile.step.result_failed_command = null; } - const zig_args = try getZigArgs(c, maker, true); - const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); + const zig_args = &compile.zig_args; + zig_args.clearRetainingCapacity(); + try lowerZigArgs(compile, maker, zig_args, true); + const maybe_output_bin_path = try compile.step.evalZigProcess(zig_args.items, progress_node, false, maker); return maybe_output_bin_path.?; } @@ -907,24 +881,24 @@ pub fn doAtomicSymLinks( filename_major_only: []const u8, filename_name_only: []const u8, ) !void { - const b = step.owner; const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into process arena const io = graph.io; const out_dir = Dir.path.dirname(output_path) orelse "."; const out_basename = Dir.path.basename(output_path); // sym link for libfoo.so.1 to libfoo.so.1.2.3 - const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); + const major_only_path = try Dir.path.join(arena, &.{ out_dir, filename_major_only }); const cwd: Io.Dir = .cwd(); cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { - return step.fail("unable to symlink {s} -> {s}: {s}", .{ - major_only_path, out_basename, @errorName(err), + return step.fail("unable to symlink {s} -> {s}: {t}", .{ + major_only_path, out_basename, err, }); }; // sym link for libfoo.so to libfoo.so.1 - const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); + const name_only_path = try Dir.path.join(arena, &.{ out_dir, filename_name_only }); cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { - return step.fail("Unable to symlink {s} -> {s}: {s}", .{ - name_only_path, filename_major_only, @errorName(err), + return step.fail("unable to symlink {s} -> {s}: {t}", .{ + name_only_path, filename_major_only, err, }); }; } @@ -983,14 +957,13 @@ fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { } } -fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { +fn addBool(gpa: Allocator, args: *std.ArrayList([]const u8), arg: []const u8, opt: bool) !void { + if (opt) try args.append(gpa, arg); +} + +fn addFlag(gpa: Allocator, args: *std.ArrayList([]const u8), comptime name: []const u8, opt: ?bool) !void { const cond = opt orelse return; - try args.ensureUnusedCapacity(1); - if (cond) { - args.appendAssumeCapacity("-f" ++ name); - } else { - args.appendAssumeCapacity("-fno-" ++ name); - } + try args.append(gpa, if (cond) "-f" ++ name else "-fno-" ++ name); } const PkgConfigResult = struct { @@ -1267,7 +1240,7 @@ const CliNamedModules = struct { try compile.modules.putNoClobber(arena, mod, {}); break; } - name = try std.fmt.allocPrint(arena, "{s}{d}", .{ orig_name, n }); + name = try allocPrint(arena, "{s}{d}", .{ orig_name, n }); n += 1; } } diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 122c394e0d93d6b293545e8677d16d142c9dd27c..b621a3cf50608ce15ccaca0cfdfe19b58f9d227c 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -1564,7 +1564,7 @@ fn runCommand( const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; try step.handleChildProcUnsupported(); - try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); + try Step.handleVerbose(step.owner, cwd, run.environ_map, argv); const allow_skip = switch (run.stdio) { .check, .zig_test => run.skip_foreign_checks, @@ -1701,7 +1701,7 @@ fn runCommand( gpa.free(step.result_failed_command.?); step.result_failed_command = null; - try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); + try Step.handleVerbose(step.owner, cwd, run.environ_map, interp_argv.items); break :term spawnChildAndCollect(run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 54113594f6429340aec1fa6c4b1236ee201393e8..392f0a42c59bb55e73531ff477cd1391aea4ca0b 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -148,39 +148,30 @@ pub fn main(init: process.Init.Minimal) !void { graph.release_mode = .any; } else if (mem.cutPrefix(u8, arg, "--release=")) |text| { graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { - fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ + fatalWithHint("expected [off|any|fast|safe|small] in {q}, found {q}", .{ arg, text, }); }; } else if (mem.eql(u8, arg, "--color")) { const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); + fatalWithHint("expected [auto|on|off] after {q}", .{arg}); color = std.meta.stringToEnum(Color, next_arg) orelse { - fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ + fatalWithHint("expected [auto|on|off] after {q}, found {q}", .{ arg, next_arg, }); }; } else if (mem.eql(u8, arg, "--error-style")) { const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected style after '{s}'", .{arg}); + fatalWithHint("expected style after {q}", .{arg}); error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { - fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--multiline-errors")) { const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected style after '{s}'", .{arg}); + fatalWithHint("expected style after {q}", .{arg}); multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { - fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; - } else if (mem.eql(u8, arg, "--build-id")) { - builder.build_id = .fast; - } else if (mem.cutPrefix(u8, arg, "--build-id=")) |style| { - builder.build_id = std.zig.BuildId.parse(style) catch |err| - fatal("unable to parse --build-id style '{s}': {t}", .{ style, err }); - } else if (mem.eql(u8, arg, "--debug-compile-errors")) { - builder.debug_compile_errors = true; - } else if (mem.eql(u8, arg, "--debug-incremental")) { - builder.debug_incremental = true; } else if (mem.eql(u8, arg, "--system")) { // The usage text shows another argument after this parameter // but it is handled by the parent process. The build runner @@ -189,7 +180,7 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--have-run-args")) { graph.have_run_args = true; } else { - fatalWithHint("unrecognized argument: '{s}'", .{arg}); + fatalWithHint("unrecognized argument: {q}", .{arg}); } } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index e7e523e0c85a9e71e450608228e9a829a87ff503..284a2da346f17edba2100a5a5b93f40986b3c9a9 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -46,8 +46,6 @@ install_prefix: []const u8, build_root: Cache.Directory, cache_root: Cache.Directory, debug_log_scopes: []const []const u8 = &.{}, -debug_compile_errors: bool = false, -debug_incremental: bool = false, /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, /// in particular at `Step` creation. /// Set to 0 to disable stack collection. @@ -75,8 +73,6 @@ pkg_hash: []const u8, /// A mapping from dependency names to package hashes. available_deps: AvailableDeps, -build_id: ?std.zig.BuildId = null, - pub const ReleaseMode = enum { off, any, @@ -227,13 +223,6 @@ pub fn create( .graph = graph, .build_root = build_root, .cache_root = cache_root, - .verbose = false, - .verbose_link = false, - .verbose_cc = false, - .verbose_air = false, - .verbose_llvm_ir = null, - .verbose_llvm_bc = null, - .verbose_llvm_cpu_features = false, .invalid_user_input = false, .allocator = arena, .user_input_options = UserInputOptionsMap.init(arena), @@ -302,22 +291,12 @@ fn createChild( .user_input_options = user_input_options, .available_options_map = AvailableOptionsMap.init(allocator), .available_options_list = std.array_list.Managed(AvailableOption).init(allocator), - .verbose = parent.verbose, - .verbose_link = parent.verbose_link, - .verbose_cc = parent.verbose_cc, - .verbose_air = parent.verbose_air, - .verbose_llvm_ir = parent.verbose_llvm_ir, - .verbose_llvm_bc = parent.verbose_llvm_bc, - .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, .invalid_user_input = false, .default_step = undefined, .top_level_steps = .{}, - .sysroot = parent.sysroot, .build_root = build_root, .cache_root = parent.cache_root, .debug_log_scopes = parent.debug_log_scopes, - .debug_compile_errors = parent.debug_compile_errors, - .debug_incremental = parent.debug_incremental, .enable_darling = parent.enable_darling, .enable_qemu = parent.enable_qemu, .enable_rosetta = parent.enable_rosetta, @@ -1125,7 +1104,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw if (std.zig.BuildId.parse(s)) |build_id| { return build_id; } else |err| { - log.err("unable to parse option '-D{s}': {s}", .{ name, @errorName(err) }); + log.err("unable to parse option '-D{s}': {t}", .{ name, err }); b.markInvalidUserInput(); return null; } @@ -1594,8 +1573,9 @@ pub fn addCheckFile( } pub fn truncateFile(b: *Build, dest_path: []const u8) (Io.Dir.CreateDirError || Io.Dir.StatFileError)!void { - const io = b.graph.io; - if (b.verbose) log.info("truncate {s}", .{dest_path}); + const graph = b.graph; + const io = graph.io; + if (graph.verbose) log.info("truncate {s}", .{dest_path}); const cwd = Io.Dir.cwd(); var src_file = cwd.createFile(io, dest_path, .{}) catch |err| switch (err) { error.FileNotFound => blk: { @@ -1705,9 +1685,13 @@ pub fn runAllowFail( const graph = b.graph; const io = graph.io; + const arena = graph.arena; const max_output_size = 400 * 1024; - try Step.handleVerbose2(b, .inherit, &graph.environ_map, argv); + if (graph.verbose) { + const text = std.zig.allocPrintCmd(arena, .inherit, null, argv); + std.log.scoped(.verbose).info("{s}", .{text}); + } var child = try std.process.spawn(io, .{ .argv = argv, @@ -1718,10 +1702,10 @@ pub fn runAllowFail( }); var stdout_reader = child.stdout.?.readerStreaming(io, &.{}); - const stdout = stdout_reader.interface.allocRemaining(b.allocator, .limited(max_output_size)) catch { + const stdout = stdout_reader.interface.allocRemaining(arena, .limited(max_output_size)) catch { return error.ReadFailure; }; - errdefer b.allocator.free(stdout); + errdefer arena.free(stdout); const term = try child.wait(io); switch (term) { @@ -2089,34 +2073,6 @@ pub fn runBuild(b: *Build, build_zig: anytype) anyerror!void { pub const GeneratedFile = struct { /// The step that generates the file. step: *Step, - /// The path to the generated file. Must be either absolute or relative to the build runner cwd. - /// This value must be set in the `fn make()` of the `step` and must not be `null` afterwards. - path: ?[]const u8 = null, - - /// Deprecated, see `getPath3`. - pub fn getPath(gen: GeneratedFile) []const u8 { - return gen.step.owner.pathFromCwd(gen.path orelse std.debug.panic( - "getPath() was called on a GeneratedFile that wasn't built yet. Is there a missing Step dependency on step '{s}'?", - .{gen.step.name}, - )); - } - - /// Deprecated, see `getPath3`. - pub fn getPath2(gen: GeneratedFile, src_builder: *Build, asking_step: ?*Step) []const u8 { - return getPath3(gen, src_builder, asking_step) catch |err| switch (err) { - error.Canceled => std.process.exit(1), - }; - } - - pub fn getPath3(gen: GeneratedFile, src_builder: *Build, asking_step: ?*Step) Io.Cancelable![]const u8 { - return gen.path orelse { - const graph = gen.step.owner.graph; - const io = graph.io; - const stderr = try io.lockStderr(&.{}, graph.stderr_mode); - dumpBadGetPathHelp(gen.step, stderr.terminal(), src_builder, asking_step) catch {}; - @panic("misconfigured build script"); - }; - } }; // dirnameAllowEmpty is a variant of fs.path.dirname @@ -2290,94 +2246,6 @@ pub const LazyPath = union(enum) { } } - /// Deprecated, see `getPath4`. - pub fn getPath(lazy_path: LazyPath, src_builder: *Build) []const u8 { - return getPath2(lazy_path, src_builder, null); - } - - /// Deprecated, see `getPath4`. - pub fn getPath2(lazy_path: LazyPath, src_builder: *Build, asking_step: ?*Step) []const u8 { - const p = getPath3(lazy_path, src_builder, asking_step); - return src_builder.pathResolve(&.{ p.root_dir.path orelse ".", p.sub_path }); - } - - /// Deprecated, see `getPath4`. - pub fn getPath3(lazy_path: LazyPath, src_builder: *Build, asking_step: ?*Step) Cache.Path { - return getPath4(lazy_path, src_builder, asking_step) catch |err| switch (err) { - error.Canceled => std.process.exit(1), - }; - } - - /// Intended to be used during the make phase only. - /// - /// `asking_step` is only used for debugging purposes; it's the step being - /// run that is asking for the path. - pub fn getPath4(lazy_path: LazyPath, src_builder: *Build, asking_step: ?*Step) Io.Cancelable!Cache.Path { - switch (lazy_path) { - .src_path => |sp| return .{ - .root_dir = sp.owner.build_root, - .sub_path = sp.sub_path, - }, - .cwd_relative => |sub_path| return .{ - .root_dir = Cache.Directory.cwd(), - .sub_path = sub_path, - }, - .generated => |gen| { - // TODO make gen.file.path not be absolute and use that as the - // basis for not traversing up too many directories. - - const graph = src_builder.graph; - - var file_path: Cache.Path = .{ - .root_dir = Cache.Directory.cwd(), - .sub_path = gen.file.path orelse { - const io = graph.io; - const stderr = try io.lockStderr(&.{}, graph.stderr_mode); - dumpBadGetPathHelp(gen.file.step, stderr.terminal(), src_builder, asking_step) catch {}; - io.unlockStderr(); - @panic("misconfigured build script"); - }, - }; - - if (gen.up > 0) { - const cache_root_path = src_builder.cache_root.path orelse - (src_builder.cache_root.join(src_builder.allocator, &.{"."}) catch @panic("OOM")); - - for (0..gen.up) |_| { - if (mem.eql(u8, file_path.sub_path, cache_root_path)) { - // If we hit the cache root and there's still more to go, - // the script attempted to go too far. - dumpBadDirnameHelp(gen.file.step, asking_step, - \\dirname() attempted to traverse outside the cache root. - \\This is not allowed. - \\ - , .{}) catch {}; - @panic("misconfigured build script"); - } - - // path is absolute. - // dirname will return null only if we're at root. - // Typically, we'll stop well before that at the cache root. - file_path.sub_path = fs.path.dirname(file_path.sub_path) orelse { - dumpBadDirnameHelp(gen.file.step, asking_step, - \\dirname() reached root. - \\No more directories left to go up. - \\ - , .{}) catch {}; - @panic("misconfigured build script"); - }; - } - } - - return file_path.join(src_builder.allocator, gen.sub_path) catch @panic("OOM"); - }, - .dependency => |dep| return .{ - .root_dir = dep.dependency.builder.build_root, - .sub_path = dep.sub_path, - }, - } - } - pub fn basename(lazy_path: LazyPath, src_builder: *Build, asking_step: ?*Step) []const u8 { return fs.path.basename(switch (lazy_path) { .src_path => |sp| sp.sub_path, @@ -2451,36 +2319,6 @@ fn dumpBadDirnameHelp( stderr.setColor(.reset) catch {}; } -/// In this function the stderr mutex has already been locked. -pub fn dumpBadGetPathHelp(s: *Step, t: Io.Terminal, src_builder: *Build, asking_step: ?*Step) anyerror!void { - const w = t.writer; - try w.print( - \\getPath() was called on a GeneratedFile that wasn't built yet. - \\ source package path: {s} - \\ Is there a missing Step dependency on step '{s}'? - \\ - , .{ - src_builder.build_root.path orelse ".", - s.name, - }); - - t.setColor(.red) catch {}; - try w.writeAll(" The step was created by this stack trace:\n"); - t.setColor(.reset) catch {}; - - s.dump(t); - if (asking_step) |as| { - t.setColor(.red) catch {}; - try w.print(" The step '{s}' that is missing a dependency on the above step was created by this stack trace:\n", .{as.name}); - t.setColor(.reset) catch {}; - - as.dump(t); - } - t.setColor(.red) catch {}; - try w.writeAll(" Proceeding to panic.\n"); - t.setColor(.reset) catch {}; -} - pub const InstallDir = union(enum) { prefix: void, lib: void, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 46b9cd0a52beab8c20c94802fae244a4c906e78c..ee4068e991091a2fb7572c421220e200c16353b6 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -87,9 +87,13 @@ libc_file: ?LazyPath = null, each_lib_rpath: ?bool = null, /// On ELF targets, this will emit a link section called ".note.gnu.build-id" /// which can be used to coordinate a stripped binary with its debug symbols. +/// /// As an example, the bloaty project refuses to work unless its inputs have /// build ids, in order to prevent accidental mismatches. +/// /// The default is to not include this section because it slows down linking. +/// +/// This option overrides the CLI argument passed to `zig build`. build_id: ?std.zig.BuildId = null, /// Create a .eh_frame_hdr section and a PT_GNU_EH_FRAME segment in the ELF diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 5fdcd67de4c87b4d93f89ffb0e5fe5af99f817d0..63d3e3d0d68e85d3b616aa2a24b8f2116fe4e8d9 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1157,6 +1157,79 @@ pub const ClangCliParam = struct { } }; +pub fn allocPrintCmd( + gpa: Allocator, + cwd: std.process.Child.Cwd, + opt_env: ?struct { + child: *const std.process.Environ.Map, + parent: *const std.process.Environ.Map, + }, + argv: []const []const u8, +) Allocator.Error![]u8 { + const shell = struct { + fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { + for (string) |c| { + if (switch (c) { + else => true, + '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, + '=' => is_argv0, + }) break; + } else return writer.writeAll(string); + + try writer.writeByte('"'); + for (string) |c| { + if (switch (c) { + std.ascii.control_code.nul => break, + '!', '"', '$', '\\', '`' => true, + else => !std.ascii.isPrint(c), + }) try writer.writeByte('\\'); + switch (c) { + std.ascii.control_code.nul => unreachable, + std.ascii.control_code.bel => try writer.writeByte('a'), + std.ascii.control_code.bs => try writer.writeByte('b'), + std.ascii.control_code.ht => try writer.writeByte('t'), + std.ascii.control_code.lf => try writer.writeByte('n'), + std.ascii.control_code.vt => try writer.writeByte('v'), + std.ascii.control_code.ff => try writer.writeByte('f'), + std.ascii.control_code.cr => try writer.writeByte('r'), + std.ascii.control_code.esc => try writer.writeByte('E'), + ' '...'~' => try writer.writeByte(c), + else => try writer.print("{o:0>3}", .{c}), + } + } + try writer.writeByte('"'); + } + }; + + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + const writer = &aw.writer; + switch (cwd) { + .inherit => {}, + .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, + .dir => @panic("TODO"), + } + if (opt_env) |env| { + var it = env.child.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + if (env.parent.get(key)) |process_value| { + if (std.mem.eql(u8, value, process_value)) continue; + } + writer.print("{s}=", .{key}) catch return error.OutOfMemory; + shell.escape(writer, value, false) catch return error.OutOfMemory; + writer.writeByte(' ') catch return error.OutOfMemory; + } + } + shell.escape(writer, argv[0], true) catch return error.OutOfMemory; + for (argv[1..]) |arg| { + writer.writeByte(' ') catch return error.OutOfMemory; + shell.escape(writer, arg, false) catch return error.OutOfMemory; + } + return aw.toOwnedSlice(); +} + test { _ = Ast; _ = AstRlAnnotate; -- 2.54.0 From 174532c78e254449c61d1729f8761b30b6f05eff Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 15:03:43 -0800 Subject: [PATCH 016/179] std.build.Configuration: sketch a data layout idea --- lib/std/zig/Configuration.zig | 69 ++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index dffc69c70e597f2ea75efd2a70b7f218d6f6be21..0e40f0ce7e93a83f6ff179a5db1a4f9b0292dfb6 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -577,39 +577,8 @@ pub const Step = extern struct { }; /// Trailing: - /// * filters_len: u32, // if flag is set - /// * exec_cmd_args_len: u32, // if flag is set - /// * installed_headers_len: u32, // if flag is set - /// * force_undefined_symbols_len: u32, // if flag is set - /// * exacts_len: u32 if expected_compile_errors is exact - /// * filter: String for each filters_len - /// * exec_cmd_arg: String for each exec_cmd_args_len - /// * InstalledHeader for each installed_headers_len - /// * force_undefined_symbol: String for each force_undefined_symbols_len - /// * String for each exacts_len - /// * linker_script: LazyPath if flag is set - /// * version_script: LazyPath if flag is set - /// * zig_lib_dir: LazyPath if flag is set - /// * libc_file: LazyPath if flag is set - /// * test_runner: LazyPath if test_runner_mode is not default - /// * win32_manifest: LazyPath if flag is set - /// * win32_module_definition: LazyPath if flag is set - /// * entitlements: LazyPath if flag is set - /// * version: String if flag is set (semantic version string) - /// * entry: String if entry is symbol - /// * install_name: String if flag is set - /// * String if expected_compile_errors is contains, starts_with, or stderr_contains - /// * initial_memory: u64 if flag is set - /// * max_memory: u64 if flag is set - /// * global_base: u64 if flag is set - /// * image_base: u64 if flag is set - /// * link_z_common_page_size if flag is set - /// * link_z_max_page_size if flag is set - /// * pagezero_size if flag is set - /// * stack_size if flag is set - /// * headerpad_size if flag is set - /// * error_limit if flag is set - /// * Hexstring if build_id is hexstring + /// * exact_match: String, // if expected_compile_errors is contains, starts_with, or stderr_contains + /// * test_runner: LazyPath, // if test_runner_mode is not default pub const Compile = struct { flags: @This().Flags, flags2: Flags2, @@ -619,6 +588,35 @@ pub const Step = extern struct { root_module: Module.Index, root_name: String, + trailing: Trailing(struct { + filters: FlagLengthPrefixedList(.flags, .filters_len, String), + exec_cmd_args: FlagLengthPrefixedList(.flags, .exec_cmd_args_len, u32), + installed_headers: FlagLengthPrefixedList(.flags, .installed_headers_len, InstalledHeader), + force_undefined_symbols: FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), + exacts: EnumConditionalPrefixedList(.flags4, .expected_compile_errors, .exact, String), + linker_script: FlagOptional(.flags4, .linker_script, LazyPath), + version_script: FlagOptional(.flags4, .version_script, LazyPath), + zig_lib_dir: FlagOptional(.flags3, .zig_lib_dir, LazyPath), + libc_file: FlagOptional(.flags4, .libc_file, LazyPath), + win32_manifest: FlagOptional(.flags3, .win32_manifest, LazyPath), + win32_module_definition: FlagOptional(.flags3, .win32_module_definition, LazyPath), + entitlements: FlagOptional(.flags4, .entitlements, LazyPath), + version: FlagOptional(.flags3, .version, String), // semantic version string + entry: EnumOptional(.flags3, .entry, .symbol, String), + install_name: FlagOptional(.flags4, .install_name, String), + initial_memory: FlagOptional(.flags3, .initial_memory, u64), + max_memory: FlagOptional(.flags3, .max_memory, u64), + global_base: FlagOptional(.flags3, .global_base, u64), + image_base: FlagOptional(.flags3, .image_base, u64), + link_z_common_page_size: FlagOptional(.flags4, .link_z_common_page_size, u64), + link_z_max_page_size: FlagOptional(.flags4, .link_z_max_page_size, u64), + pagezero_size: FlagOptional(.flags4, .pagezero_size, u64), + stack_size: FlagOptional(.flags4, .stack_size, u64), + headerpad_size: FlagOptional(.flags4, .headerpad_size, u32), + error_limit: FlagOptional(.flags4, .error_limit, u32), + build_id: EnumOptional(.flags3, .build_id, .hexstring, Hexstring), + }), + pub const ExpectedCompileErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; pub const TestRunnerMode = enum(u2) { default, simple, server }; pub const Entry = enum(u2) { default, disabled, enabled, symbol_name }; @@ -805,7 +803,10 @@ pub const Step = extern struct { error_limit: bool, install_name: bool, entitlements: bool, - _: u23 = 0, + expected_compile_errors: ExpectedCompileErrors, + linker_script: bool, + version_script: bool, + _: u18 = 0, }; }; -- 2.54.0 From 74b018ceb3822e0810ddfe434605bee69aba8b73 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 15:04:01 -0800 Subject: [PATCH 017/179] zig build: make --error-limit globally configurable still overridable by individual Compile steps --- lib/compiler/Maker.zig | 5 +++++ lib/compiler/Maker/Graph.zig | 1 + lib/compiler/Maker/Step/Compile.zig | 2 +- lib/std/Build/Step/Compile.zig | 4 ++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 48da4098138a92ef289196699ac49d8fc75d9324..278cb5340e32f1930e24c0548f7a8bc3007b071a 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -390,6 +390,10 @@ pub fn main(init: process.Init.Minimal) !void { fatal("unable to parse reference_trace count {q}: {t}", .{ num, err }); } else if (mem.eql(u8, arg, "-fno-reference-trace")) { graph.reference_trace = null; + } else if (mem.eql(u8, arg, "--error-limit")) { + const next_arg = nextArgOrFatal(args, &arg_idx); + graph.error_limit = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| + fatal("unable to parse error limit {q}: {t}", .{ next_arg, err }); } else if (mem.cutPrefix(u8, arg, "-j")) |text| { const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| fatal("unable to parse jobs count {q}: {t}", .{ text, err }); @@ -1838,6 +1842,7 @@ const ScannedConfig = struct { \\ -fno-reference-trace Disable reference trace \\ -fallow-so-scripts Allows .so files to be GNU ld scripts \\ -fno-allow-so-scripts (default) .so files must be ELF files + \\ --error-limit [num] Set the maximum amount of distinct error values \\ --build-file [file] Override path to build.zig \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index c62e49a9a5d1840e3a3c3be5b5ea530ed2953ee5..e1b9ef58757c7f31e9e94e547b877e58b22c6b41 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -43,3 +43,4 @@ libc_file: ?[]const u8 = null, sysroot: ?[]const u8 = null, search_prefixes: std.ArrayList([]const u8) = .empty, build_id: ?std.zig.BuildId = null, +error_limit: ?u32 = null, diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 1deb6d128934e21ab3e2210c011a101b135afac2..28638565fad1a807a6f188f57076dfd07810db74 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -759,7 +759,7 @@ fn lowerZigArgs( try zig_args.append(gpa, "-municode"); } - if (compile.error_limit) |err_limit| try zig_args.appendSlice(gpa, &.{ + if (compile.error_limit orelse graph.error_limit) |err_limit| try zig_args.appendSlice(gpa, &.{ "--error-limit", try allocPrint(arena, "{d}", .{err_limit}), }); diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index ee4068e991091a2fb7572c421220e200c16353b6..40d2a4a38abb4f963903efcd004565b7572e9b9e 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -224,8 +224,8 @@ generated_llvm_bc: ?*GeneratedFile, generated_llvm_ir: ?*GeneratedFile, generated_h: ?*GeneratedFile, -/// The maximum number of distinct errors within a compilation step -/// Defaults to `std.math.maxInt(u16)` +/// The maximum number of distinct errors within a compilation step Defaults to +/// `std.math.maxInt(u16)`. Overrides the argument passed to `zig build`. error_limit: ?u32 = null, /// Computed during make(). -- 2.54.0 From 612560d0198ab9a6b471e36770eb3718ccc5b57f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 18:35:53 -0800 Subject: [PATCH 018/179] std.Build.Configure: implement FlagOptional serialization --- lib/compiler/configure_runner.zig | 56 ++++- lib/std/Build/Step/Compile.zig | 11 +- lib/std/zig/Configuration.zig | 373 ++++++++++++++++++++---------- 3 files changed, 308 insertions(+), 132 deletions(-) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 392f0a42c59bb55e73531ff477cd1391aea4ca0b..901b4947135cadd74794bbe15f4e336f08ee9b61 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -239,7 +239,7 @@ const Serialize = struct { return gop.value_ptr.*; } - fn addOptionalLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.OptionalLazyPath { + fn addOptionalLazyPathEnum(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.OptionalLazyPath { const wc = s.wc; return @enumFromInt(switch (lp orelse return .none) { .src_path => |src_path| i: { @@ -274,6 +274,18 @@ const Serialize = struct { }, }); } + + fn addOptionalLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !?Configuration.LazyPath { + return (try addOptionalLazyPathEnum(s, lp)).unwrap(); + } + + fn addOptionalSemVer(s: *Serialize, sem_ver: ?std.SemanticVersion) !?Configuration.String { + return if (sem_ver) |sv| try s.wc.addSemVer(sv) else null; + } + + fn addOptionalString(s: *Serialize, opt_slice: ?[]const u8) !?Configuration.String { + return if (opt_slice) |slice| try s.wc.addString(slice) else null; + } }; fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { @@ -416,9 +428,36 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .error_limit = c.error_limit != null, .install_name = c.install_name != null, .entitlements = c.entitlements != null, + .expect_errors = if (c.expect_errors) |x| switch (x) { + .contains => .contains, + .exact => .exact, + .starts_with => .starts_with, + .stderr_contains => .stderr_contains, + } else .none, + .linker_script = c.linker_script != null, + .version_script = c.version_script != null, }, .root_module = try addModule(&s, c.root_module), .root_name = try wc.addString(c.name), + .linker_script = .{ .value = try s.addOptionalLazyPath(c.linker_script) }, + .version_script = .{ .value = try s.addOptionalLazyPath(c.version_script) }, + .zig_lib_dir = .{ .value = try s.addOptionalLazyPath(c.zig_lib_dir) }, + .libc_file = .{ .value = try s.addOptionalLazyPath(c.libc_file) }, + .win32_manifest = .{ .value = try s.addOptionalLazyPath(c.win32_manifest) }, + .win32_module_definition = .{ .value = try s.addOptionalLazyPath(c.win32_module_definition) }, + .entitlements = .{ .value = try s.addOptionalLazyPath(c.entitlements) }, + .version = .{ .value = try s.addOptionalSemVer(c.version) }, + .install_name = .{ .value = try s.addOptionalString(c.install_name) }, + .initial_memory = .{ .value = c.initial_memory }, + .max_memory = .{ .value = c.max_memory }, + .global_base = .{ .value = c.global_base }, + .image_base = .{ .value = c.image_base }, + .link_z_common_page_size = .{ .value = c.link_z_common_page_size }, + .link_z_max_page_size = .{ .value = c.link_z_max_page_size }, + .pagezero_size = .{ .value = c.pagezero_size }, + .stack_size = .{ .value = c.stack_size }, + .headerpad_size = .{ .value = c.headerpad_size }, + .error_limit = .{ .value = c.error_limit }, })); log.err("TODO serialize the trailing Compile step data", .{}); @@ -433,13 +472,13 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .dest_dir = try addInstallDir(wc, ia.dest_dir), .dest_sub_path = try wc.addString(ia.dest_sub_path), - .emitted_bin = try s.addOptionalLazyPath(ia.emitted_bin), + .emitted_bin = try s.addOptionalLazyPathEnum(ia.emitted_bin), .implib_dir = try addInstallDir(wc, ia.implib_dir), - .emitted_implib = try s.addOptionalLazyPath(ia.emitted_implib), + .emitted_implib = try s.addOptionalLazyPathEnum(ia.emitted_implib), .pdb_dir = try addInstallDir(wc, ia.pdb_dir), - .emitted_pdb = try s.addOptionalLazyPath(ia.emitted_pdb), + .emitted_pdb = try s.addOptionalLazyPathEnum(ia.emitted_pdb), .h_dir = try addInstallDir(wc, ia.h_dir), - .emitted_h = try s.addOptionalLazyPath(ia.emitted_h), + .emitted_h = try s.addOptionalLazyPathEnum(ia.emitted_h), .artifact = stepIndex(&step_map, &ia.artifact.step), })); }, @@ -490,7 +529,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .file_inputs_len = @intCast(run.file_inputs.items.len), .args_len = @intCast(run.argv.items.len), - .cwd = try s.addOptionalLazyPath(run.cwd), + .cwd = try s.addOptionalLazyPathEnum(run.cwd), .captured_stdout = captured_stdout, .captured_stderr = captured_stderr, })); @@ -565,8 +604,7 @@ fn addModule(s: *Serialize, m: *std.Build.Module) !Configuration.Module.Index { .frameworks = m.frameworks.entries.len != 0, .link_objects = m.link_objects.items.len != 0, .export_symbol_names = m.export_symbol_names.len != 0, - }, - .flags2 = .{ + .valgrind = .init(m.strip), .pic = .init(m.strip), .red_zone = .init(m.strip), @@ -577,7 +615,7 @@ fn addModule(s: *Serialize, m: *std.Build.Module) !Configuration.Module.Index { .no_builtin = .init(m.strip), }, .owner = try s.builderToPackage(m.owner), - .root_source_file = try s.addOptionalLazyPath(m.root_source_file), + .root_source_file = try s.addOptionalLazyPathEnum(m.root_source_file), .import_table = import_table, .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target), }))); diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 40d2a4a38abb4f963903efcd004565b7572e9b9e..abb89bc21839c012c0ceab2d9dd273875e754f33 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -145,8 +145,8 @@ link_z_defs: bool = false, /// (Darwin) Install name for the dylib install_name: ?[]const u8 = null, -/// (Darwin) Path to entitlements file -entitlements: ?[]const u8 = null, +/// Must be passed in via `Options`. +entitlements: ?LazyPath = null, /// (Darwin) Size of the pagezero segment. pagezero_size: ?u64 = null, @@ -286,6 +286,8 @@ pub const Options = struct { win32_manifest: ?LazyPath = null, /// Win32 module definition file. win32_module_definition: ?LazyPath = null, + /// (Darwin) Path to entitlements file + entitlements: ?LazyPath = null, }; pub const Kind = std.Build.Configuration.Step.Compile.Kind; @@ -462,6 +464,11 @@ pub fn create(owner: *std.Build, options: Options) *Compile { } } + if (options.entitlements) |lp| { + compile.entitlements = lp.dupe(compile.step.owner); + lp.addStepDependencies(&compile.step); + } + if (compile.kind == .lib) { if (compile.linkage != null and compile.linkage.? == .static) { compile.out_lib_filename = compile.out_filename; diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 0e40f0ce7e93a83f6ff179a5db1a4f9b0292dfb6..6ae31958c618d2ac0c8b2fb0401c0542fd10bf3d 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -205,9 +205,7 @@ pub const Wip = struct { null; const cpu_features_add_empty = q.cpu_features_add.isEmpty(); const cpu_features_sub_empty = q.cpu_features_sub.isEmpty(); - try wip.extra.ensureUnusedCapacity(gpa, @typeInfo(TargetQuery).@"struct".fields.len + 6 + - 2 * ((@sizeOf(std.Target.Cpu.Feature.Set) + 3) / 4)); - const result_index: TargetQuery.Index = @enumFromInt(wip.addExtraAssumeCapacity(@as(TargetQuery, .{ + const result_index: TargetQuery.Index = @enumFromInt(try wip.addExtra(@as(TargetQuery, .{ .flags = .{ .cpu_arch = .init(q.cpu_arch), .cpu_model = .init(q.cpu_model), @@ -218,19 +216,20 @@ pub const Wip = struct { .object_format = .init(q.ofmt), .os_version_min = .init(q.os_version_min), .os_version_max = .init(q.os_version_max), - .glibc_version = q.glibc_version != null, + .glibc_version = glibc_version != null, .android_api_level = q.android_api_level != null, - .dynamic_linker = q.dynamic_linker != null, + .dynamic_linker = dynamic_linker != null, }, + .cpu_features_add = .{ .value = if (cpu_features_add_empty) null else q.cpu_features_add }, + .cpu_features_sub = .{ .value = if (cpu_features_sub_empty) null else q.cpu_features_sub }, + .glibc_version = .{ .value = glibc_version }, + .android_api_level = .{ .value = q.android_api_level }, + .dynamic_linker = .{ .value = dynamic_linker }, }))); - if (!cpu_features_add_empty) wip.extra.appendSliceAssumeCapacity(@ptrCast(&q.cpu_features_add.ints)); - if (!cpu_features_sub_empty) wip.extra.appendSliceAssumeCapacity(@ptrCast(&q.cpu_features_sub.ints)); - wip.addExtraOptionalStringAssumeCapacity(cpu_name); - if (os_version_min) |v| wip.extra.appendAssumeCapacity(v); - if (os_version_max) |v| wip.extra.appendAssumeCapacity(v); - wip.addExtraOptionalStringAssumeCapacity(glibc_version); - if (q.android_api_level) |x| wip.extra.appendAssumeCapacity(x); - wip.addExtraOptionalStringAssumeCapacity(dynamic_linker); + std.log.err("TODO serialize more target query stuff", .{}); + _ = os_version_min; + _ = os_version_max; + _ = cpu_name; // Deduplicate. const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ @@ -287,9 +286,7 @@ pub const Wip = struct { .semver, .linux, .hurd => .semver, .windows => .windows, }; - try wip.extra.ensureUnusedCapacity(gpa, @typeInfo(TargetQuery).@"struct".fields.len + 6 + - 2 * ((@sizeOf(std.Target.Cpu.Feature.Set) + 3) / 4)); - const result_index: TargetQuery.Index = @enumFromInt(wip.addExtraAssumeCapacity(@as(TargetQuery, .{ + const result_index: TargetQuery.Index = @enumFromInt(try wip.addExtra(@as(TargetQuery, .{ .flags = .{ .cpu_arch = .init(t.cpu.arch), .cpu_model = .explicit, @@ -304,14 +301,16 @@ pub const Wip = struct { .android_api_level = android_api_level != null, .dynamic_linker = dynamic_linker != null, }, + .cpu_features_add = .{ .value = if (cpu_features_add_empty) null else t.cpu.features }, + .cpu_features_sub = .{ .value = null }, + .glibc_version = .{ .value = glibc_version }, + .android_api_level = .{ .value = android_api_level }, + .dynamic_linker = .{ .value = dynamic_linker }, }))); - if (!cpu_features_add_empty) wip.extra.appendSliceAssumeCapacity(@ptrCast(&t.cpu.features.ints)); - wip.addExtraOptionalStringAssumeCapacity(cpu_name); - if (os_version_min) |v| wip.extra.appendAssumeCapacity(v); - if (os_version_max) |v| wip.extra.appendAssumeCapacity(v); - wip.addExtraOptionalStringAssumeCapacity(glibc_version); - if (android_api_level) |x| wip.extra.appendAssumeCapacity(x); - wip.addExtraOptionalStringAssumeCapacity(dynamic_linker); + std.log.err("TODO serialize more target stuff", .{}); + _ = cpu_name; + _ = os_version_min; + _ = os_version_max; // Deduplicate. const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ @@ -345,17 +344,14 @@ pub const Wip = struct { } pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { - const gpa = wip.gpa; - const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; - try wip.extra.ensureUnusedCapacity(gpa, fields.len); + const extra_len = Storage.calculateExtraLenUpperBound(@TypeOf(extra)); + try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len); return addExtraAssumeCapacity(wip, extra); } pub fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 { - const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; const result: u32 = @intCast(wip.extra.items.len); - wip.extra.items.len += fields.len; - setExtra(wip, result, extra); + wip.extra.items.len = Storage.setExtra(wip.extra.allocatedSlice(), result, extra); return result; } @@ -363,21 +359,6 @@ pub const Wip = struct { const string = optional_string orelse return; wip.extra.appendAssumeCapacity(@intFromEnum(string)); } - - fn setExtra(wip: *Wip, index: usize, extra: anytype) void { - const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; - var i = index; - inline for (fields) |field| { - comptime assert(@sizeOf(field.type) == @sizeOf(u32)); - wip.extra.items[i] = switch (@typeInfo(field.type)) { - .int => @field(extra, field.name), - .@"enum" => @intFromEnum(@field(extra, field.name)), - .@"struct" => @bitCast(@field(extra, field.name)), - else => @compileError("bad field type: " ++ @typeName(field.type)), - }; - i += 1; - } - } }; pub const SystemIntegration = extern struct { @@ -577,7 +558,7 @@ pub const Step = extern struct { }; /// Trailing: - /// * exact_match: String, // if expected_compile_errors is contains, starts_with, or stderr_contains + /// * exact_match: String, // if expect_errors is contains, starts_with, or stderr_contains /// * test_runner: LazyPath, // if test_runner_mode is not default pub const Compile = struct { flags: @This().Flags, @@ -588,36 +569,34 @@ pub const Step = extern struct { root_module: Module.Index, root_name: String, - trailing: Trailing(struct { - filters: FlagLengthPrefixedList(.flags, .filters_len, String), - exec_cmd_args: FlagLengthPrefixedList(.flags, .exec_cmd_args_len, u32), - installed_headers: FlagLengthPrefixedList(.flags, .installed_headers_len, InstalledHeader), - force_undefined_symbols: FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), - exacts: EnumConditionalPrefixedList(.flags4, .expected_compile_errors, .exact, String), - linker_script: FlagOptional(.flags4, .linker_script, LazyPath), - version_script: FlagOptional(.flags4, .version_script, LazyPath), - zig_lib_dir: FlagOptional(.flags3, .zig_lib_dir, LazyPath), - libc_file: FlagOptional(.flags4, .libc_file, LazyPath), - win32_manifest: FlagOptional(.flags3, .win32_manifest, LazyPath), - win32_module_definition: FlagOptional(.flags3, .win32_module_definition, LazyPath), - entitlements: FlagOptional(.flags4, .entitlements, LazyPath), - version: FlagOptional(.flags3, .version, String), // semantic version string - entry: EnumOptional(.flags3, .entry, .symbol, String), - install_name: FlagOptional(.flags4, .install_name, String), - initial_memory: FlagOptional(.flags3, .initial_memory, u64), - max_memory: FlagOptional(.flags3, .max_memory, u64), - global_base: FlagOptional(.flags3, .global_base, u64), - image_base: FlagOptional(.flags3, .image_base, u64), - link_z_common_page_size: FlagOptional(.flags4, .link_z_common_page_size, u64), - link_z_max_page_size: FlagOptional(.flags4, .link_z_max_page_size, u64), - pagezero_size: FlagOptional(.flags4, .pagezero_size, u64), - stack_size: FlagOptional(.flags4, .stack_size, u64), - headerpad_size: FlagOptional(.flags4, .headerpad_size, u32), - error_limit: FlagOptional(.flags4, .error_limit, u32), - build_id: EnumOptional(.flags3, .build_id, .hexstring, Hexstring), - }), + //filters: FlagLengthPrefixedList(.flags, .filters_len, String), + //exec_cmd_args: FlagLengthPrefixedList(.flags, .exec_cmd_args_len, u32), + //installed_headers: FlagLengthPrefixedList(.flags, .installed_headers_len, InstalledHeader), + //force_undefined_symbols: FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), + //exacts: EnumConditionalPrefixedList(.flags4, .expect_errors, .exact, String), + linker_script: Storage.FlagOptional(.flags4, .linker_script, LazyPath), + version_script: Storage.FlagOptional(.flags4, .version_script, LazyPath), + zig_lib_dir: Storage.FlagOptional(.flags3, .zig_lib_dir, LazyPath), + libc_file: Storage.FlagOptional(.flags4, .libc_file, LazyPath), + win32_manifest: Storage.FlagOptional(.flags3, .win32_manifest, LazyPath), + win32_module_definition: Storage.FlagOptional(.flags3, .win32_module_definition, LazyPath), + entitlements: Storage.FlagOptional(.flags4, .entitlements, LazyPath), + version: Storage.FlagOptional(.flags3, .version, String), // semantic version string + //entry: EnumOptional(.flags3, .entry, .symbol, String), + install_name: Storage.FlagOptional(.flags4, .install_name, String), + initial_memory: Storage.FlagOptional(.flags3, .initial_memory, u64), + max_memory: Storage.FlagOptional(.flags3, .max_memory, u64), + global_base: Storage.FlagOptional(.flags3, .global_base, u64), + image_base: Storage.FlagOptional(.flags3, .image_base, u64), + link_z_common_page_size: Storage.FlagOptional(.flags4, .link_z_common_page_size, u64), + link_z_max_page_size: Storage.FlagOptional(.flags4, .link_z_max_page_size, u64), + pagezero_size: Storage.FlagOptional(.flags4, .pagezero_size, u64), + stack_size: Storage.FlagOptional(.flags4, .stack_size, u64), + headerpad_size: Storage.FlagOptional(.flags4, .headerpad_size, u32), + error_limit: Storage.FlagOptional(.flags4, .error_limit, u32), + //build_id: EnumOptional(.flags3, .build_id, .hexstring, Hexstring), - pub const ExpectedCompileErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; + pub const ExpectErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; pub const TestRunnerMode = enum(u2) { default, simple, server }; pub const Entry = enum(u2) { default, disabled, enabled, symbol_name }; @@ -803,7 +782,7 @@ pub const Step = extern struct { error_limit: bool, install_name: bool, entitlements: bool, - expected_compile_errors: ExpectedCompileErrors, + expect_errors: ExpectErrors, linker_script: bool, version_script: bool, _: u18 = 0, @@ -833,6 +812,13 @@ pub const MaxRss = enum(u32) { pub const OptionalLazyPath = enum(u32) { none = maxInt(u32), _, + + pub fn unwrap(this: @This()) ?LazyPath { + return switch (this) { + .none => null, + else => @enumFromInt(@intFromEnum(this)), + }; + } }; /// An index into `extra`. @@ -909,7 +895,6 @@ pub const Package = struct { /// * link_objects: UnionList(LinkObject), // if flag is set pub const Module = struct { flags: Flags, - flags2: Flags2, owner: Package.Index, root_source_file: OptionalLazyPath, import_table: ImportTable, @@ -979,7 +964,7 @@ pub const Module = struct { _, }; - pub const Flags = packed struct(u32) { + pub const Flags = packed struct(u64) { optimize: Optimize, strip: DefaultingBool, unwind_tables: UnwindTables, @@ -998,9 +983,7 @@ pub const Module = struct { frameworks: bool, link_objects: bool, export_symbol_names: bool, - }; - pub const Flags2 = packed struct(u32) { valgrind: DefaultingBool, pic: DefaultingBool, red_zone: DefaultingBool, @@ -1257,20 +1240,20 @@ pub const ResolvedTarget = struct { }; }; -/// Trailing: -/// * cpu_features_add: std.Target.Feature.Set, // if flag set -/// * cpu_features_sub: std.Target.Feature.Set, // if flag set -/// * cpu_name: String, // if cpu_model is explicit -/// * os_version_min: WindowsVersion // if os_version_min is windows -/// * os_version_min: String // if os_version_min is semver -/// * os_version_max: WindowsVersion // if os_version_max is windows -/// * os_version_max: String // if os_version_max is semver -/// * glibc_version: String, // if flag is set -/// * android_api_level: u32, // if flag is set -/// * dynamic_linker: String, // if flag is set pub const TargetQuery = struct { flags: Flags, + cpu_features_add: Storage.FlagOptional(.flags, .cpu_features_add, std.Target.Cpu.Feature.Set), + cpu_features_sub: Storage.FlagOptional(.flags, .cpu_features_sub, std.Target.Cpu.Feature.Set), + //cpu_name: Storage.EnumOptional(.flags, .cpu_name, .explicit, String), + //os_version_min: Storage.EnumOptional(.flags, .os_version_min, .windows, WindowsVersion), + //os_version_min: Storage.EnumOptional(.flags, .os_version_min, .semver, String), + //os_version_max: Storage.EnumOptional(.flags, .os_version_max, .windows, WindowsVersion), + //os_version_max: Storage.EnumOptional(.flags, .os_version_max, .semver, String), + glibc_version: Storage.FlagOptional(.flags, .glibc_version, String), + android_api_level: Storage.FlagOptional(.flags, .android_api_level, u32), + dynamic_linker: Storage.FlagOptional(.flags, .dynamic_linker, String), + pub const Index = enum(u32) { _, @@ -1279,24 +1262,7 @@ pub const TargetQuery = struct { } pub fn length(i: Index, extra: []const u32) usize { - //const flags = getExtra(extra, @intFromEnum(i), TargetQuery).flags; - const flags: Flags = @bitCast(extra[@intFromEnum(i)]); - const feature_set_size: usize = (@sizeOf(std.Target.Cpu.Feature.Set) + 3) / 4; - return @typeInfo(TargetQuery).@"struct".fields.len + - (if (flags.cpu_features_add) feature_set_size else 0) + - (if (flags.cpu_features_sub) feature_set_size else 0) + - @intFromBool(flags.cpu_model == .explicit) + - @as(usize, switch (flags.os_version_min) { - .semver, .windows => 1, - else => 0, - }) + - @as(usize, switch (flags.os_version_max) { - .semver, .windows => 1, - else => 0, - }) + - @intFromBool(flags.glibc_version) + - @intFromBool(flags.android_api_level) + - @intFromBool(flags.dynamic_linker); + return Storage.dataLength(extra, @intFromEnum(i), TargetQuery); } }; @@ -1528,21 +1494,186 @@ pub const TargetQuery = struct { }; }; +pub const Storage = enum { + flag_optional, + + pub fn FlagOptional( + comptime flags_arg: @EnumLiteral(), + comptime flag_arg: @EnumLiteral(), + comptime ValueArg: type, + ) type { + return struct { + value: ?Value, + + pub const flags = flags_arg; + pub const flag = flag_arg; + pub const storage: Storage = .flag_optional; + pub const Value = ValueArg; + }; + } + + pub fn dataLength(buffer: []const u32, i: usize, comptime S: type) usize { + var end = i; + _ = data(buffer, &end, S); + return end - i; + } + + pub fn data(buffer: []const u32, i: *usize, comptime S: type) S { + var result: S = undefined; + const fields = @typeInfo(S).@"struct".fields; + inline for (fields) |field| { + @field(result, field.name) = dataField(buffer, i, &result, field.type); + } + return result; + } + + fn dataField(buffer: []const u32, i: *usize, container: anytype, comptime Field: type) Field { + switch (@typeInfo(Field)) { + .int => |info| switch (info.bits) { + 32 => { + defer i.* += 1; + return buffer[i.*]; + }, + 64 => { + defer i.* += 2; + return buffer[i.*..][0..2].*; + }, + else => comptime unreachable, + }, + .@"enum" => { + defer i.* += 1; + return @enumFromInt(buffer[i.*]); + }, + .@"struct" => |info| switch (info.layout) { + .@"packed" => switch (info.backing_integer.?) { + u32 => { + defer i.* += 1; + return @bitCast(buffer[i.*]); + }, + u64 => { + defer i.* += 2; + return @bitCast(buffer[i.*..][0..2].*); + }, + else => comptime unreachable, + }, + .auto => switch (Field) { + std.Target.Cpu.Feature.Set => { + const u32_count = (Field.usize_count * @sizeOf(usize)) / @sizeOf(u32); + defer i.* += u32_count; + return .{ .ints = @as( + *align(@alignOf(u32)) const [Field.usize_count]usize, + @ptrCast(buffer[i.*..][0..u32_count]), + ).* }; + }, + else => switch (Field.storage) { + .flag_optional => { + const flags = @field(container, @tagName(Field.flags)); + const flag = @field(flags, @tagName(Field.flag)); + return .{ + .value = if (flag) dataField(buffer, i, container, Field.Value) else null, + }; + }, + }, + }, + .@"extern" => comptime unreachable, + }, + else => comptime unreachable, + } + } + + /// Returns new end index. + fn setExtra(buffer: []u32, index: usize, extra: anytype) usize { + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + var i = index; + inline for (fields) |field| { + i += setExtraField(buffer, i, field.type, @field(extra, field.name)); + } + return i; + } + + fn calculateExtraLenUpperBound(comptime Extra: type) comptime_int { + var i = 0; + const fields = @typeInfo(Extra).@"struct".fields; + inline for (fields) |field| { + i += calculateExtraFieldLenUpperBound(field.type); + } + return i; + } + + inline fn setExtraField(buffer: []u32, i: usize, comptime Field: type, value: anytype) usize { + switch (@typeInfo(Field)) { + .int => |info| switch (info.bits) { + 32 => { + buffer[i] = value; + return 1; + }, + 64 => { + buffer[i..][0..2].* = @bitCast(value); + return 2; + }, + else => comptime unreachable, + }, + .@"enum" => { + buffer[i] = @intFromEnum(value); + return 1; + }, + .@"struct" => |info| switch (info.layout) { + .@"packed" => switch (info.backing_integer.?) { + u32 => { + buffer[i] = @bitCast(value); + return 1; + }, + u64 => { + buffer[i..][0..2].* = @bitCast(value); + return 2; + }, + else => comptime unreachable, + }, + .auto => switch (Field) { + std.Target.Cpu.Feature.Set => { + const casted: []const u32 = @ptrCast(&value.ints); + @memcpy(buffer[i..][0..casted.len], casted); + return casted.len; + }, + else => switch (Field.storage) { + .flag_optional => { + return if (value.value) |v| setExtraField(buffer, i, Field.Value, v) else 0; + }, + }, + }, + .@"extern" => comptime unreachable, + }, + else => @compileError("bad field type: " ++ @typeName(Field)), + } + } + + fn calculateExtraFieldLenUpperBound(comptime Field: type) comptime_int { + return switch (@typeInfo(Field)) { + .int => |info| switch (info.bits) { + 32 => 1, + 64 => 2, + else => comptime unreachable, + }, + .@"enum" => 1, + .@"struct" => |info| switch (info.layout) { + .@"packed" => switch (info.backing_integer.?) { + u32 => 1, + u64 => 2, + else => comptime unreachable, + }, + .auto => switch (Field.storage) { + .flag_optional => 1, + }, + .@"extern" => comptime unreachable, + }, + else => comptime unreachable, + }; + } +}; + pub fn extraData(c: *const Configuration, comptime T: type, index: usize) T { - const extra = c.extra; var i: usize = index; - var result: T = undefined; - inline for (@typeInfo(T).@"struct".fields) |field| { - comptime assert(@sizeOf(field.type) == @sizeOf(u32)); - @field(result, field.name) = switch (@typeInfo(field.type)) { - .int => extra[i], - .@"enum" => @enumFromInt(extra[i]), - .@"struct" => @bitCast(extra[i]), - else => comptime unreachable, - }; - i += 1; - } - return result; + return Storage.data(c.extra, &i, T); } pub const LoadFileError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; -- 2.54.0 From 9baf02de5f17157c0e79e765b26588ec551f971c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 18:40:34 -0800 Subject: [PATCH 019/179] Maker: move ScannedConfig to separate file --- lib/compiler/Maker.zig | 203 +------------------------- lib/compiler/Maker/ScannedConfig.zig | 207 +++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 202 deletions(-) create mode 100644 lib/compiler/Maker/ScannedConfig.zig diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 278cb5340e32f1930e24c0548f7a8bc3007b071a..16e12904a44661a27c73483cd7978dde5eeb4551 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -21,6 +21,7 @@ const Graph = @import("Maker/Graph.zig"); const Step = @import("Maker/Step.zig"); const Watch = @import("Maker/Watch.zig"); const WebServer = @import("Maker/WebServer.zig"); +const ScannedConfig = @import("Maker/ScannedConfig.zig"); pub const std_options: std.Options = .{ .side_channels_mitigations = .none, @@ -1667,205 +1668,3 @@ fn initStdoutWriter(io: Io) *Writer { stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); return &stdout_writer_allocation.interface; } - -const ScannedConfig = struct { - configuration: Configuration, - top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), - - fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { - const c = &sc.configuration; - var serializer: std.zon.Serializer = .{ .writer = w }; - var s = try serializer.beginStruct(.{}); - - try s.field("default_step", @intFromEnum(c.default_step), .{}); - { - var ss = try s.beginStructField("top_level_steps", .{}); - for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| { - try ss.field(name, @intFromEnum(step), .{}); - } - try ss.end(); - } - - try s.end(); - } - - fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { - const arena = graph.arena; - const c = &sc.configuration; - for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| { - const step = step_index.ptr(c); - const decorated_name = if (step_index == c.default_step) - try fmt.allocPrint(arena, "{s} (default)", .{name}) - else - name; - const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); - const description = top_level.description.slice(c); - try w.print(" {s:<28} {s}\n", .{ decorated_name, description }); - } - } - - fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { - const arena = graph.arena; - - try w.print( - \\Usage: {s} build [steps] [options] - \\ - \\Steps: - \\ - , .{graph.zig_exe}); - try printSteps(sc, graph, w); - try w.writeAll( - \\ - \\Project-Specific Options: - \\ - ); - - const available_options = sc.configuration.available_options; - if (available_options.len == 0) { - try w.print(" (none)\n", .{}); - } else { - for (available_options) |option| { - const name = option.name.slice(&sc.configuration); - const description = option.description.slice(&sc.configuration); - const help = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type }); - try w.print("{s:<30} {s}\n", .{ help, description }); - if (option.enum_options.slice(&sc.configuration)) |enum_options| { - const padding: [33]u8 = @splat(' '); - try w.writeAll(padding ++ "Supported Values:\n"); - for (enum_options) |enum_option_index| { - const enum_option = enum_option_index.slice(&sc.configuration); - try w.print(padding ++ " {s}\n", .{enum_option}); - } - } - } - } - - try w.writeAll( - \\ - \\System Integration Options: - \\ --search-prefix [path] Add a path to look for binaries, libraries, headers - \\ --sysroot [path] Set the system root directory (usually /) - \\ --libc [file] Provide a file which specifies libc paths - \\ - \\ --system [pkgdir] Disable package fetching; enable all integrations - \\ -fsys=[name] Enable a system integration - \\ -fno-sys=[name] Disable a system integration - \\ - \\ -fdarling, -fno-darling Integration with system-installed Darling to - \\ execute macOS programs on Linux hosts - \\ (default: no) - \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute - \\ foreign-architecture programs on Linux hosts - \\ (default: no) - \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc - \\ (e.g. glibc or musl) built for multiple foreign - \\ architectures, allowing execution of non-native - \\ programs that link with libc. - \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on - \\ ARM64 macOS hosts. (default: no) - \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to - \\ execute WASI binaries. (default: no) - \\ -fwine, -fno-wine Integration with system-installed Wine to execute - \\ Windows programs on Linux hosts. (default: no) - \\ - \\ Available System Integrations: Enabled: - \\ - ); - if (sc.configuration.system_integrations.len == 0) { - try w.writeAll(" (none) -\n"); - } else { - for (sc.configuration.system_integrations) |system_integration| { - const name = system_integration.name.slice(&sc.configuration); - const status = switch (system_integration.status) { - .disabled => "no", - .enabled => "yes", - }; - try w.print(" {s:<43} {s}\n", .{ name, status }); - } - } - - try w.writeAll( - \\ - \\General Options: - \\ -h, --help Print this help to stdout and exit - \\ -l, --list-steps Print available steps to stdout and exit - \\ - \\ -p, --prefix [path] Where to install files (default: zig-out) - \\ --prefix-lib-dir [path] Where to install libraries - \\ --prefix-exe-dir [path] Where to install executables - \\ --prefix-include-dir [path] Where to install C header files - \\ --release[=mode] Request release mode, optionally specifying a - \\ preferred optimization mode: fast, safe, small - \\ - \\ --verbose Print commands before executing them - \\ --color [auto|off|on] Enable or disable colored error messages - \\ --error-style [style] Control how build errors are printed - \\ verbose (Default) Report errors with full context - \\ minimal Report errors after summary, excluding context like command lines - \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update - \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update - \\ --multiline-errors [style] Control how multi-line error messages are printed - \\ indent (Default) Indent non-initial lines to align with initial line - \\ newline Include a leading newline so that the error message is on its own lines - \\ none Print as usual so the first line is misaligned - \\ --summary [mode] Control the printing of the build summary - \\ all Print the build summary in its entirety - \\ new Omit cached steps - \\ failures (Default if short-lived) Only print failed steps - \\ line (Default if long-lived) Only print the single-line summary - \\ none Do not print the build summary - \\ -j Limit concurrent jobs (default is to use all CPU cores) - \\ --maxrss Limit memory usage (default is to use available memory) - \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss - \\ --test-timeout Limit execution time of unit tests, terminating if exceeded. - \\ The timeout must include a unit: ns, us, ms, s, m, h - \\ --watch Continuously rebuild when source files are modified - \\ --debounce Delay before rebuilding after changed file detected - \\ --webui[=ip] Enable the web interface on the given IP address - \\ --fuzz[=limit] Continuously search for unit test failures with an optional - \\ limit to the max number of iterations. The argument supports - \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies - \\ '--webui' when no limit is specified. - \\ --time-report Force full rebuild and provide detailed information on - \\ compilation time of Zig source code (implies '--webui') - \\ -fincremental Enable incremental compilation - \\ -fno-incremental Disable incremental compilation - \\ - \\Package Management Options: - \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit - \\ needed (Default) Lazy dependencies are fetched as needed - \\ all Lazy dependencies are always fetched - \\ --fork=[path] Override one or more projects from dependency tree - \\ - \\Advanced Options: - \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error - \\ -fno-reference-trace Disable reference trace - \\ -fallow-so-scripts Allows .so files to be GNU ld scripts - \\ -fno-allow-so-scripts (default) .so files must be ELF files - \\ --error-limit [num] Set the maximum amount of distinct error values - \\ --build-file [file] Override path to build.zig - \\ --cache-dir [path] Override path to local Zig cache directory - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --zig-lib-dir [arg] Override path to Zig lib directory - \\ --build-runner [file] Override path to build runner - \\ --seed [integer] For shuffling dependency traversal order (default: random) - \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries - \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) - \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) - \\ md5 16-byte cryptographic hash (ELF) - \\ uuid 16-byte random UUID (ELF, WASM) - \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) - \\ none (default) No build ID - \\ --debug-log [scope] Enable debugging the compiler - \\ --debug-pkg-config Fail if unknown pkg-config flags encountered - \\ --debug-rt Debug compiler runtime libraries - \\ --verbose-link Enable compiler debug output for linking - \\ --verbose-air Enable compiler debug output for Zig AIR - \\ --verbose-llvm-ir Enable compiler debug output for LLVM IR - \\ --verbose-cimport Enable compiler debug output for C imports - \\ --verbose-cc Enable compiler debug output for C compilation - \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features - \\ - ); - } -}; diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig new file mode 100644 index 0000000000000000000000000000000000000000..ef253c9170f181931b3f3a5a0568ef26c56f5a39 --- /dev/null +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -0,0 +1,207 @@ +const ScannedConfig = @This(); + +const std = @import("std"); +const Configuration = std.Build.Configuration; +const Writer = std.Io.Writer; + +const Graph = @import("Graph.zig"); + +configuration: Configuration, +top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), + +pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { + const c = &sc.configuration; + var serializer: std.zon.Serializer = .{ .writer = w }; + var s = try serializer.beginStruct(.{}); + + try s.field("default_step", @intFromEnum(c.default_step), .{}); + { + var ss = try s.beginStructField("top_level_steps", .{}); + for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| { + try ss.field(name, @intFromEnum(step), .{}); + } + try ss.end(); + } + + try s.end(); +} + +pub fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + const c = &sc.configuration; + for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| { + const step = step_index.ptr(c); + const decorated_name = if (step_index == c.default_step) + try std.fmt.allocPrint(arena, "{s} (default)", .{name}) + else + name; + const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); + const description = top_level.description.slice(c); + try w.print(" {s:<28} {s}\n", .{ decorated_name, description }); + } +} + +pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + + try w.print( + \\Usage: {s} build [steps] [options] + \\ + \\Steps: + \\ + , .{graph.zig_exe}); + try printSteps(sc, graph, w); + try w.writeAll( + \\ + \\Project-Specific Options: + \\ + ); + + const available_options = sc.configuration.available_options; + if (available_options.len == 0) { + try w.print(" (none)\n", .{}); + } else { + for (available_options) |option| { + const name = option.name.slice(&sc.configuration); + const description = option.description.slice(&sc.configuration); + const help = try std.fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type }); + try w.print("{s:<30} {s}\n", .{ help, description }); + if (option.enum_options.slice(&sc.configuration)) |enum_options| { + const padding: [33]u8 = @splat(' '); + try w.writeAll(padding ++ "Supported Values:\n"); + for (enum_options) |enum_option_index| { + const enum_option = enum_option_index.slice(&sc.configuration); + try w.print(padding ++ " {s}\n", .{enum_option}); + } + } + } + } + + try w.writeAll( + \\ + \\System Integration Options: + \\ --search-prefix [path] Add a path to look for binaries, libraries, headers + \\ --sysroot [path] Set the system root directory (usually /) + \\ --libc [file] Provide a file which specifies libc paths + \\ + \\ --system [pkgdir] Disable package fetching; enable all integrations + \\ -fsys=[name] Enable a system integration + \\ -fno-sys=[name] Disable a system integration + \\ + \\ -fdarling, -fno-darling Integration with system-installed Darling to + \\ execute macOS programs on Linux hosts + \\ (default: no) + \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute + \\ foreign-architecture programs on Linux hosts + \\ (default: no) + \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc + \\ (e.g. glibc or musl) built for multiple foreign + \\ architectures, allowing execution of non-native + \\ programs that link with libc. + \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on + \\ ARM64 macOS hosts. (default: no) + \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to + \\ execute WASI binaries. (default: no) + \\ -fwine, -fno-wine Integration with system-installed Wine to execute + \\ Windows programs on Linux hosts. (default: no) + \\ + \\ Available System Integrations: Enabled: + \\ + ); + if (sc.configuration.system_integrations.len == 0) { + try w.writeAll(" (none) -\n"); + } else { + for (sc.configuration.system_integrations) |system_integration| { + const name = system_integration.name.slice(&sc.configuration); + const status = switch (system_integration.status) { + .disabled => "no", + .enabled => "yes", + }; + try w.print(" {s:<43} {s}\n", .{ name, status }); + } + } + + try w.writeAll( + \\ + \\General Options: + \\ -h, --help Print this help to stdout and exit + \\ -l, --list-steps Print available steps to stdout and exit + \\ + \\ -p, --prefix [path] Where to install files (default: zig-out) + \\ --prefix-lib-dir [path] Where to install libraries + \\ --prefix-exe-dir [path] Where to install executables + \\ --prefix-include-dir [path] Where to install C header files + \\ --release[=mode] Request release mode, optionally specifying a + \\ preferred optimization mode: fast, safe, small + \\ + \\ --verbose Print commands before executing them + \\ --color [auto|off|on] Enable or disable colored error messages + \\ --error-style [style] Control how build errors are printed + \\ verbose (Default) Report errors with full context + \\ minimal Report errors after summary, excluding context like command lines + \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update + \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update + \\ --multiline-errors [style] Control how multi-line error messages are printed + \\ indent (Default) Indent non-initial lines to align with initial line + \\ newline Include a leading newline so that the error message is on its own lines + \\ none Print as usual so the first line is misaligned + \\ --summary [mode] Control the printing of the build summary + \\ all Print the build summary in its entirety + \\ new Omit cached steps + \\ failures (Default if short-lived) Only print failed steps + \\ line (Default if long-lived) Only print the single-line summary + \\ none Do not print the build summary + \\ -j Limit concurrent jobs (default is to use all CPU cores) + \\ --maxrss Limit memory usage (default is to use available memory) + \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss + \\ --test-timeout Limit execution time of unit tests, terminating if exceeded. + \\ The timeout must include a unit: ns, us, ms, s, m, h + \\ --watch Continuously rebuild when source files are modified + \\ --debounce Delay before rebuilding after changed file detected + \\ --webui[=ip] Enable the web interface on the given IP address + \\ --fuzz[=limit] Continuously search for unit test failures with an optional + \\ limit to the max number of iterations. The argument supports + \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies + \\ '--webui' when no limit is specified. + \\ --time-report Force full rebuild and provide detailed information on + \\ compilation time of Zig source code (implies '--webui') + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation + \\ + \\Package Management Options: + \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit + \\ needed (Default) Lazy dependencies are fetched as needed + \\ all Lazy dependencies are always fetched + \\ --fork=[path] Override one or more projects from dependency tree + \\ + \\Advanced Options: + \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error + \\ -fno-reference-trace Disable reference trace + \\ -fallow-so-scripts Allows .so files to be GNU ld scripts + \\ -fno-allow-so-scripts (default) .so files must be ELF files + \\ --error-limit [num] Set the maximum amount of distinct error values + \\ --build-file [file] Override path to build.zig + \\ --cache-dir [path] Override path to local Zig cache directory + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --zig-lib-dir [arg] Override path to Zig lib directory + \\ --build-runner [file] Override path to build runner + \\ --seed [integer] For shuffling dependency traversal order (default: random) + \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries + \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) + \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) + \\ md5 16-byte cryptographic hash (ELF) + \\ uuid 16-byte random UUID (ELF, WASM) + \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) + \\ none (default) No build ID + \\ --debug-log [scope] Enable debugging the compiler + \\ --debug-pkg-config Fail if unknown pkg-config flags encountered + \\ --debug-rt Debug compiler runtime libraries + \\ --verbose-link Enable compiler debug output for linking + \\ --verbose-air Enable compiler debug output for Zig AIR + \\ --verbose-llvm-ir Enable compiler debug output for LLVM IR + \\ --verbose-cimport Enable compiler debug output for C imports + \\ --verbose-cc Enable compiler debug output for C compilation + \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features + \\ + ); +} -- 2.54.0 From 4777ad964ce6339fa78b51714a630f484daf842a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 19:16:52 -0800 Subject: [PATCH 020/179] ScannedConfig: print Step header data --- lib/compiler/Maker/ScannedConfig.zig | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index ef253c9170f181931b3f3a5a0568ef26c56f5a39..b67166a529ded62bb11ae8800bfc4aa61627fde6 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -16,11 +16,33 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { try s.field("default_step", @intFromEnum(c.default_step), .{}); { - var ss = try s.beginStructField("top_level_steps", .{}); + var sf = try s.beginStructField("top_level_steps", .{}); for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| { - try ss.field(name, @intFromEnum(step), .{}); + try sf.field(name, @intFromEnum(step), .{}); } - try ss.end(); + try sf.end(); + } + + { + var tf = try s.beginTupleField("steps", .{}); + for (c.steps) |*step| { + var step_field = try tf.beginStructField(.{}); + try step_field.field("name", step.name.slice(c), .{}); + switch (step.owner) { + .root => try step_field.field("owner", .root, .{}), + _ => try step_field.field("owner", @intFromEnum(step.owner), .{}), + } + { + var deps_field = try step_field.beginTupleField("deps", .{}); + for (step.deps.slice(c)) |dep| { + try deps_field.field(@intFromEnum(dep), .{}); + } + try deps_field.end(); + } + try step_field.field("max_rss", step.max_rss.toBytes(), .{}); + try step_field.end(); + } + try tf.end(); } try s.end(); -- 2.54.0 From 20bb5bde88e4280154780eb6c5ba7810fa6ab810 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 19:24:38 -0800 Subject: [PATCH 021/179] ScannedConfig: print Step.TopLevel --- lib/compiler/Maker/ScannedConfig.zig | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index b67166a529ded62bb11ae8800bfc4aa61627fde6..be46898734eb46fe00b8224f5b7106994af210d8 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -40,6 +40,31 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { try deps_field.end(); } try step_field.field("max_rss", step.max_rss.toBytes(), .{}); + const type_erased_flags: Configuration.Step.Flags = @bitCast(c.extra[step.extra_index]); + switch (type_erased_flags.tag) { + .check_file => try step_field.field("check_file", .TODO, .{}), + .check_object => try step_field.field("check_object", .TODO, .{}), + .compile => try step_field.field("compile", .TODO, .{}), + .config_header => try step_field.field("config_header", .TODO, .{}), + .fail => try step_field.field("fail", .TODO, .{}), + .fmt => try step_field.field("fmt", .TODO, .{}), + .install_artifact => try step_field.field("install_artifact", .TODO, .{}), + .install_dir => try step_field.field("install_dir", .TODO, .{}), + .install_file => try step_field.field("install_file", .TODO, .{}), + .objcopy => try step_field.field("objcopy", .TODO, .{}), + .options => try step_field.field("options", .TODO, .{}), + .remove_dir => try step_field.field("remove_dir", .TODO, .{}), + .run => try step_field.field("run", .TODO, .{}), + .top_level => { + const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); + var sf = try step_field.beginStructField("top_level", .{}); + try sf.field("description", top_level.description.slice(c), .{}); + try sf.end(); + }, + .translate_c => try step_field.field("translate_c", .TODO, .{}), + .update_source_files => try step_field.field("update_source_files", .TODO, .{}), + .write_file => try step_field.field("write_file", .TODO, .{}), + } try step_field.end(); } try tf.end(); -- 2.54.0 From 603e92cdde0afaa046ec8b560ae9e216c72cf588 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 19:26:05 -0800 Subject: [PATCH 022/179] Maker: don't include non-root top level steps --- lib/compiler/Maker.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 16e12904a44661a27c73483cd7978dde5eeb4551..fa701001390e005a926ba17d1a6cc5bf62da7199 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -431,6 +431,7 @@ pub fn main(init: process.Init.Minimal) !void { }; var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; for (configuration.steps, 0..) |*conf_step, step_index_usize| { + if (conf_step.owner != .root) continue; const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); const flags = conf_step.flags(&configuration); if (flags.tag == .top_level) { -- 2.54.0 From b9aeedd23c303efc3c74a308ed2620ab1c470af7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 21:13:00 -0800 Subject: [PATCH 023/179] Configuration: type safety for extended pattern --- lib/compiler/Maker/ScannedConfig.zig | 8 +- lib/compiler/configure_runner.zig | 16 +-- lib/std/zig/Configuration.zig | 189 +++++++++++++++++++++++++-- 3 files changed, 187 insertions(+), 26 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index be46898734eb46fe00b8224f5b7106994af210d8..3c1da26a1f704cf7b97c8bf0ad247e4de3ccff97 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -40,8 +40,7 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { try deps_field.end(); } try step_field.field("max_rss", step.max_rss.toBytes(), .{}); - const type_erased_flags: Configuration.Step.Flags = @bitCast(c.extra[step.extra_index]); - switch (type_erased_flags.tag) { + switch (step.extended.get(c.extra)) { .check_file => try step_field.field("check_file", .TODO, .{}), .check_object => try step_field.field("check_object", .TODO, .{}), .compile => try step_field.field("compile", .TODO, .{}), @@ -55,8 +54,7 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { .options => try step_field.field("options", .TODO, .{}), .remove_dir => try step_field.field("remove_dir", .TODO, .{}), .run => try step_field.field("run", .TODO, .{}), - .top_level => { - const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); + .top_level => |top_level| { var sf = try step_field.beginStructField("top_level", .{}); try sf.field("description", top_level.description.slice(c), .{}); try sf.end(); @@ -82,7 +80,7 @@ pub fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { try std.fmt.allocPrint(arena, "{s} (default)", .{name}) else name; - const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); + const top_level = step.extended.get(c.extra).top_level; const description = top_level.description.slice(c); try w.print(" {s:<28} {s}\n", .{ decorated_name, description }); } diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 901b4947135cadd74794bbe15f4e336f08ee9b61..f3df4d6ec1a390a825d76fd8bb0354a1ba752c58 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -331,12 +331,12 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .owner = try s.builderToPackage(step.owner), .deps = deps, .max_rss = .fromBytes(step.max_rss), - .extra_index = switch (step.tag) { + .extended = switch (step.tag) { .top_level => e: { const top_level: *Step.TopLevel = @fieldParentPtr("step", step); - break :e try wc.addExtra(@as(Configuration.Step.TopLevel, .{ + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.TopLevel, .{ .description = try wc.addString(top_level.description), - })); + }))); }, .compile => e: { const c: *Step.Compile = @fieldParentPtr("step", step); @@ -462,11 +462,11 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { log.err("TODO serialize the trailing Compile step data", .{}); - break :e extra_index; + break :e @enumFromInt(extra_index); }, .install_artifact => e: { const ia: *Step.InstallArtifact = @fieldParentPtr("step", step); - break :e try wc.addExtra(@as(Configuration.Step.InstallArtifact, .{ + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.InstallArtifact, .{ .flags = .{ .dylib_symlinks = ia.dylib_symlinks != null, }, @@ -480,7 +480,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .h_dir = try addInstallDir(wc, ia.h_dir), .emitted_h = try s.addOptionalLazyPathEnum(ia.emitted_h), .artifact = stepIndex(&step_map, &ia.artifact.step), - })); + }))); }, .install_file => @panic("TODO"), .install_dir => @panic("TODO"), @@ -536,7 +536,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { log.err("TODO serialize the trailing Run step data", .{}); - break :e extra_index; + break :e @enumFromInt(extra_index); }, .check_file => @panic("TODO"), .check_object => @panic("TODO"), @@ -639,7 +639,7 @@ fn addOptionalResolvedTarget( }))); } -fn addInstallDir(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !Configuration.InstallDir { +fn addInstallDir(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !Configuration.InstallDestDir { switch (install_dir orelse return .none) { .prefix => return .prefix, .lib => return .lib, diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 6ae31958c618d2ac0c8b2fb0401c0542fd10bf3d..fbd9d7e570b5f455be91efb62a7197e2f00894d8 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -397,9 +397,25 @@ pub const Step = extern struct { owner: Package.Index, deps: Deps, max_rss: MaxRss, - /// Points into `extra` for step-specific data. First element has flags - /// with `Tag`. - extra_index: u32, + extended: Storage.ExtendedIndex(Flags, union(Tag) { + check_file: CheckFile, + check_object: CheckObject, + compile: Compile, + config_header: ConfigHeader, + fail: Fail, + fmt: Fmt, + install_artifact: InstallArtifact, + install_dir: InstallDir, + install_file: InstallFile, + objcopy: Objcopy, + options: Options, + remove_dir: RemoveDir, + run: Run, + top_level: TopLevel, + translate_c: TranslateC, + update_source_files: UpdateSourceFiles, + write_file: WriteFile, + }), /// Points into `steps`. pub const Index = enum(u32) { @@ -449,17 +465,17 @@ pub const Step = extern struct { pub const InstallArtifact = struct { flags: @This().Flags, - dest_dir: InstallDir, + dest_dir: InstallDestDir, dest_sub_path: String, emitted_bin: OptionalLazyPath, - implib_dir: InstallDir, + implib_dir: InstallDestDir, emitted_implib: OptionalLazyPath, - pdb_dir: InstallDir, + pdb_dir: InstallDestDir, emitted_pdb: OptionalLazyPath, - h_dir: InstallDir, + h_dir: InstallDestDir, emitted_h: OptionalLazyPath, /// Always a compile step. @@ -789,8 +805,125 @@ pub const Step = extern struct { }; }; + pub const CheckFile = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .check_file, + _: u27 = 0, + }; + }; + + pub const CheckObject = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .check_object, + _: u27 = 0, + }; + }; + + pub const ConfigHeader = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .config_header, + _: u27 = 0, + }; + }; + + pub const Fail = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .fail, + _: u27 = 0, + }; + }; + + pub const Fmt = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .fmt, + _: u27 = 0, + }; + }; + + pub const InstallDir = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .install_dir, + _: u27 = 0, + }; + }; + + pub const InstallFile = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .install_file, + _: u27 = 0, + }; + }; + + pub const Objcopy = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .objcopy, + _: u27 = 0, + }; + }; + + pub const Options = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .options, + _: u27 = 0, + }; + }; + + pub const RemoveDir = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .remove_dir, + _: u27 = 0, + }; + }; + + pub const TranslateC = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .translate_c, + _: u27 = 0, + }; + }; + + pub const UpdateSourceFiles = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .update_source_files, + _: u27 = 0, + }; + }; + + pub const WriteFile = struct { + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + tag: Tag = .write_file, + _: u27 = 0, + }; + }; + pub fn flags(s: *const Step, c: *const Configuration) Flags { - return @bitCast(c.extra[s.extra_index]); + return @bitCast(c.extra[@intFromEnum(s.extended)]); } }; @@ -1081,7 +1214,7 @@ pub const Path = extern struct { } }; -pub const InstallDir = enum(u32) { +pub const InstallDestDir = enum(u32) { none = maxInt(u32) - 4, prefix = maxInt(u32) - 3, lib = maxInt(u32) - 2, @@ -1090,8 +1223,8 @@ pub const InstallDir = enum(u32) { /// A `String` path relative to the prefix. _, - pub fn initCustom(sub_path: String) InstallDir { - assert(@intFromEnum(sub_path) < @intFromEnum(InstallDir.none)); + pub fn initCustom(sub_path: String) InstallDestDir { + assert(@intFromEnum(sub_path) < @intFromEnum(InstallDestDir.none)); return @enumFromInt(@intFromEnum(sub_path)); } }; @@ -1496,7 +1629,10 @@ pub const TargetQuery = struct { pub const Storage = enum { flag_optional, + extended, + /// The presence of the field is determined by a boolean within a packed + /// struct. pub fn FlagOptional( comptime flags_arg: @EnumLiteral(), comptime flag_arg: @EnumLiteral(), @@ -1512,6 +1648,31 @@ pub const Storage = enum { }; } + /// The field indexes into an auxilary buffer, with the first element being + /// a packed struct that contains the tag. + pub fn Extended(comptime U: type) type { + return struct { + value: U, + + pub const storage: Storage = .extended; + }; + } + + /// Equivalent to `Extended` but works in an `extern struct`. + pub fn ExtendedIndex(comptime BaseFlags: type, comptime U: type) type { + return enum(u32) { + _, + + pub fn get(this: @This(), buffer: []const u32) U { + var i: usize = @intFromEnum(this); + const base_flags: BaseFlags = @bitCast(buffer[i]); + return switch (base_flags.tag) { + inline else => |tag| @unionInit(U, @tagName(tag), data(buffer, &i, @FieldType(U, @tagName(tag)))), + }; + } + }; + } + pub fn dataLength(buffer: []const u32, i: usize, comptime S: type) usize { var end = i; _ = data(buffer, &end, S); @@ -1536,7 +1697,7 @@ pub const Storage = enum { }, 64 => { defer i.* += 2; - return buffer[i.*..][0..2].*; + return @bitCast(buffer[i.*..][0..2].*); }, else => comptime unreachable, }, @@ -1573,6 +1734,7 @@ pub const Storage = enum { .value = if (flag) dataField(buffer, i, container, Field.Value) else null, }; }, + .extended => @compileError("TODO"), }, }, .@"extern" => comptime unreachable, @@ -1639,6 +1801,7 @@ pub const Storage = enum { .flag_optional => { return if (value.value) |v| setExtraField(buffer, i, Field.Value, v) else 0; }, + .extended => @compileError("TODO"), }, }, .@"extern" => comptime unreachable, @@ -1662,7 +1825,7 @@ pub const Storage = enum { else => comptime unreachable, }, .auto => switch (Field.storage) { - .flag_optional => 1, + .flag_optional, .extended => 1, }, .@"extern" => comptime unreachable, }, -- 2.54.0 From 052aeee4150eba68051311982d883688670f6ca4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 22:06:44 -0800 Subject: [PATCH 024/179] std.zon.Serializer: slightly more helpful message when a type is unserializable --- lib/std/zon/Serializer.zig | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/std/zon/Serializer.zig b/lib/std/zon/Serializer.zig index c30c0e09d8051c0eafab2e8e4ba2a85d46aa1db2..9839129e316906fd20110f47ae40c28d193f8ec4 100644 --- a/lib/std/zon/Serializer.zig +++ b/lib/std/zon/Serializer.zig @@ -122,7 +122,7 @@ pub fn valueMaxDepth(self: *Serializer, val: anytype, options: ValueOptions, dep /// Serialize a value, similar to `serializeArbitraryDepth`. pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(canSerializeType(@TypeOf(val))); + comptime assertCanSerializeType(@TypeOf(val)); switch (@typeInfo(@TypeOf(val))) { .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { self.codePoint(c) catch |err| switch (err) { @@ -321,7 +321,7 @@ pub fn tupleArbitraryDepth( } fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(canSerializeType(@TypeOf(val))); + comptime assertCanSerializeType(@TypeOf(val)); switch (@typeInfo(@TypeOf(val))) { .@"struct" => { var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); @@ -814,6 +814,10 @@ test checkValueDepth { try expectValueDepthEquals(3, @as([]const []const u8, &.{&.{ 1, 2, 3 }})); } +inline fn assertCanSerializeType(T: type) void { + if (!canSerializeType(T)) @compileError("cannot serialize: " ++ @typeName(T)); +} + inline fn canSerializeType(T: type) bool { comptime return canSerializeTypeInner(T, &.{}, false); } -- 2.54.0 From b53e4e84bd5641d3e506f88481c92e1abafb1325 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 22:07:52 -0800 Subject: [PATCH 025/179] ScannedConfig: more general zon printing it's almost all automated now --- lib/compiler/Maker/ScannedConfig.zig | 112 ++++++++++++++++++--------- lib/std/zig/Configuration.zig | 2 + 2 files changed, 77 insertions(+), 37 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 3c1da26a1f704cf7b97c8bf0ad247e4de3ccff97..2b264ef1ef41e052bfd36f5b31bff934dbd7ac49 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -3,6 +3,7 @@ const ScannedConfig = @This(); const std = @import("std"); const Configuration = std.Build.Configuration; const Writer = std.Io.Writer; +const Serializer = std.zon.Serializer; const Graph = @import("Graph.zig"); @@ -10,8 +11,12 @@ configuration: Configuration, top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { + std.log.err("TODO also print paths", .{}); + std.log.err("TODO also print unlazy deps", .{}); + std.log.err("TODO also print system integrations", .{}); + std.log.err("TODO also print available options", .{}); const c = &sc.configuration; - var serializer: std.zon.Serializer = .{ .writer = w }; + var serializer: Serializer = .{ .writer = w }; var s = try serializer.beginStruct(.{}); try s.field("default_step", @intFromEnum(c.default_step), .{}); @@ -27,42 +32,7 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { var tf = try s.beginTupleField("steps", .{}); for (c.steps) |*step| { var step_field = try tf.beginStructField(.{}); - try step_field.field("name", step.name.slice(c), .{}); - switch (step.owner) { - .root => try step_field.field("owner", .root, .{}), - _ => try step_field.field("owner", @intFromEnum(step.owner), .{}), - } - { - var deps_field = try step_field.beginTupleField("deps", .{}); - for (step.deps.slice(c)) |dep| { - try deps_field.field(@intFromEnum(dep), .{}); - } - try deps_field.end(); - } - try step_field.field("max_rss", step.max_rss.toBytes(), .{}); - switch (step.extended.get(c.extra)) { - .check_file => try step_field.field("check_file", .TODO, .{}), - .check_object => try step_field.field("check_object", .TODO, .{}), - .compile => try step_field.field("compile", .TODO, .{}), - .config_header => try step_field.field("config_header", .TODO, .{}), - .fail => try step_field.field("fail", .TODO, .{}), - .fmt => try step_field.field("fmt", .TODO, .{}), - .install_artifact => try step_field.field("install_artifact", .TODO, .{}), - .install_dir => try step_field.field("install_dir", .TODO, .{}), - .install_file => try step_field.field("install_file", .TODO, .{}), - .objcopy => try step_field.field("objcopy", .TODO, .{}), - .options => try step_field.field("options", .TODO, .{}), - .remove_dir => try step_field.field("remove_dir", .TODO, .{}), - .run => try step_field.field("run", .TODO, .{}), - .top_level => |top_level| { - var sf = try step_field.beginStructField("top_level", .{}); - try sf.field("description", top_level.description.slice(c), .{}); - try sf.end(); - }, - .translate_c => try step_field.field("translate_c", .TODO, .{}), - .update_source_files => try step_field.field("update_source_files", .TODO, .{}), - .write_file => try step_field.field("write_file", .TODO, .{}), - } + try printStruct(sc, &step_field, Configuration.Step, step); try step_field.end(); } try tf.end(); @@ -71,6 +41,74 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { try s.end(); } +fn printStruct(sc: *const ScannedConfig, s: *Serializer.Struct, comptime S: type, v: *const S) !void { + inline for (@typeInfo(S).@"struct".fields) |field| { + try s.fieldPrefix(field.name); + try printValue(sc, s.container.serializer, field.type, @field(v, field.name)); + } +} + +fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, field_value: Field) !void { + const c = &sc.configuration; + switch (Field) { + Configuration.String => { + try s.value(field_value.slice(c), .{}); + }, + Configuration.Deps => { + var deps_field = try s.beginTuple(.{}); + for (field_value.slice(c)) |dep| { + try deps_field.field(@intFromEnum(dep), .{}); + } + try deps_field.end(); + }, + Configuration.MaxRss => { + try s.value(field_value.toBytes(), .{}); + }, + else => switch (@typeInfo(Field)) { + .int => try s.int(field_value), + .@"enum" => { + if (@hasDecl(Field, "storage")) switch (Field.storage) { + .extended => { + var sub_struct = try s.beginStruct(.{}); + try printTaggedUnion(sc, &sub_struct, field_value.get(c.extra)); + try sub_struct.end(); + }, + .flag_optional => comptime unreachable, + } else if (std.enums.tagName(Field, field_value)) |name| { + try s.ident(name); + } else { + try s.int(@intFromEnum(field_value)); + } + }, + .@"struct" => |info| switch (info.layout) { + .@"packed" => { + try s.value(field_value, .{}); + }, + .auto => switch (Field.storage) { + .flag_optional => { + if (field_value.value) |some| { + try printValue(sc, s, Field.Value, some); + } else { + try s.value(null, .{}); + } + }, + .extended => @compileError("TODO"), + }, + else => @compileError("not implemented: " ++ @typeName(Field)), + }, + else => @compileError("not implemented: " ++ @typeName(Field)), + }, + } +} + +fn printTaggedUnion(sc: *const ScannedConfig, s: *Serializer.Struct, value: anytype) !void { + switch (value) { + inline else => |*u| { + try printStruct(sc, s, @TypeOf(u.*), u); + }, + } +} + pub fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { const arena = graph.arena; const c = &sc.configuration; diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index fbd9d7e570b5f455be91efb62a7197e2f00894d8..d3cb1bc8dff5f72baef6054550c346a2d209805e 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -1663,6 +1663,8 @@ pub const Storage = enum { return enum(u32) { _, + pub const storage: Storage = .extended; + pub fn get(this: @This(), buffer: []const u32) U { var i: usize = @intFromEnum(this); const base_flags: BaseFlags = @bitCast(buffer[i]); -- 2.54.0 From e8e7fbf8432899d12888bcabeba17ea8e51dc62d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Feb 2026 23:27:43 -0800 Subject: [PATCH 026/179] Configuration: implement Storage.EnumOptional --- lib/compiler/Maker/ScannedConfig.zig | 3 +- lib/compiler/configure_runner.zig | 8 ++++ lib/std/zig/Configuration.zig | 58 +++++++++++++++++++++------- 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 2b264ef1ef41e052bfd36f5b31bff934dbd7ac49..f26b5d95dd63d40e5341a060df3baf4e01dc1fa3 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -74,6 +74,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try sub_struct.end(); }, .flag_optional => comptime unreachable, + .enum_optional => comptime unreachable, } else if (std.enums.tagName(Field, field_value)) |name| { try s.ident(name); } else { @@ -85,7 +86,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try s.value(field_value, .{}); }, .auto => switch (Field.storage) { - .flag_optional => { + .flag_optional, .enum_optional => { if (field_value.value) |some| { try printValue(sc, s, Field.Value, some); } else { diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index f3df4d6ec1a390a825d76fd8bb0354a1ba752c58..9bc76da59ada922e37e3e626a3548ffd1a9018d1 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -458,6 +458,14 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .stack_size = .{ .value = c.stack_size }, .headerpad_size = .{ .value = c.headerpad_size }, .error_limit = .{ .value = c.error_limit }, + .entry = .{ .value = switch (c.entry) { + .symbol_name => |name| try wc.addString(name), + .default, .disabled, .enabled => null, + } }, + .build_id = .{ .value = if (c.build_id) |id| switch (id) { + .hexstring => |*hexstring| try wc.addString(hexstring.toSlice()), + .none, .fast, .uuid, .sha1, .md5 => null, + } else null }, })); log.err("TODO serialize the trailing Compile step data", .{}); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index d3cb1bc8dff5f72baef6054550c346a2d209805e..6cb11453518fc84d3d6dc74ef72e07c1eb6b2ec9 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -225,11 +225,11 @@ pub const Wip = struct { .glibc_version = .{ .value = glibc_version }, .android_api_level = .{ .value = q.android_api_level }, .dynamic_linker = .{ .value = dynamic_linker }, + .cpu_name = .{ .value = cpu_name }, }))); std.log.err("TODO serialize more target query stuff", .{}); _ = os_version_min; _ = os_version_max; - _ = cpu_name; // Deduplicate. const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ @@ -306,9 +306,9 @@ pub const Wip = struct { .glibc_version = .{ .value = glibc_version }, .android_api_level = .{ .value = android_api_level }, .dynamic_linker = .{ .value = dynamic_linker }, + .cpu_name = .{ .value = cpu_name }, }))); std.log.err("TODO serialize more target stuff", .{}); - _ = cpu_name; _ = os_version_min; _ = os_version_max; @@ -598,7 +598,7 @@ pub const Step = extern struct { win32_module_definition: Storage.FlagOptional(.flags3, .win32_module_definition, LazyPath), entitlements: Storage.FlagOptional(.flags4, .entitlements, LazyPath), version: Storage.FlagOptional(.flags3, .version, String), // semantic version string - //entry: EnumOptional(.flags3, .entry, .symbol, String), + entry: Storage.EnumOptional(.flags3, .entry, .symbol_name, String), install_name: Storage.FlagOptional(.flags4, .install_name, String), initial_memory: Storage.FlagOptional(.flags3, .initial_memory, u64), max_memory: Storage.FlagOptional(.flags3, .max_memory, u64), @@ -610,7 +610,7 @@ pub const Step = extern struct { stack_size: Storage.FlagOptional(.flags4, .stack_size, u64), headerpad_size: Storage.FlagOptional(.flags4, .headerpad_size, u32), error_limit: Storage.FlagOptional(.flags4, .error_limit, u32), - //build_id: EnumOptional(.flags3, .build_id, .hexstring, Hexstring), + build_id: Storage.EnumOptional(.flags3, .build_id, .hexstring, String), pub const ExpectErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; pub const TestRunnerMode = enum(u2) { default, simple, server }; @@ -1378,15 +1378,20 @@ pub const TargetQuery = struct { cpu_features_add: Storage.FlagOptional(.flags, .cpu_features_add, std.Target.Cpu.Feature.Set), cpu_features_sub: Storage.FlagOptional(.flags, .cpu_features_sub, std.Target.Cpu.Feature.Set), - //cpu_name: Storage.EnumOptional(.flags, .cpu_name, .explicit, String), - //os_version_min: Storage.EnumOptional(.flags, .os_version_min, .windows, WindowsVersion), - //os_version_min: Storage.EnumOptional(.flags, .os_version_min, .semver, String), - //os_version_max: Storage.EnumOptional(.flags, .os_version_max, .windows, WindowsVersion), - //os_version_max: Storage.EnumOptional(.flags, .os_version_max, .semver, String), + cpu_name: Storage.EnumOptional(.flags, .cpu_model, .explicit, String), + //os_version_min: Storage.FlagsUnion(.flags, .os_version_min, VersionStorage), + //os_version_max: Storage.FlagsUnion(.flags, .os_version_max, VersionStorage), glibc_version: Storage.FlagOptional(.flags, .glibc_version, String), android_api_level: Storage.FlagOptional(.flags, .android_api_level, u32), dynamic_linker: Storage.FlagOptional(.flags, .dynamic_linker, String), + const VersionStorage = union(OsVersion) { + none: void, + semver: String, + windows: std.Target.Os.WindowsVersion, + default: void, + }; + pub const Index = enum(u32) { _, @@ -1629,6 +1634,7 @@ pub const TargetQuery = struct { pub const Storage = enum { flag_optional, + enum_optional, extended, /// The presence of the field is determined by a boolean within a packed @@ -1641,9 +1647,27 @@ pub const Storage = enum { return struct { value: ?Value, - pub const flags = flags_arg; - pub const flag = flag_arg; pub const storage: Storage = .flag_optional; + pub const flags = flags_arg; + pub const flag = flag_arg; + pub const Value = ValueArg; + }; + } + + /// The field is present if an enum tag from flags matches a specific value. + pub fn EnumOptional( + comptime flags_arg: @EnumLiteral(), + comptime flag_arg: @EnumLiteral(), + comptime tag_arg: @EnumLiteral(), + comptime ValueArg: type, + ) type { + return struct { + value: ?Value, + + pub const storage: Storage = .enum_optional; + pub const flags = flags_arg; + pub const flag = flag_arg; + pub const tag = tag_arg; pub const Value = ValueArg; }; } @@ -1736,6 +1760,14 @@ pub const Storage = enum { .value = if (flag) dataField(buffer, i, container, Field.Value) else null, }; }, + .enum_optional => { + const flags = @field(container, @tagName(Field.flags)); + const tag = @field(flags, @tagName(Field.flag)); + const match = tag == Field.tag; + return .{ + .value = if (match) dataField(buffer, i, container, Field.Value) else null, + }; + }, .extended => @compileError("TODO"), }, }, @@ -1800,7 +1832,7 @@ pub const Storage = enum { return casted.len; }, else => switch (Field.storage) { - .flag_optional => { + .flag_optional, .enum_optional => { return if (value.value) |v| setExtraField(buffer, i, Field.Value, v) else 0; }, .extended => @compileError("TODO"), @@ -1827,7 +1859,7 @@ pub const Storage = enum { else => comptime unreachable, }, .auto => switch (Field.storage) { - .flag_optional, .extended => 1, + .flag_optional, .enum_optional, .extended => 1, }, .@"extern" => comptime unreachable, }, -- 2.54.0 From 92803903f3e96995cc705087615bd67749757862 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 24 Feb 2026 19:15:30 -0800 Subject: [PATCH 027/179] Configuration: implement FlagLengthPrefixedList --- lib/compiler/Maker/ScannedConfig.zig | 21 ++- lib/compiler/configure_runner.zig | 52 ++++++- lib/std/zig/Configuration.zig | 215 +++++++++++++++++++-------- 3 files changed, 215 insertions(+), 73 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index f26b5d95dd63d40e5341a060df3baf4e01dc1fa3..51a4894195f47cc2dfad0da4414d0d48620c90f2 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -55,17 +55,24 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try s.value(field_value.slice(c), .{}); }, Configuration.Deps => { - var deps_field = try s.beginTuple(.{}); - for (field_value.slice(c)) |dep| { - try deps_field.field(@intFromEnum(dep), .{}); - } - try deps_field.end(); + try printValue(sc, s, []Configuration.Step.Index, field_value.slice(c)); }, Configuration.MaxRss => { try s.value(field_value.toBytes(), .{}); }, else => switch (@typeInfo(Field)) { .int => try s.int(field_value), + .pointer => |info| switch (info.size) { + .slice => { + var slice_field = try s.beginTuple(.{}); + for (field_value) |elem| { + try slice_field.fieldPrefix(); + try printValue(sc, s, info.child, elem); + } + try slice_field.end(); + }, + else => comptime unreachable, + }, .@"enum" => { if (@hasDecl(Field, "storage")) switch (Field.storage) { .extended => { @@ -74,6 +81,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try sub_struct.end(); }, .flag_optional => comptime unreachable, + .flag_length_prefixed_list => comptime unreachable, .enum_optional => comptime unreachable, } else if (std.enums.tagName(Field, field_value)) |name| { try s.ident(name); @@ -93,6 +101,9 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try s.value(null, .{}); } }, + .flag_length_prefixed_list => { + try printValue(sc, s, @TypeOf(field_value.slice), field_value.slice); + }, .extended => @compileError("TODO"), }, else => @compileError("not implemented: " ++ @typeName(Field)), diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 9bc76da59ada922e37e3e626a3548ffd1a9018d1..014cfd9539bdf51cc1b1ac505e214f0176f1e212 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -279,6 +279,10 @@ const Serialize = struct { return (try addOptionalLazyPathEnum(s, lp)).unwrap(); } + fn addLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.LazyPath { + return @enumFromInt(@intFromEnum(try addOptionalLazyPathEnum(s, lp))); + } + fn addOptionalSemVer(s: *Serialize, sem_ver: ?std.SemanticVersion) !?Configuration.String { return if (sem_ver) |sv| try s.wc.addSemVer(sv) else null; } @@ -286,6 +290,20 @@ const Serialize = struct { fn addOptionalString(s: *Serialize, opt_slice: ?[]const u8) !?Configuration.String { return if (opt_slice) |slice| try s.wc.addString(slice) else null; } + + fn initStringList(s: *Serialize, list: []const []const u8) ![]const Configuration.String { + const wc = s.wc; + const result = try s.arena.alloc(Configuration.String, list.len); + for (result, list) |*dest, src| dest.* = try wc.addString(src); + return result; + } + + fn initOptionalStringList(s: *Serialize, list: []const ?[]const u8) ![]const Configuration.OptionalString { + const wc = s.wc; + const result = try s.arena.alloc(Configuration.OptionalString, list.len); + for (result, list) |*dest, src| dest.* = try wc.addOptionalString(src); + return result; + } }; fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { @@ -320,7 +338,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { // Add and then de-duplicate dependencies. const deps = d: { const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len); - for (try wc.prepareDeps(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| + for (try wc.reserveLengthPrefixed(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| dep.* = @intCast(step_map.getIndex(dep_step).?); break :d try wc.dedupeDeps(deps); }; @@ -340,11 +358,35 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .compile => e: { const c: *Step.Compile = @fieldParentPtr("step", step); + const exec_cmd_args: []const ?[]const u8 = c.exec_cmd_args orelse &.{}; + const installed_headers: []u32 = try arena.alloc(u32, c.installed_headers.items.len); + for (installed_headers, c.installed_headers.items) |*dst, src| switch (src) { + .file => |file| { + dst.* = try wc.addExtra(@as(Configuration.Step.Compile.InstalledHeader.File, .{ + .source = try s.addLazyPath(file.source), + .dest_sub_path = try wc.addString(file.dest_rel_path), + })); + }, + .directory => |directory| { + const include_extensions = directory.options.include_extensions orelse &.{}; + dst.* = try wc.addExtra(@as(Configuration.Step.Compile.InstalledHeader.Directory, .{ + .flags = .{ + .include_extensions = include_extensions.len != 0, + .exclude_extensions = directory.options.exclude_extensions.len != 0, + }, + .source = try s.addLazyPath(directory.source), + .dest_sub_path = try wc.addString(directory.dest_rel_path), + .exclude_extensions = .{ .slice = try s.initStringList(directory.options.exclude_extensions) }, + .include_extensions = .{ .slice = try s.initStringList(include_extensions) }, + })); + }, + }; + const extra_index = try wc.addExtra(@as(Configuration.Step.Compile, .{ .flags = .{ .filters_len = c.filters.len != 0, - .exec_cmd_args_len = if (c.exec_cmd_args) |a| a.len != 0 else false, - .installed_headers_len = c.installed_headers.items.len != 0, + .exec_cmd_args_len = exec_cmd_args.len != 0, + .installed_headers_len = installed_headers.len != 0, .force_undefined_symbols_len = c.force_undefined_symbols.entries.len != 0, .verbose_link = c.verbose_link, @@ -466,6 +508,10 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .hexstring => |*hexstring| try wc.addString(hexstring.toSlice()), .none, .fast, .uuid, .sha1, .md5 => null, } else null }, + .filters = .{ .slice = try s.initStringList(c.filters) }, + .exec_cmd_args = .{ .slice = try s.initOptionalStringList(exec_cmd_args) }, + .installed_headers = .initErased(installed_headers), + .force_undefined_symbols = .{ .slice = try s.initStringList(c.force_undefined_symbols.keys()) }, })); log.err("TODO serialize the trailing Compile step data", .{}); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 6cb11453518fc84d3d6dc74ef72e07c1eb6b2ec9..069154a34e2e287c86ec78406ad77d0a4162dad9 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -33,7 +33,9 @@ pub const Header = extern struct { pub const Wip = struct { gpa: Allocator, string_table: StringTable = .empty, - deps_table: DepsTable = .empty, + /// De-duplicates an array inside `extra` that has first element length + /// followed by length elements. + length_prefixed_table: LengthPrefixedTable = .empty, targets_table: TargetsTable = .empty, string_bytes: std.ArrayList(u8) = .empty, @@ -44,23 +46,23 @@ pub const Wip = struct { path_deps: std.MultiArrayList(Path) = .empty, extra: std.ArrayList(u32) = .empty, - const DepsTable = std.HashMapUnmanaged(Deps, void, DepsTableContext, std.hash_map.default_max_load_percentage); + const LengthPrefixedTable = std.HashMapUnmanaged(u32, void, LengthPrefixedContext, std.hash_map.default_max_load_percentage); const TargetsTable = std.HashMapUnmanaged(TargetQuery.Index, void, TargetsTableContext, std.hash_map.default_max_load_percentage); - const DepsTableContext = struct { + const LengthPrefixedContext = struct { extra: []const u32, - pub fn eql(ctx: @This(), a: Deps, b: Deps) bool { - const len_a = ctx.extra[@intFromEnum(a)]; - const len_b = ctx.extra[@intFromEnum(b)]; - const slice_a = ctx.extra[@intFromEnum(a) + 1 ..][0..len_a]; - const slice_b = ctx.extra[@intFromEnum(b) + 1 ..][0..len_b]; + pub fn eql(ctx: @This(), a: u32, b: u32) bool { + const len_a = ctx.extra[a]; + const len_b = ctx.extra[b]; + const slice_a = ctx.extra[a + 1 ..][0..len_a]; + const slice_b = ctx.extra[b + 1 ..][0..len_b]; return std.mem.eql(u32, slice_a, slice_b); } - pub fn hash(ctx: @This(), key: Deps) u64 { - const len = ctx.extra[@intFromEnum(key)]; - const slice = ctx.extra[@intFromEnum(key) + 1 ..][0..len]; + pub fn hash(ctx: @This(), key: u32) u64 { + const len = ctx.extra[key]; + const slice = ctx.extra[key + 1 ..][0..len]; return std.hash_map.hashString(@ptrCast(slice)); } }; @@ -174,6 +176,10 @@ pub const Wip = struct { return new_off; } + pub fn addOptionalString(wip: *Wip, bytes: ?[]const u8) Allocator.Error!OptionalString { + return .init(try addString(wip, bytes orelse return .none)); + } + pub fn addSemVer(wip: *Wip, sv: std.SemanticVersion) Allocator.Error!String { var buffer: [256]u8 = undefined; var writer: std.Io.Writer = .fixed(&buffer); @@ -324,27 +330,32 @@ pub const Wip = struct { } } - pub fn prepareDeps(wip: *Wip, n: usize) Allocator.Error![]u32 { + pub fn reserveLengthPrefixed(wip: *Wip, n: usize) Allocator.Error![]u32 { const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1); slice[0] = @intCast(n); return slice[1..]; } + pub fn dedupeLengthPrefixed(wip: *Wip, index: u32) Allocator.Error!u32 { + assert(wip.extra.items.len == index + wip.extra.items[index] + 1); + const gpa = wip.gpa; + const gop = try wip.length_prefixed_table.getOrPutContext(gpa, index, @as(LengthPrefixedContext, .{ + .extra = wip.extra.items, + })); + if (gop.found_existing) { + wip.extra.items.len = index; + return gop.key_ptr.*; + } else { + return index; + } + } + pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps { - const gpa = wip.gpa; - const gop = try wip.deps_table.getOrPutContext(gpa, deps, @as(DepsTableContext, .{ - .extra = wip.extra.items, - })); - if (gop.found_existing) { - wip.extra.items.len = @intFromEnum(deps); - return gop.key_ptr.*; - } else { - return deps; - } + return @enumFromInt(try dedupeLengthPrefixed(wip, @intFromEnum(deps))); } pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { - const extra_len = Storage.calculateExtraLenUpperBound(@TypeOf(extra)); + const extra_len = Storage.extraLen(extra); try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len); return addExtraAssumeCapacity(wip, extra); } @@ -397,7 +408,7 @@ pub const Step = extern struct { owner: Package.Index, deps: Deps, max_rss: MaxRss, - extended: Storage.ExtendedIndex(Flags, union(Tag) { + extended: Storage.Extended(Flags, union(Tag) { check_file: CheckFile, check_object: CheckObject, compile: Compile, @@ -585,10 +596,10 @@ pub const Step = extern struct { root_module: Module.Index, root_name: String, - //filters: FlagLengthPrefixedList(.flags, .filters_len, String), - //exec_cmd_args: FlagLengthPrefixedList(.flags, .exec_cmd_args_len, u32), - //installed_headers: FlagLengthPrefixedList(.flags, .installed_headers_len, InstalledHeader), - //force_undefined_symbols: FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), + filters: Storage.FlagLengthPrefixedList(.flags, .filters_len, String), + exec_cmd_args: Storage.FlagLengthPrefixedList(.flags, .exec_cmd_args_len, OptionalString), + installed_headers: Storage.FlagLengthPrefixedList(.flags, .installed_headers_len, Storage.Extended(InstalledHeader.Flags, InstalledHeader)), + force_undefined_symbols: Storage.FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), //exacts: EnumConditionalPrefixedList(.flags4, .expect_errors, .exact, String), linker_script: Storage.FlagOptional(.flags4, .linker_script, LazyPath), version_script: Storage.FlagOptional(.flags4, .version_script, LazyPath), @@ -612,6 +623,46 @@ pub const Step = extern struct { error_limit: Storage.FlagOptional(.flags4, .error_limit, u32), build_id: Storage.EnumOptional(.flags3, .build_id, .hexstring, String), + pub const InstalledHeader = union(@This().Tag) { + file: File, + directory: Directory, + + pub const Flags = packed struct(u32) { + tag: InstalledHeader.Tag, + _: u24 = 0, + }; + + pub const Tag = enum(u8) { + file, + directory, + }; + + pub const File = struct { + flags: @This().Flags = .{}, + source: LazyPath, + dest_sub_path: String, + + pub const Flags = packed struct(u32) { + tag: InstalledHeader.Tag = .file, + _: u24 = 0, + }; + }; + + pub const Directory = struct { + flags: @This().Flags, + source: LazyPath, + dest_sub_path: String, + exclude_extensions: Storage.FlagLengthPrefixedList(.flags, .exclude_extensions, String), + include_extensions: Storage.FlagLengthPrefixedList(.flags, .include_extensions, String), + + pub const Flags = packed struct(u32) { + tag: InstalledHeader.Tag = .directory, + exclude_extensions: bool, + include_extensions: bool, + _: u22 = 0, + }; + }; + }; pub const ExpectErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; pub const TestRunnerMode = enum(u2) { default, simple, server }; pub const Entry = enum(u2) { default, disabled, enabled, symbol_name }; @@ -1636,6 +1687,7 @@ pub const Storage = enum { flag_optional, enum_optional, extended, + flag_length_prefixed_list, /// The presence of the field is determined by a boolean within a packed /// struct. @@ -1674,16 +1726,7 @@ pub const Storage = enum { /// The field indexes into an auxilary buffer, with the first element being /// a packed struct that contains the tag. - pub fn Extended(comptime U: type) type { - return struct { - value: U, - - pub const storage: Storage = .extended; - }; - } - - /// Equivalent to `Extended` but works in an `extern struct`. - pub fn ExtendedIndex(comptime BaseFlags: type, comptime U: type) type { + pub fn Extended(comptime BaseFlags: type, comptime U: type) type { return enum(u32) { _, @@ -1699,6 +1742,30 @@ pub const Storage = enum { }; } + /// A field in flags determines whether the length is zero or nonzero. If the length is + /// nonzero, then there is a length field followed by the list. + /// + /// When deserializing, the slice field is set. When serializing, the index + /// field must be set. + pub fn FlagLengthPrefixedList( + comptime flags_arg: @EnumLiteral(), + comptime flag_arg: @EnumLiteral(), + comptime ValueArg: type, + ) type { + return struct { + slice: []const Value, + + pub const storage: Storage = .flag_length_prefixed_list; + pub const flags = flags_arg; + pub const flag = flag_arg; + pub const Value = ValueArg; + + pub fn initErased(s: []const u32) @This() { + return .{ .slice = @ptrCast(s) }; + } + }; + } + pub fn dataLength(buffer: []const u32, i: usize, comptime S: type) usize { var end = i; _ = data(buffer, &end, S); @@ -1769,6 +1836,15 @@ pub const Storage = enum { }; }, .extended => @compileError("TODO"), + .flag_length_prefixed_list => { + const flags = @field(container, @tagName(Field.flags)); + const flag = @field(flags, @tagName(Field.flag)); + if (!flag) return .{ .slice = &.{} }; + const data_start = i.* + 1; + const len = buffer[data_start - 1]; + defer i.* = data_start + len; + return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; + }, }, }, .@"extern" => comptime unreachable, @@ -1787,11 +1863,36 @@ pub const Storage = enum { return i; } - fn calculateExtraLenUpperBound(comptime Extra: type) comptime_int { - var i = 0; - const fields = @typeInfo(Extra).@"struct".fields; + fn extraFieldLen(field: anytype) usize { + const Field = @TypeOf(field); + return switch (@typeInfo(Field)) { + .int => |info| switch (info.bits) { + 32 => 1, + 64 => 2, + else => comptime unreachable, + }, + .@"enum" => 1, + .@"struct" => |info| switch (info.layout) { + .@"packed" => switch (info.backing_integer.?) { + u32 => 1, + u64 => 2, + else => comptime unreachable, + }, + .auto => switch (Field.storage) { + .flag_optional, .enum_optional, .extended => 1, + .flag_length_prefixed_list => field.slice.len + 1, + }, + .@"extern" => comptime unreachable, + }, + else => @compileError("bad type: " ++ @typeName(Field)), + }; + } + + fn extraLen(extra: anytype) usize { + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + var i: usize = 0; inline for (fields) |field| { - i += calculateExtraFieldLenUpperBound(field.type); + i += Storage.extraFieldLen(@field(extra, field.name)); } return i; } @@ -1836,6 +1937,13 @@ pub const Storage = enum { return if (value.value) |v| setExtraField(buffer, i, Field.Value, v) else 0; }, .extended => @compileError("TODO"), + .flag_length_prefixed_list => { + const len: u32 = @intCast(value.slice.len); + if (len == 0) return 0; + buffer[i] = len; + @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); + return len + 1; + }, }, }, .@"extern" => comptime unreachable, @@ -1843,29 +1951,6 @@ pub const Storage = enum { else => @compileError("bad field type: " ++ @typeName(Field)), } } - - fn calculateExtraFieldLenUpperBound(comptime Field: type) comptime_int { - return switch (@typeInfo(Field)) { - .int => |info| switch (info.bits) { - 32 => 1, - 64 => 2, - else => comptime unreachable, - }, - .@"enum" => 1, - .@"struct" => |info| switch (info.layout) { - .@"packed" => switch (info.backing_integer.?) { - u32 => 1, - u64 => 2, - else => comptime unreachable, - }, - .auto => switch (Field.storage) { - .flag_optional, .enum_optional, .extended => 1, - }, - .@"extern" => comptime unreachable, - }, - else => comptime unreachable, - }; - } }; pub fn extraData(c: *const Configuration, comptime T: type, index: usize) T { -- 2.54.0 From a180012dd2bdae6dd30e165e6c74923eeeeb95b7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 24 Feb 2026 19:41:27 -0800 Subject: [PATCH 028/179] configurer: serialize 3 more Module fields --- lib/compiler/configure_runner.zig | 153 ++++++++++++++++-------------- lib/std/zig/Configuration.zig | 11 ++- 2 files changed, 89 insertions(+), 75 deletions(-) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index 014cfd9539bdf51cc1b1ac505e214f0176f1e212..8841aaf553ee46dcd82dbc91a770e9f6a1cb3733 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -279,7 +279,7 @@ const Serialize = struct { return (try addOptionalLazyPathEnum(s, lp)).unwrap(); } - fn addLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.LazyPath { + fn addLazyPath(s: *Serialize, lp: std.Build.LazyPath) !Configuration.LazyPath { return @enumFromInt(@intFromEnum(try addOptionalLazyPathEnum(s, lp))); } @@ -304,6 +304,86 @@ const Serialize = struct { for (result, list) |*dest, src| dest.* = try wc.addOptionalString(src); return result; } + + fn addModule(s: *Serialize, m: *std.Build.Module) !Configuration.Module.Index { + if (s.module_map.get(m)) |index| return index; + + const wc = s.wc; + const arena = s.arena; + const gpa = wc.gpa; + + const lib_paths = try arena.alloc(Configuration.LazyPath, m.lib_paths.items.len); + for (lib_paths, m.lib_paths.items) |*dest, src| dest.* = try addLazyPath(s, src); + + const c_macros = try initStringList(s, m.c_macros.items); + const export_symbol_names = try initStringList(s, m.export_symbol_names); + + const import_table: Configuration.ImportTable = @enumFromInt(wc.extra.items.len); + const import_table_extra_len = 1 + 2 * m.import_table.entries.len; + try wc.extra.ensureUnusedCapacity(gpa, import_table_extra_len); + wc.extra.items.len += import_table_extra_len; + wc.extra.appendAssumeCapacity(@intCast(m.import_table.entries.len)); + wc.extra.items[@intFromEnum(import_table)] = @intCast(m.import_table.entries.len); + for ( + m.import_table.keys(), + @intFromEnum(import_table) + 1.., + ) |mod_name, extra_index| { + wc.extra.items[extra_index] = @intFromEnum(try wc.addString(mod_name)); + } + for ( + m.import_table.values(), + @intFromEnum(import_table) + 1 + m.import_table.entries.len.., + ) |dep, extra_index| { + log.err("TODO module dependencies can be cyclic", .{}); + wc.extra.items[extra_index] = @intFromEnum(try addModule(s, dep)); + } + + const module_index: Configuration.Module.Index = @enumFromInt(try wc.addExtra(@as(Configuration.Module, .{ + .flags = .{ + .optimize = .init(m.optimize), + .strip = .init(m.strip), + .unwind_tables = .init(m.unwind_tables), + .dwarf_format = .init(m.dwarf_format), + .single_threaded = .init(m.strip), + .stack_protector = .init(m.strip), + .stack_check = .init(m.strip), + .sanitize_c = .init(m.sanitize_c), + .sanitize_thread = .init(m.strip), + .fuzz = .init(m.strip), + .code_model = m.code_model, + .c_macros = c_macros.len != 0, + .include_dirs = m.include_dirs.items.len != 0, + .lib_paths = lib_paths.len != 0, + .rpaths = m.rpaths.items.len != 0, + .frameworks = m.frameworks.entries.len != 0, + .link_objects = m.link_objects.items.len != 0, + .export_symbol_names = export_symbol_names.len != 0, + }, + .flags2 = .{ + .valgrind = .init(m.strip), + .pic = .init(m.strip), + .red_zone = .init(m.strip), + .omit_frame_pointer = .init(m.strip), + .error_tracing = .init(m.strip), + .link_libc = .init(m.strip), + .link_libcpp = .init(m.strip), + .no_builtin = .init(m.strip), + }, + .owner = try s.builderToPackage(m.owner), + .root_source_file = try s.addOptionalLazyPathEnum(m.root_source_file), + .import_table = import_table, + .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target), + .c_macros = .{ .slice = c_macros }, + .lib_paths = .{ .slice = lib_paths }, + .export_symbol_names = .{ .slice = export_symbol_names }, + }))); + + log.err("TODO serialize the trailing Module data", .{}); + + try s.module_map.putNoClobber(arena, m, module_index); + + return module_index; + } }; fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { @@ -479,7 +559,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .linker_script = c.linker_script != null, .version_script = c.version_script != null, }, - .root_module = try addModule(&s, c.root_module), + .root_module = try s.addModule(c.root_module), .root_name = try wc.addString(c.name), .linker_script = .{ .value = try s.addOptionalLazyPath(c.linker_script) }, .version_script = .{ .value = try s.addOptionalLazyPath(c.version_script) }, @@ -612,75 +692,6 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }); } -fn addModule(s: *Serialize, m: *std.Build.Module) !Configuration.Module.Index { - if (s.module_map.get(m)) |index| return index; - - const wc = s.wc; - const arena = s.arena; - const gpa = wc.gpa; - const import_table: Configuration.ImportTable = @enumFromInt(wc.extra.items.len); - const import_table_extra_len = 1 + 2 * m.import_table.entries.len; - try wc.extra.ensureUnusedCapacity(gpa, import_table_extra_len); - wc.extra.items.len += import_table_extra_len; - wc.extra.appendAssumeCapacity(@intCast(m.import_table.entries.len)); - wc.extra.items[@intFromEnum(import_table)] = @intCast(m.import_table.entries.len); - for ( - m.import_table.keys(), - @intFromEnum(import_table) + 1.., - ) |mod_name, extra_index| { - wc.extra.items[extra_index] = @intFromEnum(try wc.addString(mod_name)); - } - for ( - m.import_table.values(), - @intFromEnum(import_table) + 1 + m.import_table.entries.len.., - ) |dep, extra_index| { - log.err("TODO module dependencies can be cyclic", .{}); - wc.extra.items[extra_index] = @intFromEnum(try addModule(s, dep)); - } - - const module_index: Configuration.Module.Index = @enumFromInt(try wc.addExtra(@as(Configuration.Module, .{ - .flags = .{ - .optimize = .init(m.optimize), - .strip = .init(m.strip), - .unwind_tables = .init(m.unwind_tables), - .dwarf_format = .init(m.dwarf_format), - .single_threaded = .init(m.strip), - .stack_protector = .init(m.strip), - .stack_check = .init(m.strip), - .sanitize_c = .init(m.sanitize_c), - .sanitize_thread = .init(m.strip), - .fuzz = .init(m.strip), - .code_model = m.code_model, - .c_macros = m.c_macros.items.len != 0, - .include_dirs = m.include_dirs.items.len != 0, - .lib_paths = m.lib_paths.items.len != 0, - .rpaths = m.rpaths.items.len != 0, - .frameworks = m.frameworks.entries.len != 0, - .link_objects = m.link_objects.items.len != 0, - .export_symbol_names = m.export_symbol_names.len != 0, - - .valgrind = .init(m.strip), - .pic = .init(m.strip), - .red_zone = .init(m.strip), - .omit_frame_pointer = .init(m.strip), - .error_tracing = .init(m.strip), - .link_libc = .init(m.strip), - .link_libcpp = .init(m.strip), - .no_builtin = .init(m.strip), - }, - .owner = try s.builderToPackage(m.owner), - .root_source_file = try s.addOptionalLazyPathEnum(m.root_source_file), - .import_table = import_table, - .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target), - }))); - - log.err("TODO serialize the trailing Module data", .{}); - - try s.module_map.putNoClobber(arena, m, module_index); - - return module_index; -} - fn addOptionalResolvedTarget( wc: *Configuration.Wip, optional_resolved_target: ?std.Build.ResolvedTarget, diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 069154a34e2e287c86ec78406ad77d0a4162dad9..41adeb088e91171915828f0c8148b6a5e680ae79 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -1070,19 +1070,20 @@ pub const Package = struct { }; /// Trailing: -/// * c_macros: LengthPrefixedList(String), // if flag is set -/// * lib_paths: LengthPrefixedList(LazyPath), // if flag is set -/// * export_symbol_names: LengthPrefixedList(String), // if flag is set /// * frameworks: FlagsPrefixedList(FrameworkFlags), // if flag is set /// * include_dirs: UnionList(IncludeDir), // if flag is set /// * rpaths: UnionList(RPath), // if flag is set /// * link_objects: UnionList(LinkObject), // if flag is set pub const Module = struct { flags: Flags, + flags2: Flags2, owner: Package.Index, root_source_file: OptionalLazyPath, import_table: ImportTable, resolved_target: ResolvedTarget.OptionalIndex, + c_macros: Storage.FlagLengthPrefixedList(.flags, .c_macros, String), + lib_paths: Storage.FlagLengthPrefixedList(.flags, .lib_paths, LazyPath), + export_symbol_names: Storage.FlagLengthPrefixedList(.flags, .export_symbol_names, String), pub const Optimize = enum(u3) { debug, @@ -1148,7 +1149,7 @@ pub const Module = struct { _, }; - pub const Flags = packed struct(u64) { + pub const Flags = packed struct(u32) { optimize: Optimize, strip: DefaultingBool, unwind_tables: UnwindTables, @@ -1167,7 +1168,9 @@ pub const Module = struct { frameworks: bool, link_objects: bool, export_symbol_names: bool, + }; + pub const Flags2 = packed struct(u32) { valgrind: DefaultingBool, pic: DefaultingBool, red_zone: DefaultingBool, -- 2.54.0 From b04818644c958e05de9b0d5fb4a8a2a3da6d2164 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 24 Feb 2026 19:42:49 -0800 Subject: [PATCH 029/179] rename configure_runner to configurer --- lib/compiler/{configure_runner.zig => configurer.zig} | 0 src/main.zig | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/compiler/{configure_runner.zig => configurer.zig} (100%) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configurer.zig similarity index 100% rename from lib/compiler/configure_runner.zig rename to lib/compiler/configurer.zig diff --git a/src/main.zig b/src/main.zig index 1366deef2bc3646540643588915af21a1c571008..2e6da4b02d72b3aad66d90b9a31059ba0308774c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5333,7 +5333,7 @@ fn cmdBuild( .root_src_path = fs.path.basename(runner), } else .{ .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), - .root_src_path = "configure_runner.zig", + .root_src_path = "configurer.zig", }; const config = try Compilation.Config.resolve(.{ -- 2.54.0 From 6925a57d2fd4e4b1330d980cbfb84f7007f117ef Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 26 Feb 2026 18:44:23 -0800 Subject: [PATCH 030/179] Configuration: implement UnionList storage --- lib/compiler/Maker/ScannedConfig.zig | 3 + lib/compiler/configurer.zig | 120 +++++++++++--- lib/std/Build/Module.zig | 14 +- lib/std/lang.zig | 2 +- lib/std/zig/Configuration.zig | 228 +++++++++++++++++++++++---- 5 files changed, 301 insertions(+), 66 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 51a4894195f47cc2dfad0da4414d0d48620c90f2..07c92f295ae4d64219c82712e530cd84920c903e 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -15,6 +15,7 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { std.log.err("TODO also print unlazy deps", .{}); std.log.err("TODO also print system integrations", .{}); std.log.err("TODO also print available options", .{}); + std.log.err("TODO also print modules", .{}); const c = &sc.configuration; var serializer: Serializer = .{ .writer = w }; var s = try serializer.beginStruct(.{}); @@ -83,6 +84,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi .flag_optional => comptime unreachable, .flag_length_prefixed_list => comptime unreachable, .enum_optional => comptime unreachable, + .union_list => comptime unreachable, } else if (std.enums.tagName(Field, field_value)) |name| { try s.ident(name); } else { @@ -105,6 +107,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try printValue(sc, s, @TypeOf(field_value.slice), field_value.slice); }, .extended => @compileError("TODO"), + .union_list => @compileError("TODO"), }, else => @compileError("not implemented: " ++ @typeName(Field)), }, diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 8841aaf553ee46dcd82dbc91a770e9f6a1cb3733..beb8c7df2630baf1f931b8476f8dddc04f1559ae 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -224,6 +224,8 @@ const Serialize = struct { wc: *Configuration.Wip, module_map: std.AutoArrayHashMapUnmanaged(*std.Build.Module, Configuration.Module.Index) = .empty, package_map: std.AutoArrayHashMapUnmanaged(*std.Build, Configuration.Package.Index) = .empty, + /// Index corresponds to `Configuration.steps` index. + step_map: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty, fn builderToPackage(s: *Serialize, b: *std.Build) !Configuration.Package.Index { if (b.pkg_hash.len == 0) return .root; @@ -291,6 +293,56 @@ const Serialize = struct { return if (opt_slice) |slice| try s.wc.addString(slice) else null; } + fn addSystemLib(s: *Serialize, sl: *const std.Build.Module.SystemLib) !Configuration.SystemLib.Index { + log.err("TODO deduplicate addSystemLib", .{}); + const wc = s.wc; + return @enumFromInt(try wc.addExtra(@as(Configuration.SystemLib, .{ + .flags = .{ + .needed = sl.needed, + .weak = sl.weak, + .use_pkg_config = sl.use_pkg_config, + .preferred_link_mode = sl.preferred_link_mode, + .search_strategy = sl.search_strategy, + }, + .name = try wc.addString(sl.name), + }))); + } + + fn addCSourceFile(s: *Serialize, csf: *const std.Build.Module.CSourceFile) !Configuration.CSourceFile.Index { + log.err("TODO addCSourceFile trailing data", .{}); + const wc = s.wc; + return @enumFromInt(try wc.addExtra(@as(Configuration.CSourceFile, .{ + .flags = .{ + .args_len = @intCast(csf.flags.len), + .lang = .init(csf.language), + }, + .file = try addLazyPath(s, csf.file), + }))); + } + + fn addCSourceFiles(s: *Serialize, csf: *const std.Build.Module.CSourceFiles) !Configuration.CSourceFiles.Index { + log.err("TODO addCSourceFiles trailing data", .{}); + const wc = s.wc; + return @enumFromInt(try wc.addExtra(@as(Configuration.CSourceFiles, .{ + .flags = .{ + .args_len = @intCast(csf.flags.len), + .lang = .init(csf.language), + }, + .root = try addLazyPath(s, csf.root), + .files_len = @intCast(csf.files.len), + }))); + } + + fn addRcSourceFile(s: *Serialize, rsf: *const std.Build.Module.RcSourceFile) !Configuration.RcSourceFile.Index { + log.err("TODO addRcSourceFile trailing data", .{}); + const wc = s.wc; + return @enumFromInt(try wc.addExtra(@as(Configuration.RcSourceFile, .{ + .file = try addLazyPath(s, rsf.file), + .args_len = @intCast(rsf.flags.len), + .include_paths_len = @intCast(rsf.include_paths.len), + }))); + } + fn initStringList(s: *Serialize, list: []const []const u8) ![]const Configuration.String { const wc = s.wc; const result = try s.arena.alloc(Configuration.String, list.len); @@ -312,6 +364,35 @@ const Serialize = struct { const arena = s.arena; const gpa = wc.gpa; + const include_dirs = try arena.alloc(Configuration.Module.IncludeDir, m.include_dirs.items.len); + for (include_dirs, m.include_dirs.items) |*dest, src| dest.* = switch (src) { + .path => |lp| .{ .path = try addLazyPath(s, lp) }, + .path_system => |lp| .{ .path_system = try addLazyPath(s, lp) }, + .path_after => |lp| .{ .path_after = try addLazyPath(s, lp) }, + .framework_path => |lp| .{ .framework_path = try addLazyPath(s, lp) }, + .framework_path_system => |lp| .{ .framework_path_system = try addLazyPath(s, lp) }, + .embed_path => |lp| .{ .embed_path = try addLazyPath(s, lp) }, + .other_step => |cs| .{ .other_step = stepIndex(s, &cs.step) }, + .config_header_step => |chs| .{ .config_header_step = stepIndex(s, &chs.step) }, + }; + + const rpaths = try arena.alloc(Configuration.Module.RPath, m.rpaths.items.len); + for (rpaths, m.rpaths.items) |*dest, src| dest.* = switch (src) { + .lazy_path => |lp| .{ .lazy_path = try addLazyPath(s, lp) }, + .special => |slice| .{ .special = try wc.addString(slice) }, + }; + + const link_objects = try arena.alloc(Configuration.Module.LinkObject, m.link_objects.items.len); + for (link_objects, m.link_objects.items) |*dest, *src| dest.* = switch (src.*) { + .static_path => |lp| .{ .static_path = try addLazyPath(s, lp) }, + .other_step => |cs| .{ .other_step = stepIndex(s, &cs.step) }, + .system_lib => |*sl| .{ .system_lib = try addSystemLib(s, sl) }, + .assembly_file => |lp| .{ .assembly_file = try addLazyPath(s, lp) }, + .c_source_file => |csf| .{ .c_source_file = try addCSourceFile(s, csf) }, + .c_source_files => |csf| .{ .c_source_files = try addCSourceFiles(s, csf) }, + .win32_resource_file => |wrf| .{ .win32_resource_file = try addRcSourceFile(s, wrf) }, + }; + const lib_paths = try arena.alloc(Configuration.LazyPath, m.lib_paths.items.len); for (lib_paths, m.lib_paths.items) |*dest, src| dest.* = try addLazyPath(s, src); @@ -352,11 +433,11 @@ const Serialize = struct { .fuzz = .init(m.strip), .code_model = m.code_model, .c_macros = c_macros.len != 0, - .include_dirs = m.include_dirs.items.len != 0, + .include_dirs = include_dirs.len != 0, .lib_paths = lib_paths.len != 0, - .rpaths = m.rpaths.items.len != 0, + .rpaths = rpaths.len != 0, .frameworks = m.frameworks.entries.len != 0, - .link_objects = m.link_objects.items.len != 0, + .link_objects = link_objects.len != 0, .export_symbol_names = export_symbol_names.len != 0, }, .flags2 = .{ @@ -376,6 +457,9 @@ const Serialize = struct { .c_macros = .{ .slice = c_macros }, .lib_paths = .{ .slice = lib_paths }, .export_symbol_names = .{ .slice = export_symbol_names }, + .include_dirs = .init(include_dirs), + .rpaths = .init(rpaths), + .link_objects = .init(link_objects), }))); log.err("TODO serialize the trailing Module data", .{}); @@ -384,6 +468,10 @@ const Serialize = struct { return module_index; } + + fn stepIndex(s: *const Serialize, step: *Step) Configuration.Step.Index { + return @enumFromInt(s.step_map.getIndex(step).?); + } }; fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { @@ -396,34 +484,32 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { // Starting from all top-level steps in `b`, traverse the entire step graph // and add all step dependencies implied by module graphs. const top_level_steps = b.top_level_steps.values(); - // Index corresponds to `Configuration.steps` index. - var step_map: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; - try step_map.ensureUnusedCapacity(arena, top_level_steps.len); + try s.step_map.ensureUnusedCapacity(arena, top_level_steps.len); for (top_level_steps) |tls| { - step_map.putAssumeCapacityNoClobber(&tls.step, {}); + s.step_map.putAssumeCapacityNoClobber(&tls.step, {}); } { - while (wc.steps.items.len < step_map.count()) { - const step = step_map.keys()[wc.steps.items.len]; + while (wc.steps.items.len < s.step_map.count()) { + const step = s.step_map.keys()[wc.steps.items.len]; // Set up any implied dependencies for this step. It's important that we do this first, so // that the loop below discovers steps implied by the module graph. try createModuleDependenciesForStep(step); - try step_map.ensureUnusedCapacity(arena, step.dependencies.items.len); + try s.step_map.ensureUnusedCapacity(arena, step.dependencies.items.len); for (step.dependencies.items) |other_step| { - step_map.putAssumeCapacity(other_step, {}); + s.step_map.putAssumeCapacity(other_step, {}); } // Add and then de-duplicate dependencies. const deps = d: { const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len); for (try wc.reserveLengthPrefixed(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| - dep.* = @intCast(step_map.getIndex(dep_step).?); + dep.* = @intCast(s.step_map.getIndex(dep_step).?); break :d try wc.dedupeDeps(deps); }; - try wc.steps.ensureTotalCapacity(gpa, step_map.entries.capacity); + try wc.steps.ensureTotalCapacity(gpa, s.step_map.entries.capacity); wc.steps.appendAssumeCapacity(.{ .name = try wc.addString(step.name), .owner = try s.builderToPackage(step.owner), @@ -613,7 +699,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .emitted_pdb = try s.addOptionalLazyPathEnum(ia.emitted_pdb), .h_dir = try addInstallDir(wc, ia.h_dir), .emitted_h = try s.addOptionalLazyPathEnum(ia.emitted_h), - .artifact = stepIndex(&step_map, &ia.artifact.step), + .artifact = s.stepIndex(&ia.artifact.step), }))); }, .install_file => @panic("TODO"), @@ -688,7 +774,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { } try wc.write(writer, .{ - .default_step = stepIndex(&step_map, b.default_step), + .default_step = s.stepIndex(b.default_step), }); } @@ -714,10 +800,6 @@ fn addInstallDir(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !Co } } -fn stepIndex(step_map: *const std.AutoArrayHashMapUnmanaged(*Step, void), step: *Step) Configuration.Step.Index { - return @enumFromInt(step_map.getIndex(step).?); -} - /// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which /// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { diff --git a/lib/std/Build/Module.zig b/lib/std/Build/Module.zig index 7b42c9a1100334ca4fcfbd28a3da5db1d772b591..dd3b6b0251aa02dce746ac68280bc7eab8a4a10b 100644 --- a/lib/std/Build/Module.zig +++ b/lib/std/Build/Module.zig @@ -73,18 +73,8 @@ pub const SystemLib = struct { preferred_link_mode: std.builtin.LinkMode, search_strategy: SystemLib.SearchStrategy, - pub const UsePkgConfig = enum { - /// Don't use pkg-config, just pass -lfoo where foo is name. - no, - /// Try to get information on how to link the library from pkg-config. - /// If that fails, fall back to passing -lfoo where foo is name. - yes, - /// Try to get information on how to link the library from pkg-config. - /// If that fails, error out. - force, - }; - - pub const SearchStrategy = enum { paths_first, mode_first, no_fallback }; + pub const UsePkgConfig = std.Build.Configuration.SystemLib.UsePkgConfig; + pub const SearchStrategy = std.Build.Configuration.SystemLib.SearchStrategy; }; pub const CSourceLanguage = enum { diff --git a/lib/std/lang.zig b/lib/std/lang.zig index b515a5180d62c534ce37f643608e2a93dfb74358..811e506e537e7eb5edd99adccb79071ecff0d677 100644 --- a/lib/std/lang.zig +++ b/lib/std/lang.zig @@ -873,7 +873,7 @@ pub const OutputMode = enum { /// This data structure is used by the Zig language code generation and /// therefore must be kept in sync with the compiler implementation. -pub const LinkMode = enum { +pub const LinkMode = enum(u1) { static, dynamic, }; diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 41adeb088e91171915828f0c8148b6a5e680ae79..8f6d03d1c5a53ac457fde9b037735e96c6f34f20 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -1071,9 +1071,6 @@ pub const Package = struct { /// Trailing: /// * frameworks: FlagsPrefixedList(FrameworkFlags), // if flag is set -/// * include_dirs: UnionList(IncludeDir), // if flag is set -/// * rpaths: UnionList(RPath), // if flag is set -/// * link_objects: UnionList(LinkObject), // if flag is set pub const Module = struct { flags: Flags, flags2: Flags2, @@ -1084,6 +1081,9 @@ pub const Module = struct { c_macros: Storage.FlagLengthPrefixedList(.flags, .c_macros, String), lib_paths: Storage.FlagLengthPrefixedList(.flags, .lib_paths, LazyPath), export_symbol_names: Storage.FlagLengthPrefixedList(.flags, .export_symbol_names, String), + include_dirs: Storage.UnionList(.flags, .include_dirs, IncludeDir), + rpaths: Storage.UnionList(.flags, .rpaths, RPath), + link_objects: Storage.UnionList(.flags, .link_objects, LinkObject), pub const Optimize = enum(u3) { debug, @@ -1204,7 +1204,7 @@ pub const Module = struct { static_path: LazyPath, /// Always `Step.Tag.compile`. other_step: Step.Index, - system_lib: SystemLib, + system_lib: SystemLib.Index, assembly_file: LazyPath, c_source_file: CSourceFile.Index, c_source_files: CSourceFiles.Index, @@ -1328,8 +1328,18 @@ pub const SystemLib = struct { _, }; - pub const UsePkgConfig = enum(u2) { no, yes, force }; - pub const LinkMode = enum { static, dynamic }; + pub const UsePkgConfig = enum(u2) { + /// Don't use pkg-config, just pass -lfoo where foo is name. + no, + /// Try to get information on how to link the library from pkg-config. + /// If that fails, fall back to passing -lfoo where foo is name. + yes, + /// Try to get information on how to link the library from pkg-config. + /// If that fails, error out. + force, + }; + + pub const LinkMode = std.builtin.LinkMode; pub const Flags = packed struct(u32) { needed: bool, @@ -1337,18 +1347,19 @@ pub const SystemLib = struct { use_pkg_config: UsePkgConfig, preferred_link_mode: LinkMode, search_strategy: SearchStrategy, + _: u25 = 0, }; pub const SearchStrategy = enum(u2) { paths_first, mode_first, no_fallback }; }; /// Trailing: -/// * flag: String, // for each flags_len +/// * arg: String, // for each args_len /// * sub_path: String, // for each files_len pub const CSourceFiles = struct { + flags: Flags, root: LazyPath, files_len: u32, - flags: Flags, pub const Index = enum(u32) { _, @@ -1356,16 +1367,16 @@ pub const CSourceFiles = struct { pub const Flags = packed struct(u32) { /// C compiler CLI flags. - flags_len: u29, + args_len: u29, lang: OptionalCSourceLanguage, }; }; /// Trailing: -/// * flag: String, // for each flags_len +/// * arg: String, // for each args_len pub const CSourceFile = struct { - file: LazyPath, flags: Flags, + file: LazyPath, pub const Index = enum(u32) { _, @@ -1373,11 +1384,24 @@ pub const CSourceFile = struct { pub const Flags = packed struct(u32) { /// C compiler CLI flags. - flags_len: u29, + args_len: u29, lang: OptionalCSourceLanguage, }; }; +/// Trailing: +/// * arg: String, // for each args_len +/// * include_path: String, // for each include_paths_len +pub const RcSourceFile = struct { + file: LazyPath, + args_len: u32, + include_paths_len: u32, + + pub const Index = enum(u32) { + _, + }; +}; + pub const OptionalCSourceLanguage = enum(u3) { c, cpp, @@ -1386,29 +1410,17 @@ pub const OptionalCSourceLanguage = enum(u3) { assembly, assembly_with_preprocessor, default, -}; -pub const RcSourceFile = struct { - file: LazyPath, - /// Any option that rc.exe accepts will work here, with the exception of: - /// - `/fo`: The output filename is set by the build system - /// - `/p`: Only running the preprocessor is not supported in this context - /// - `/:no-preprocess` (non-standard option): Not supported in this context - /// - Any MUI-related option - /// https://learn.microsoft.com/en-us/windows/win32/menurc/using-rc-the-rc-command-line- - /// - /// Implicitly defined options: - /// /x (ignore the INCLUDE environment variable) - /// /D_DEBUG or /DNDEBUG depending on the optimization mode - flags: []const []const u8 = &.{}, - /// Include paths that may or may not exist yet and therefore need to be - /// specified as a LazyPath. Each path will be appended to the flags - /// as `/I `. - include_paths: []const LazyPath = &.{}, - - pub const Index = enum(u32) { - _, - }; + pub fn init(x: ?std.Build.Module.CSourceLanguage) @This() { + return switch (x orelse return .default) { + .c => .c, + .cpp => .cpp, + .objective_c => .objective_c, + .objective_cpp => .objective_cpp, + .assembly => .assembly, + .assembly_with_preprocessor => .assembly_with_preprocessor, + }; + } }; pub const ResolvedTarget = struct { @@ -1691,6 +1703,7 @@ pub const Storage = enum { enum_optional, extended, flag_length_prefixed_list, + union_list, /// The presence of the field is determined by a boolean within a packed /// struct. @@ -1769,6 +1782,63 @@ pub const Storage = enum { }; } + /// `UnionArg` is a tagged union with a small integer for the enum tag. + /// + /// A field in flags determines whether the metadata is present. + /// + /// The metadata is bit-packed consecutive packed struct which is the + /// `UnionArg` enum tag combined with a "last" marker boolean field. + /// When "last" is true, the element is the last one, providing + /// the length of the list. + /// + /// Following is each element of the list; each bitcastable to u32. + pub fn UnionList( + comptime flags_arg: @EnumLiteral(), + comptime flag_arg: @EnumLiteral(), + comptime UnionArg: type, + ) type { + return struct { + /// When serializing it is UnionArg slice pointer. + /// When deserializing it is extra index of first UnionArg element. + data: ?*const anyopaque, + len: usize, + + pub const storage: Storage = .union_list; + pub const flags = flags_arg; + pub const flag = flag_arg; + pub const Union = UnionArg; + + pub const Tag = @typeInfo(Union).@"union".tag_type.?; + pub const MetaInt = @Int(.unsigned, @bitSizeOf(Tag) + 1); + pub const Meta = packed struct(MetaInt) { + tag: Tag, + last: bool, + }; + + /// Valid to call only when serializing. + pub fn init(slice: []const Union) @This() { + return .{ .data = slice.ptr, .len = slice.len }; + } + + /// Valid to call only when deserializing. + pub fn get(this: *const @This(), extra: []const u32) []const u32 { + return extra[@intFromPtr(this.data)..][0..this.len]; + } + + /// Valid to call only when deserializing. + pub fn tag(this: *const @This(), extra: []const u32, i: usize) Tag { + _ = this; + _ = extra; + _ = i; + @panic("TODO implement UnionList.tag"); + } + + fn extraLen(len: usize) usize { + return len + (len * @bitSizeOf(Meta) + 31) / 32; + } + }; + } + pub fn dataLength(buffer: []const u32, i: usize, comptime S: type) usize { var end = i; _ = data(buffer, &end, S); @@ -1848,6 +1918,23 @@ pub const Storage = enum { defer i.* = data_start + len; return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; }, + .union_list => { + const flags = @field(container, @tagName(Field.flags)); + const flag = @field(flags, @tagName(Field.flag)); + if (!flag) return .{ .data = null, .len = 0 }; + const meta_start = i.*; + const meta_buffer = buffer[meta_start..]; + var len: u32 = 0; + var bit_offset: usize = 0; + while (true) : (bit_offset += @bitSizeOf(Field.Meta)) { + const meta = loadBits(u32, meta_buffer, bit_offset, Field.Meta); + len += 1; + if (meta.last) break; + } + const end = meta_start + Field.extraLen(len); + i.* = end; + return .{ .data = end - len, .len = len }; + }, }, }, .@"extern" => comptime unreachable, @@ -1884,6 +1971,7 @@ pub const Storage = enum { .auto => switch (Field.storage) { .flag_optional, .enum_optional, .extended => 1, .flag_length_prefixed_list => field.slice.len + 1, + .union_list => Field.extraLen(field.len), }, .@"extern" => comptime unreachable, }, @@ -1947,6 +2035,33 @@ pub const Storage = enum { @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); return len + 1; }, + .union_list => { + if (value.len == 0) return 0; + const Tag = @typeInfo(Field.Union).@"union".tag_type.?; + const slice_ptr: [*]const Field.Union = @ptrCast(@alignCast(value.data)); + const slice = slice_ptr[0..value.len]; + const meta_buffer = buffer[i..][0 .. (slice.len * @bitSizeOf(Field.Meta) + 31) / 32]; + for (slice[0 .. slice.len - 1], 0..) |elem, elem_index| { + const union_tag: Tag = elem; + storeBits(u32, meta_buffer, elem_index * @bitSizeOf(Field.Meta), @as(Field.Meta, .{ + .tag = union_tag, + .last = false, + })); + } else { + const elem_index = slice.len - 1; + const elem = slice[elem_index]; + const union_tag: Tag = elem; + storeBits(u32, meta_buffer, elem_index * @bitSizeOf(Field.Meta), @as(Field.Meta, .{ + .tag = union_tag, + .last = true, + })); + } + var total: usize = meta_buffer.len; + for (i + meta_buffer.len.., slice) |elem_index, src| switch (src) { + inline else => |x| total += setExtraField(buffer, elem_index, @TypeOf(x), x), + }; + return total; + }, }, }, .@"extern" => comptime unreachable, @@ -2000,3 +2115,48 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { try reader.readVecAll(&vecs); return result; } + +pub fn loadBits(comptime Int: type, buffer: []const Int, bit_offset: usize, comptime Result: type) Result { + const index = bit_offset / @bitSizeOf(Int); + const small_bit_offset = bit_offset % @bitSizeOf(Int); + const ResultInt = @Int(.unsigned, @bitSizeOf(Result)); + const result: ResultInt = @truncate(buffer[index] >> @intCast(small_bit_offset)); + const available_bits = @bitSizeOf(Int) - small_bit_offset; + if (available_bits >= @bitSizeOf(ResultInt)) return @bitCast(result); + const missing_bits = @bitSizeOf(ResultInt) - available_bits; + const upper: ResultInt = @truncate(buffer[index + 1] & ((@as(usize, 1) << @intCast(missing_bits)) - 1)); + return @bitCast(result | (upper << @intCast(available_bits))); +} + +pub fn storeBits(comptime Int: type, buffer: []Int, bit_offset: usize, value: anytype) void { + const Value = @TypeOf(value); + const ValueInt = @Int(.unsigned, @bitSizeOf(Value)); + const value_int: ValueInt = @bitCast(value); + const index = bit_offset / @bitSizeOf(Int); + const small_bit_offset = bit_offset % @bitSizeOf(Int); + const available_bits = @bitSizeOf(Int) - small_bit_offset; + if (available_bits >= @bitSizeOf(ValueInt)) { + buffer[index] &= ~(((@as(Int, 1) << @intCast(@bitSizeOf(Value))) - 1) << @intCast(small_bit_offset)); + buffer[index] |= @as(Int, value_int) << @intCast(small_bit_offset); + } else { + const DoubleInt = @Int(.unsigned, @bitSizeOf(Int) * 2); + const ptr: *align(@alignOf(Int)) DoubleInt = @ptrCast(buffer[index..][0..2]); + ptr.* &= ~(((@as(DoubleInt, 1) << @intCast(@bitSizeOf(Value))) - 1) << @intCast(small_bit_offset)); + ptr.* |= @as(DoubleInt, value_int) << @intCast(small_bit_offset); + } +} + +test "loadBits and storeBits" { + var buffer: [2]u32 = .{ + 0b01111111000000001111111100000000, + 0b11111111000000001111111100000100, + }; + try std.testing.expectEqual(0b100, loadBits(u32, &buffer, 6, u3)); + try std.testing.expectEqual(0b100011, loadBits(u32, &buffer, 29, u6)); + + storeBits(u32, &buffer, 6, @as(u3, 0b010)); + storeBits(u32, &buffer, 29, @as(u6, 0b010010)); + + try std.testing.expectEqual(0b010, loadBits(u32, &buffer, 6, u3)); + try std.testing.expectEqual(0b010010, loadBits(u32, &buffer, 29, u6)); +} -- 2.54.0 From d3d3fb8473e25273ddca2e0da8d9c76cc43c794b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 26 Feb 2026 20:33:59 -0800 Subject: [PATCH 031/179] maker: progress towards updating zig CLI lowering --- lib/compiler/Maker/Step/Compile.zig | 50 +++++++++++++++----------- lib/std/zig/Configuration.zig | 56 ++++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 28638565fad1a807a6f188f57076dfd07810db74..86f4149338a9f742520f4989ec64bb41ab3e6c80 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -30,8 +30,8 @@ pub fn make( const graph = maker.graph; const step = maker.stepByIndex(step_index); compile.zig_args.clearRetainingCapacity(); - if (true) @panic("TODO implement compile.make()"); try lowerZigArgs(compile, step_index, maker, &compile.zig_args, false); + if (true) @panic("TODO implement compile.make()"); const process_arena = graph.arena; // TODO don't leak into the process_arena const maybe_output_dir = step.evalZigProcess( @@ -92,10 +92,13 @@ fn lowerZigArgs( const graph = maker.graph; const arena = graph.arena; // TODO don't leak into the process arena const gpa = maker.gpa; + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_comp = conf_step.extended.get(conf.extra).compile; try zig_args.append(gpa, graph.zig_exe); - const cmd = switch (compile.kind) { + const cmd = switch (conf_comp.flags3.kind) { .lib => "build-lib", .exe => "build-exe", .obj => "build-obj", @@ -107,25 +110,32 @@ fn lowerZigArgs( if (graph.reference_trace) |some| { try zig_args.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{some})); } - try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse graph.allow_so_scripts); + try addFlag(gpa, zig_args, "allow-so-scripts", conf_comp.flags2.allow_so_scripts.toBool() orelse graph.allow_so_scripts); - try addFlag(&zig_args, "llvm", compile.use_llvm); - try addFlag(&zig_args, "lld", compile.use_lld); - try addFlag(&zig_args, "new-linker", compile.use_new_linker); + try addFlag(gpa, zig_args, "llvm", conf_comp.flags2.use_llvm.toBool()); + try addFlag(gpa, zig_args, "lld", conf_comp.flags2.use_lld.toBool()); + try addFlag(gpa, zig_args, "new-linker", conf_comp.flags2.use_new_linker.toBool()); - if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { - try zig_args.append(gpa, try allocPrint(arena, "-ofmt={t}", .{ofmt})); + const root_module = conf_comp.root_module.get(conf); + + if (root_module.resolved_target.get(conf).?.query.unwrap()) |query| { + if (query.get(conf).flags.object_format.get()) |ofmt| { + try zig_args.append(gpa, try allocPrint(arena, "-ofmt={t}", .{ofmt})); + } } - switch (compile.entry) { + switch (conf_comp.flags3.entry) { .default => {}, .disabled => try zig_args.append(gpa, "-fno-entry"), .enabled => try zig_args.append(gpa, "-fentry"), - .symbol_name => |entry_name| { - try zig_args.append(gpa, try allocPrint(arena, "-fentry={s}", .{entry_name})); + .symbol_name => { + const symbol_name = conf_comp.entry.value.?.slice(conf); + try zig_args.append(gpa, try allocPrint(arena, "-fentry={s}", .{symbol_name})); }, } + if (true) @panic("TODO"); + { for (compile.force_undefined_symbols.keys()) |symbol_name| { try zig_args.append(gpa, "--force_undefined"); @@ -408,7 +418,7 @@ fn lowerZigArgs( if (!my_responsibility) continue; if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; - try mod.appendZigProcessFlags(&zig_args, step); + try mod.appendZigProcessFlags(zig_args, step); // --dep arguments try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); @@ -507,7 +517,7 @@ fn lowerZigArgs( if (compile.generated_llvm_ir != null) try zig_args.append(gpa, "-femit-llvm-ir"); if (compile.generated_h != null) try zig_args.append(gpa, "-femit-h"); - try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); + try addFlag(gpa, zig_args, "formatted-panics", compile.formatted_panics); switch (compile.compress_debug_sections) { .none => {}, @@ -612,9 +622,9 @@ fn lowerZigArgs( try zig_args.append(gpa, "--discard-all"); } - try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); - try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); - try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); + try addFlag(gpa, zig_args, "compiler-rt", compile.bundle_compiler_rt); + try addFlag(gpa, zig_args, "ubsan-rt", compile.bundle_ubsan_rt); + try addFlag(gpa, zig_args, "dll-export-fns", compile.dll_export_fns); if (compile.rdynamic) { try zig_args.append(gpa, "-rdynamic"); } @@ -718,7 +728,7 @@ fn lowerZigArgs( try zig_args.appendSlice(gpa, &.{ "-rcincludes", @tagName(compile.rc_includes) }); } - try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); + try addFlag(gpa, zig_args, "each-lib-rpath", compile.each_lib_rpath); if (compile.build_id orelse graph.build_id) |build_id| { try zig_args.append(gpa, switch (build_id) { @@ -739,7 +749,7 @@ fn lowerZigArgs( try zig_args.append(gpa, zig_lib_dir); } - try addFlag(&zig_args, "PIE", compile.pie); + try addFlag(gpa, zig_args, "PIE", compile.pie); if (compile.lto) |lto| { try zig_args.append(gpa, switch (lto) { @@ -749,7 +759,7 @@ fn lowerZigArgs( }); } - try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); + try addFlag(gpa, zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); if (compile.subsystem) |subsystem| { try zig_args.appendSlice(gpa, &.{ "--subsystem", @tagName(subsystem) }); @@ -763,7 +773,7 @@ fn lowerZigArgs( "--error-limit", try allocPrint(arena, "{d}", .{err_limit}), }); - try addFlag(&zig_args, "incremental", graph.incremental); + try addFlag(gpa, zig_args, "incremental", graph.incremental); try zig_args.append(gpa, "--listen=-"); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 8f6d03d1c5a53ac457fde9b037735e96c6f34f20..072e7ec6022b37b8ce9a3bffbd7b3b7ac77eaba0 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -1147,6 +1147,10 @@ pub const Module = struct { pub const Index = enum(u32) { _, + + pub fn get(this: @This(), c: *const Configuration) Module { + return extraData(c, Module, @intFromEnum(this)); + } }; pub const Flags = packed struct(u32) { @@ -1318,6 +1322,14 @@ pub const DefaultingBool = enum(u2) { true => .true, }; } + + pub fn toBool(db: DefaultingBool) ?bool { + return switch (db) { + .false => false, + .true => true, + .default => null, + }; + } }; pub const SystemLib = struct { @@ -1431,11 +1443,26 @@ pub const ResolvedTarget = struct { pub const Index = enum(u32) { _, + + pub fn get(this: @This(), c: *const Configuration) ?ResolvedTarget { + return extraData(c, ResolvedTarget, @intFromEnum(this)); + } }; pub const OptionalIndex = enum(u32) { none = maxInt(u32), _, + + pub fn unwrap(this: @This()) ?Index { + return switch (this) { + .none => null, + _ => @enumFromInt(@intFromEnum(this)), + }; + } + + pub fn get(this: @This(), c: *const Configuration) ?ResolvedTarget { + return (unwrap(this) orelse return null).get(c); + } }; }; @@ -1468,6 +1495,10 @@ pub const TargetQuery = struct { pub fn length(i: Index, extra: []const u32) usize { return Storage.dataLength(extra, @intFromEnum(i), TargetQuery); } + + pub fn get(this: @This(), c: *const Configuration) TargetQuery { + return extraData(c, TargetQuery, @intFromEnum(this)); + } }; pub const OptionalIndex = enum(u32) { @@ -1479,6 +1510,13 @@ pub const TargetQuery = struct { assert(result != .none); return result; } + + pub fn unwrap(this: @This()) ?Index { + return switch (this) { + .none => null, + _ => @enumFromInt(@intFromEnum(this)), + }; + } }; pub const CpuModel = enum(u2) { @@ -1680,6 +1718,22 @@ pub const TargetQuery = struct { // TODO comptime assert the enums match return @enumFromInt(@intFromEnum(x orelse return .default)); } + + pub fn get(this: @This()) ?std.Target.ObjectFormat { + return switch (this) { + .c => .c, + .coff => .coff, + .elf => .elf, + .hex => .hex, + .macho => .macho, + .plan9 => .plan9, + .raw => .raw, + .spirv => .spirv, + .wasm => .wasm, + + .default => null, + }; + } }; pub const Flags = packed struct(u32) { @@ -1933,7 +1987,7 @@ pub const Storage = enum { } const end = meta_start + Field.extraLen(len); i.* = end; - return .{ .data = end - len, .len = len }; + return .{ .data = @ptrFromInt(end - len), .len = len }; }, }, }, -- 2.54.0 From 4381a387bfa73c523a49d6b5c4d8ea336c7d4275 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 27 Feb 2026 18:48:27 -0800 Subject: [PATCH 032/179] Configuration: complete serialization of Compile steps --- lib/compiler/Maker/ScannedConfig.zig | 19 ++++- lib/compiler/Maker/Step/Compile.zig | 51 +++++++------ lib/compiler/configurer.zig | 14 +++- lib/std/zig/Configuration.zig | 108 +++++++++++++++++++++++---- 4 files changed, 146 insertions(+), 46 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 07c92f295ae4d64219c82712e530cd84920c903e..6ccc7c421d33d30805e231b67d8a10df0a66154a 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -85,6 +85,8 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi .flag_length_prefixed_list => comptime unreachable, .enum_optional => comptime unreachable, .union_list => comptime unreachable, + .length_prefixed_list => comptime unreachable, + .flag_union => comptime unreachable, } else if (std.enums.tagName(Field, field_value)) |name| { try s.ident(name); } else { @@ -103,14 +105,29 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try s.value(null, .{}); } }, - .flag_length_prefixed_list => { + .length_prefixed_list, .flag_length_prefixed_list => { try printValue(sc, s, @TypeOf(field_value.slice), field_value.slice); }, .extended => @compileError("TODO"), .union_list => @compileError("TODO"), + .flag_union => try printValue(sc, s, Field.Union, field_value.u), }, else => @compileError("not implemented: " ++ @typeName(Field)), }, + .@"union" => { + switch (field_value) { + inline else => |u, tag| { + if (@TypeOf(u) == void) { + try s.ident(@tagName(tag)); + } else { + var sub_struct = try s.beginStruct(.{}); + try sub_struct.fieldPrefix(@tagName(tag)); + try printValue(sc, s, @TypeOf(u), u); + try sub_struct.end(); + } + }, + } + }, else => @compileError("not implemented: " ++ @typeName(Field)), }, } diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 86f4149338a9f742520f4989ec64bb41ab3e6c80..798d3c05e143931c10eb27e0ff34339d29a13431 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -134,21 +134,20 @@ fn lowerZigArgs( }, } + for (conf_comp.force_undefined_symbols.slice) |symbol_name| { + try zig_args.appendSlice(gpa, &.{ "--force_undefined", symbol_name.slice(conf) }); + } + + if (conf_comp.stack_size.value) |stack_size| { + try zig_args.appendSlice(gpa, &.{ "--stack", try allocPrint(arena, "{d}", .{stack_size}) }); + } + + try addBool(gpa, zig_args, "-ffuzz", fuzz); + if (true) @panic("TODO"); - { - for (compile.force_undefined_symbols.keys()) |symbol_name| { - try zig_args.append(gpa, "--force_undefined"); - try zig_args.append(gpa, symbol_name.*); - } - } - - if (compile.stack_size) |stack_size| { - try zig_args.append(gpa, "--stack"); - try zig_args.append(gpa, try allocPrint(arena, "{}", .{stack_size})); - } - - try addBool(gpa, zig_args, fuzz, "-ffuzz"); + var is_linking_libc = conf_comp.flags3.is_linking_libc; + var is_linking_libcpp = conf_comp.flags3.is_linking_libcpp; { // Stores system libraries that have already been seen for at least one @@ -163,14 +162,14 @@ fn lowerZigArgs( var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; // Track the number of positional arguments so that a nice error can be // emitted if there is nothing to link. - var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); + var total_linker_objects: usize = @intFromBool(root_module.root_source_file != .none); // Fully recursive iteration including dynamic libraries to detect // libc and libc++ linkage. for (getCompileDependencies(true)) |some_compile| { for (some_compile.root_module.getGraph().modules) |mod| { - if (mod.link_libc == true) compile.is_linking_libc = true; - if (mod.link_libcpp == true) compile.is_linking_libcpp = true; + if (mod.link_libc == true) is_linking_libc = true; + if (mod.link_libcpp == true) is_linking_libcpp = true; } } @@ -465,11 +464,11 @@ fn lowerZigArgs( try zig_args.append(gpa, name); } - if (compile.is_linking_libcpp) { + if (is_linking_libcpp) { try zig_args.append(gpa, "-lc++"); } - if (compile.is_linking_libc) { + if (is_linking_libc) { try zig_args.append(gpa, "-lc"); } } @@ -500,14 +499,14 @@ fn lowerZigArgs( try zig_args.appendSlice(gpa, &.{ "--debug-log", log_scope }); } - try addBool(gpa, zig_args, graph.debug_compile_errors, "--debug-compile-errors"); - try addBool(gpa, zig_args, graph.debug_incremental, "--debug-incremental"); - try addBool(gpa, zig_args, graph.verbose_air, "--verbose-air"); - try addBool(gpa, zig_args, graph.verbose_llvm_ir, "--verbose-llvm-ir"); - try addBool(gpa, zig_args, graph.verbose_link or compile.verbose_link, "--verbose-link"); - try addBool(gpa, zig_args, graph.verbose_cc or compile.verbose_cc, "--verbose-cc"); - try addBool(gpa, zig_args, graph.verbose_llvm_cpu_features, "--verbose-llvm-cpu-features"); - try addBool(gpa, zig_args, graph.time_report, "--time-report"); + try addBool(gpa, zig_args, "--debug-compile-errors", graph.debug_compile_errors); + try addBool(gpa, zig_args, "--debug-incremental", graph.debug_incremental); + try addBool(gpa, zig_args, "--verbose-air", graph.verbose_air); + try addBool(gpa, zig_args, "--verbose-llvm-ir", graph.verbose_llvm_ir); + try addBool(gpa, zig_args, "--verbose-link", graph.verbose_link or compile.verbose_link); + try addBool(gpa, zig_args, "--verbose-cc", graph.verbose_cc or compile.verbose_cc); + try addBool(gpa, zig_args, "--verbose-llvm-cpu-features", graph.verbose_llvm_cpu_features); + try addBool(gpa, zig_args, "--time-report", graph.time_report); if (compile.generated_asm != null) try zig_args.append(gpa, "-femit-asm"); if (compile.generated_bin == null) try zig_args.append(gpa, "-fno-emit-bin"); diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index beb8c7df2630baf1f931b8476f8dddc04f1559ae..63ac01697390647a6bbbc6200549934b4ef7ada2 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -606,7 +606,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .max_memory = c.max_memory != null, .kind = c.kind, .global_base = c.global_base != null, - .test_runner_mode = if (c.test_runner) |tr| switch (tr.mode) { + .test_runner = if (c.test_runner) |tr| switch (tr.mode) { .simple => .simple, .server => .server, } else .default, @@ -678,10 +678,18 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .exec_cmd_args = .{ .slice = try s.initOptionalStringList(exec_cmd_args) }, .installed_headers = .initErased(installed_headers), .force_undefined_symbols = .{ .slice = try s.initStringList(c.force_undefined_symbols.keys()) }, + .expect_errors = .{ .u = if (c.expect_errors) |x| switch (x) { + .contains => |slice| .{ .contains = try wc.addString(slice) }, + .exact => |exact| .{ .exact = .{ .slice = try s.initStringList(exact) } }, + .starts_with => |slice| .{ .starts_with = try wc.addString(slice) }, + .stderr_contains => |slice| .{ .stderr_contains = try wc.addString(slice) }, + } else .none }, + .test_runner = .{ .u = if (c.test_runner) |tr| switch (tr.mode) { + .simple => .{ .simple = try s.addLazyPath(tr.path) }, + .server => .{ .server = try s.addLazyPath(tr.path) }, + } else .default }, })); - log.err("TODO serialize the trailing Compile step data", .{}); - break :e @enumFromInt(extra_index); }, .install_artifact => e: { diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 072e7ec6022b37b8ce9a3bffbd7b3b7ac77eaba0..014a16de2a8635c6402640ea359d15e7741e2a53 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -584,9 +584,6 @@ pub const Step = extern struct { }; }; - /// Trailing: - /// * exact_match: String, // if expect_errors is contains, starts_with, or stderr_contains - /// * test_runner: LazyPath, // if test_runner_mode is not default pub const Compile = struct { flags: @This().Flags, flags2: Flags2, @@ -600,7 +597,7 @@ pub const Step = extern struct { exec_cmd_args: Storage.FlagLengthPrefixedList(.flags, .exec_cmd_args_len, OptionalString), installed_headers: Storage.FlagLengthPrefixedList(.flags, .installed_headers_len, Storage.Extended(InstalledHeader.Flags, InstalledHeader)), force_undefined_symbols: Storage.FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), - //exacts: EnumConditionalPrefixedList(.flags4, .expect_errors, .exact, String), + expect_errors: Storage.FlagUnion(.flags4, .expect_errors, ExpectErrors), linker_script: Storage.FlagOptional(.flags4, .linker_script, LazyPath), version_script: Storage.FlagOptional(.flags4, .version_script, LazyPath), zig_lib_dir: Storage.FlagOptional(.flags3, .zig_lib_dir, LazyPath), @@ -622,6 +619,7 @@ pub const Step = extern struct { headerpad_size: Storage.FlagOptional(.flags4, .headerpad_size, u32), error_limit: Storage.FlagOptional(.flags4, .error_limit, u32), build_id: Storage.EnumOptional(.flags3, .build_id, .hexstring, String), + test_runner: Storage.FlagUnion(.flags3, .test_runner, TestRunner), pub const InstalledHeader = union(@This().Tag) { file: File, @@ -663,8 +661,22 @@ pub const Step = extern struct { }; }; }; - pub const ExpectErrors = enum(u3) { contains, exact, starts_with, stderr_contains, none }; - pub const TestRunnerMode = enum(u2) { default, simple, server }; + pub const ExpectErrors = union(@This().Tag) { + pub const Tag = enum(u3) { contains, exact, starts_with, stderr_contains, none }; + + contains: String, + exact: Storage.LengthPrefixedList(String), + starts_with: String, + stderr_contains: String, + none: void, + }; + pub const TestRunner = union(@This().Tag) { + pub const Tag = enum(u2) { default, simple, server }; + + default: void, + simple: LazyPath, + server: LazyPath, + }; pub const Entry = enum(u2) { default, disabled, enabled, symbol_name }; pub const Lto = enum(u2) { @@ -826,7 +838,7 @@ pub const Step = extern struct { kind: Kind, compress_debug_sections: std.zig.CompressDebugSections, global_base: bool, - test_runner_mode: TestRunnerMode, + test_runner: TestRunner.Tag, wasi_exec_model: WasiExecModel, win32_manifest: bool, win32_module_definition: bool, @@ -849,7 +861,7 @@ pub const Step = extern struct { error_limit: bool, install_name: bool, entitlements: bool, - expect_errors: ExpectErrors, + expect_errors: ExpectErrors.Tag, linker_script: bool, version_script: bool, _: u18 = 0, @@ -1756,8 +1768,10 @@ pub const Storage = enum { flag_optional, enum_optional, extended, + length_prefixed_list, flag_length_prefixed_list, union_list, + flag_union, /// The presence of the field is determined by a boolean within a packed /// struct. @@ -1776,6 +1790,24 @@ pub const Storage = enum { }; } + /// The type of the field is determined by an enum within a packed struct. + pub fn FlagUnion( + comptime flags_arg: @EnumLiteral(), + comptime flag_arg: @EnumLiteral(), + comptime UnionArg: type, + ) type { + return struct { + u: Union, + + pub const storage: Storage = .flag_union; + pub const flags = flags_arg; + pub const flag = flag_arg; + pub const Union = UnionArg; + + pub const Tag = @typeInfo(Union).@"union".tag_type.?; + }; + } + /// The field is present if an enum tag from flags matches a specific value. pub fn EnumOptional( comptime flags_arg: @EnumLiteral(), @@ -1814,21 +1846,32 @@ pub const Storage = enum { /// A field in flags determines whether the length is zero or nonzero. If the length is /// nonzero, then there is a length field followed by the list. - /// - /// When deserializing, the slice field is set. When serializing, the index - /// field must be set. pub fn FlagLengthPrefixedList( comptime flags_arg: @EnumLiteral(), comptime flag_arg: @EnumLiteral(), - comptime ValueArg: type, + comptime ElemArg: type, ) type { return struct { - slice: []const Value, + slice: []const Elem, pub const storage: Storage = .flag_length_prefixed_list; pub const flags = flags_arg; pub const flag = flag_arg; - pub const Value = ValueArg; + pub const Elem = ElemArg; + + pub fn initErased(s: []const u32) @This() { + return .{ .slice = @ptrCast(s) }; + } + }; + } + + /// The field contains a u32 length followed by that many items. + pub fn LengthPrefixedList(comptime ElemArg: type) type { + return struct { + slice: []const Elem, + + pub const storage: Storage = .length_prefixed_list; + pub const Elem = ElemArg; pub fn initErased(s: []const u32) @This() { return .{ .slice = @ptrCast(s) }; @@ -1910,6 +1953,7 @@ pub const Storage = enum { fn dataField(buffer: []const u32, i: *usize, container: anytype, comptime Field: type) Field { switch (@typeInfo(Field)) { + .void => return {}, .int => |info| switch (info.bits) { 32 => { defer i.* += 1; @@ -1954,6 +1998,24 @@ pub const Storage = enum { .value = if (flag) dataField(buffer, i, container, Field.Value) else null, }; }, + .flag_union => { + const flags = @field(container, @tagName(Field.flags)); + const tag: Field.Tag = @field(flags, @tagName(Field.flag)); + return .{ + .u = switch (tag) { + inline else => |comptime_tag| @unionInit( + Field.Union, + @tagName(comptime_tag), + dataField( + buffer, + i, + container, + @typeInfo(Field.Union).@"union".fields[@intFromEnum(comptime_tag)].type, + ), + ), + }, + }; + }, .enum_optional => { const flags = @field(container, @tagName(Field.flags)); const tag = @field(flags, @tagName(Field.flag)); @@ -1963,6 +2025,12 @@ pub const Storage = enum { }; }, .extended => @compileError("TODO"), + .length_prefixed_list => { + const data_start = i.* + 1; + const len = buffer[data_start - 1]; + defer i.* = data_start + len; + return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; + }, .flag_length_prefixed_list => { const flags = @field(container, @tagName(Field.flags)); const flag = @field(flags, @tagName(Field.flag)); @@ -2010,6 +2078,7 @@ pub const Storage = enum { fn extraFieldLen(field: anytype) usize { const Field = @TypeOf(field); return switch (@typeInfo(Field)) { + .void => 0, .int => |info| switch (info.bits) { 32 => 1, 64 => 2, @@ -2024,8 +2093,11 @@ pub const Storage = enum { }, .auto => switch (Field.storage) { .flag_optional, .enum_optional, .extended => 1, - .flag_length_prefixed_list => field.slice.len + 1, + .length_prefixed_list, .flag_length_prefixed_list => field.slice.len + 1, .union_list => Field.extraLen(field.len), + .flag_union => switch (field.u) { + inline else => |v| extraFieldLen(v), + }, }, .@"extern" => comptime unreachable, }, @@ -2044,6 +2116,7 @@ pub const Storage = enum { inline fn setExtraField(buffer: []u32, i: usize, comptime Field: type, value: anytype) usize { switch (@typeInfo(Field)) { + .void => return 0, .int => |info| switch (info.bits) { 32 => { buffer[i] = value; @@ -2081,8 +2154,11 @@ pub const Storage = enum { .flag_optional, .enum_optional => { return if (value.value) |v| setExtraField(buffer, i, Field.Value, v) else 0; }, + .flag_union => return switch (value.u) { + inline else => |x| setExtraField(buffer, i, @TypeOf(x), x), + }, .extended => @compileError("TODO"), - .flag_length_prefixed_list => { + .flag_length_prefixed_list, .length_prefixed_list => { const len: u32 = @intCast(value.slice.len); if (len == 0) return 0; buffer[i] = len; -- 2.54.0 From 5fbfeabd5f3bcfaaec7e157df171753940f4872c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 2 Mar 2026 12:10:48 -0800 Subject: [PATCH 033/179] compiler: no longer need ThreadSafeArena wrapper --- src/main.zig | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main.zig b/src/main.zig index 2e6da4b02d72b3aad66d90b9a31059ba0308774c..c03bbc4e101247d875d6228023e4bf10d661063c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -298,11 +298,7 @@ fn mainArgs( return process.exit(try llvmArMain(arena, args)); } else if (mem.eql(u8, cmd, "build")) { dev.check(.build_command); - var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ - .child_allocator = arena, - .io = io, - }; - return cmdBuild(gpa, thread_safe_arena.allocator(), io, cmd_args, environ_map); + return cmdBuild(gpa, arena, io, cmd_args, environ_map); } else if (mem.eql(u8, cmd, "clang") or mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) { @@ -4935,7 +4931,6 @@ test sanitizeExampleName { fn cmdBuild( gpa: Allocator, - /// Needs a thread-safe arena. arena: Allocator, io: Io, args: []const []const u8, -- 2.54.0 From 8331c59ee2ec3e57ffcc4c0ae57792a995afa865 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 2 Mar 2026 12:27:12 -0800 Subject: [PATCH 034/179] Configuration: serialize remaining Target information --- lib/std/zig/Configuration.zig | 92 +++++++++++++++-------------------- 1 file changed, 40 insertions(+), 52 deletions(-) diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 014a16de2a8635c6402640ea359d15e7741e2a53..b768d722655fe1b22a5a9092d90f2a159a3dd615 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -194,16 +194,16 @@ pub const Wip = struct { .native, .baseline, .determined_by_arch_os => null, .explicit => |model| try wip.addString(model.name), }; - const os_version_min: ?u32 = if (q.os_version_min) |ver| switch (ver) { - .none => null, - .semver => |sem_ver| @intFromEnum(try wip.addSemVer(sem_ver)), - .windows => |win_ver| @intFromEnum(win_ver), - } else null; - const os_version_max: ?u32 = if (q.os_version_max) |ver| switch (ver) { - .none => null, - .semver => |sem_ver| @intFromEnum(try wip.addSemVer(sem_ver)), - .windows => |win_ver| @intFromEnum(win_ver), - } else null; + const os_version_min: TargetQuery.OsVersion = if (q.os_version_min) |ver| switch (ver) { + .none => .none, + .semver => |sem_ver| .{ .semver = try wip.addSemVer(sem_ver) }, + .windows => |win_ver| .{ .windows = win_ver }, + } else .default; + const os_version_max: TargetQuery.OsVersion = if (q.os_version_max) |ver| switch (ver) { + .none => .none, + .semver => |sem_ver| .{ .semver = try wip.addSemVer(sem_ver) }, + .windows => |win_ver| .{ .windows = win_ver }, + } else .default; const glibc_version: ?String = if (q.glibc_version) |sem_ver| try wip.addSemVer(sem_ver) else null; const dynamic_linker: ?String = if (q.dynamic_linker) |*dl| if (dl.get()) |s| try wip.addString(s) else .empty @@ -220,8 +220,8 @@ pub const Wip = struct { .os_tag = .init(q.os_tag), .abi = .init(q.abi), .object_format = .init(q.ofmt), - .os_version_min = .init(q.os_version_min), - .os_version_max = .init(q.os_version_max), + .os_version_min = os_version_min, + .os_version_max = os_version_max, .glibc_version = glibc_version != null, .android_api_level = q.android_api_level != null, .dynamic_linker = dynamic_linker != null, @@ -232,10 +232,9 @@ pub const Wip = struct { .android_api_level = .{ .value = q.android_api_level }, .dynamic_linker = .{ .value = dynamic_linker }, .cpu_name = .{ .value = cpu_name }, + .os_version_min = .{ .u = os_version_min }, + .os_version_max = .{ .u = os_version_max }, }))); - std.log.err("TODO serialize more target query stuff", .{}); - _ = os_version_min; - _ = os_version_max; // Deduplicate. const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ @@ -253,45 +252,40 @@ pub const Wip = struct { const gpa = wip.gpa; const cpu_name: String = try wip.addString(t.cpu.model.name); - const os_version_min: ?u32, const os_version_max: ?u32, const glibc_version: ?String, const android_api_level: ?u32 = switch (t.os.versionRange()) { + const os_version_min: TargetQuery.OsVersion, const os_version_max: TargetQuery.OsVersion, const glibc_version: ?String, const android_api_level: ?u32 = switch (t.os.versionRange()) { .none => .{ - null, - null, + .none, + .none, null, null, }, .semver => |range| .{ - @intFromEnum(try wip.addSemVer(range.min)), - @intFromEnum(try wip.addSemVer(range.max)), + .{ .semver = try wip.addSemVer(range.min) }, + .{ .semver = try wip.addSemVer(range.max) }, null, null, }, .hurd => |hurd| .{ - @intFromEnum(try wip.addSemVer(hurd.range.min)), - @intFromEnum(try wip.addSemVer(hurd.range.max)), + .{ .semver = try wip.addSemVer(hurd.range.min) }, + .{ .semver = try wip.addSemVer(hurd.range.max) }, try wip.addSemVer(hurd.glibc), null, }, .linux => |linux| .{ - @intFromEnum(try wip.addSemVer(linux.range.min)), - @intFromEnum(try wip.addSemVer(linux.range.max)), + .{ .semver = try wip.addSemVer(linux.range.min) }, + .{ .semver = try wip.addSemVer(linux.range.max) }, try wip.addSemVer(linux.glibc), linux.android, }, .windows => |range| .{ - @intFromEnum(range.min), - @intFromEnum(range.max), + .{ .windows = range.min }, + .{ .windows = range.max }, null, null, }, }; const dynamic_linker: ?String = if (t.dynamic_linker.get()) |dl| try wip.addString(dl) else null; const cpu_features_add_empty = t.cpu.features.isEmpty(); - const os_version: TargetQuery.OsVersion = switch (t.os.versionRange()) { - .none => .none, - .semver, .linux, .hurd => .semver, - .windows => .windows, - }; const result_index: TargetQuery.Index = @enumFromInt(try wip.addExtra(@as(TargetQuery, .{ .flags = .{ .cpu_arch = .init(t.cpu.arch), @@ -301,8 +295,8 @@ pub const Wip = struct { .os_tag = .init(t.os.tag), .abi = .init(t.abi), .object_format = .init(t.ofmt), - .os_version_min = os_version, - .os_version_max = os_version, + .os_version_min = os_version_min, + .os_version_max = os_version_max, .glibc_version = glibc_version != null, .android_api_level = android_api_level != null, .dynamic_linker = dynamic_linker != null, @@ -313,10 +307,9 @@ pub const Wip = struct { .android_api_level = .{ .value = android_api_level }, .dynamic_linker = .{ .value = dynamic_linker }, .cpu_name = .{ .value = cpu_name }, + .os_version_min = .{ .u = os_version_min }, + .os_version_max = .{ .u = os_version_max }, }))); - std.log.err("TODO serialize more target stuff", .{}); - _ = os_version_min; - _ = os_version_max; // Deduplicate. const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ @@ -1484,19 +1477,12 @@ pub const TargetQuery = struct { cpu_features_add: Storage.FlagOptional(.flags, .cpu_features_add, std.Target.Cpu.Feature.Set), cpu_features_sub: Storage.FlagOptional(.flags, .cpu_features_sub, std.Target.Cpu.Feature.Set), cpu_name: Storage.EnumOptional(.flags, .cpu_model, .explicit, String), - //os_version_min: Storage.FlagsUnion(.flags, .os_version_min, VersionStorage), - //os_version_max: Storage.FlagsUnion(.flags, .os_version_max, VersionStorage), + os_version_min: Storage.FlagUnion(.flags, .os_version_min, OsVersion), + os_version_max: Storage.FlagUnion(.flags, .os_version_max, OsVersion), glibc_version: Storage.FlagOptional(.flags, .glibc_version, String), android_api_level: Storage.FlagOptional(.flags, .android_api_level, u32), dynamic_linker: Storage.FlagOptional(.flags, .dynamic_linker, String), - const VersionStorage = union(OsVersion) { - none: void, - semver: String, - windows: std.Target.Os.WindowsVersion, - default: void, - }; - pub const Index = enum(u32) { _, @@ -1546,11 +1532,13 @@ pub const TargetQuery = struct { }; } }; - pub const OsVersion = enum(u2) { - none, - semver, - windows, - default, + pub const OsVersion = union(@This().Tag) { + pub const Tag = enum(u2) { none, semver, windows, default }; + + none: void, + semver: String, + windows: std.Target.Os.WindowsVersion, + default: void, pub fn init(x: ?std.Target.Query.OsVersion) @This() { return switch (x orelse return .default) { @@ -1756,8 +1744,8 @@ pub const TargetQuery = struct { os_tag: OsTag, abi: Abi, object_format: ObjectFormat, - os_version_min: OsVersion, - os_version_max: OsVersion, + os_version_min: OsVersion.Tag, + os_version_max: OsVersion.Tag, glibc_version: bool, android_api_level: bool, dynamic_linker: bool, -- 2.54.0 From 8aec13b6ab1f0a8586f3e5cf5600d70697ac209d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 2 Mar 2026 19:46:22 -0800 Subject: [PATCH 035/179] Configuration: serialize remaining Module information also handle properly Module circular references and introduce a general deduplication mechanism. --- lib/compiler/Maker/ScannedConfig.zig | 2 + lib/compiler/configurer.zig | 103 ++++++++------- lib/std/Build/Step/Compile.zig | 1 + lib/std/zig/Configuration.zig | 179 +++++++++++++++++---------- 4 files changed, 174 insertions(+), 111 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 6ccc7c421d33d30805e231b67d8a10df0a66154a..e5ed99506d937f5c0bc0c7d9b5a4019fd6e7ca4a 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -87,6 +87,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi .union_list => comptime unreachable, .length_prefixed_list => comptime unreachable, .flag_union => comptime unreachable, + .multi_list => comptime unreachable, } else if (std.enums.tagName(Field, field_value)) |name| { try s.ident(name); } else { @@ -111,6 +112,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi .extended => @compileError("TODO"), .union_list => @compileError("TODO"), .flag_union => try printValue(sc, s, Field.Union, field_value.u), + .multi_list => @compileError("TODO"), }, else => @compileError("not implemented: " ++ @typeName(Field)), }, diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 63ac01697390647a6bbbc6200549934b4ef7ada2..e8548d5aa9b7a26330c34ea20dc140e0d17cafa6 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -294,9 +294,8 @@ const Serialize = struct { } fn addSystemLib(s: *Serialize, sl: *const std.Build.Module.SystemLib) !Configuration.SystemLib.Index { - log.err("TODO deduplicate addSystemLib", .{}); const wc = s.wc; - return @enumFromInt(try wc.addExtra(@as(Configuration.SystemLib, .{ + return @enumFromInt(try wc.addDeduped(@as(Configuration.SystemLib, .{ .flags = .{ .needed = sl.needed, .weak = sl.weak, @@ -362,7 +361,6 @@ const Serialize = struct { const wc = s.wc; const arena = s.arena; - const gpa = wc.gpa; const include_dirs = try arena.alloc(Configuration.Module.IncludeDir, m.include_dirs.items.len); for (include_dirs, m.include_dirs.items) |*dest, src| dest.* = switch (src) { @@ -393,66 +391,55 @@ const Serialize = struct { .win32_resource_file => |wrf| .{ .win32_resource_file = try addRcSourceFile(s, wrf) }, }; + const frameworks = try arena.alloc(Configuration.Module.Framework, m.frameworks.entries.len); + for (frameworks, m.frameworks.keys(), m.frameworks.values()) |*dest, name, options| dest.* = .{ + .flags = .{ + .needed = options.needed, + .weak = options.weak, + }, + .name = try wc.addString(name), + }; + const lib_paths = try arena.alloc(Configuration.LazyPath, m.lib_paths.items.len); for (lib_paths, m.lib_paths.items) |*dest, src| dest.* = try addLazyPath(s, src); const c_macros = try initStringList(s, m.c_macros.items); const export_symbol_names = try initStringList(s, m.export_symbol_names); - const import_table: Configuration.ImportTable = @enumFromInt(wc.extra.items.len); - const import_table_extra_len = 1 + 2 * m.import_table.entries.len; - try wc.extra.ensureUnusedCapacity(gpa, import_table_extra_len); - wc.extra.items.len += import_table_extra_len; - wc.extra.appendAssumeCapacity(@intCast(m.import_table.entries.len)); - wc.extra.items[@intFromEnum(import_table)] = @intCast(m.import_table.entries.len); - for ( - m.import_table.keys(), - @intFromEnum(import_table) + 1.., - ) |mod_name, extra_index| { - wc.extra.items[extra_index] = @intFromEnum(try wc.addString(mod_name)); - } - for ( - m.import_table.values(), - @intFromEnum(import_table) + 1 + m.import_table.entries.len.., - ) |dep, extra_index| { - log.err("TODO module dependencies can be cyclic", .{}); - wc.extra.items[extra_index] = @intFromEnum(try addModule(s, dep)); - } - const module_index: Configuration.Module.Index = @enumFromInt(try wc.addExtra(@as(Configuration.Module, .{ .flags = .{ .optimize = .init(m.optimize), .strip = .init(m.strip), .unwind_tables = .init(m.unwind_tables), .dwarf_format = .init(m.dwarf_format), - .single_threaded = .init(m.strip), - .stack_protector = .init(m.strip), - .stack_check = .init(m.strip), + .single_threaded = .init(m.single_threaded), + .stack_protector = .init(m.stack_protector), + .stack_check = .init(m.stack_check), .sanitize_c = .init(m.sanitize_c), - .sanitize_thread = .init(m.strip), - .fuzz = .init(m.strip), + .sanitize_thread = .init(m.sanitize_thread), + .fuzz = .init(m.fuzz), .code_model = m.code_model, .c_macros = c_macros.len != 0, .include_dirs = include_dirs.len != 0, .lib_paths = lib_paths.len != 0, .rpaths = rpaths.len != 0, - .frameworks = m.frameworks.entries.len != 0, + .frameworks = frameworks.len != 0, .link_objects = link_objects.len != 0, .export_symbol_names = export_symbol_names.len != 0, }, .flags2 = .{ - .valgrind = .init(m.strip), - .pic = .init(m.strip), - .red_zone = .init(m.strip), - .omit_frame_pointer = .init(m.strip), - .error_tracing = .init(m.strip), - .link_libc = .init(m.strip), - .link_libcpp = .init(m.strip), - .no_builtin = .init(m.strip), + .valgrind = .init(m.valgrind), + .pic = .init(m.pic), + .red_zone = .init(m.red_zone), + .omit_frame_pointer = .init(m.omit_frame_pointer), + .error_tracing = .init(m.error_tracing), + .link_libc = .init(m.link_libc), + .link_libcpp = .init(m.link_libcpp), + .no_builtin = .init(m.no_builtin), }, .owner = try s.builderToPackage(m.owner), .root_source_file = try s.addOptionalLazyPathEnum(m.root_source_file), - .import_table = import_table, + .import_table = .invalid, .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target), .c_macros = .{ .slice = c_macros }, .lib_paths = .{ .slice = lib_paths }, @@ -460,12 +447,33 @@ const Serialize = struct { .include_dirs = .init(include_dirs), .rpaths = .init(rpaths), .link_objects = .init(link_objects), + .frameworks = .{ .slice = frameworks }, }))); - log.err("TODO serialize the trailing Module data", .{}); - + // The import table is the only place that modules can form dependency + // loops. Therefore, we populate the module indexes only after adding + // the module to module_map. try s.module_map.putNoClobber(arena, m, module_index); + var imports = try std.MultiArrayList(Configuration.ImportTable.Import).initCapacity(arena, m.import_table.entries.len); + imports.len = m.import_table.entries.len; + for ( + imports.items(.name), + imports.items(.module), + m.import_table.keys(), + m.import_table.values(), + ) |*dest_name, *dest_module, src_name, src_module| { + dest_name.* = try wc.addString(src_name); + dest_module.* = try addModule(s, src_module); + } + + comptime assert(std.mem.eql(u8, @typeInfo(Configuration.Module).@"struct".fields[2].name, "import_table")); + comptime assert(@typeInfo(Configuration.Module).@"struct".fields[2].type == Configuration.ImportTable.Index); + assert(wc.extra.items[@intFromEnum(module_index) + 2] == @intFromEnum(Configuration.ImportTable.Index.invalid)); + wc.extra.items[@intFromEnum(module_index) + 2] = try wc.addDeduped(@as(Configuration.ImportTable, .{ + .imports = .{ .mal = imports }, + })); + return module_index; } @@ -502,12 +510,12 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { } // Add and then de-duplicate dependencies. - const deps = d: { - const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len); - for (try wc.reserveLengthPrefixed(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| - dep.* = @intCast(s.step_map.getIndex(dep_step).?); - break :d try wc.dedupeDeps(deps); - }; + const dep_steps = try arena.alloc(Configuration.Step.Index, step.dependencies.items.len); + for (dep_steps, step.dependencies.items) |*dest, src| + dest.* = @enumFromInt(s.step_map.getIndex(src).?); + const deps: Configuration.Deps.Index = @enumFromInt(try wc.addDeduped(@as(Configuration.Deps, .{ + .steps = .{ .slice = dep_steps }, + }))); try wc.steps.ensureTotalCapacity(gpa, s.step_map.entries.capacity); wc.steps.appendAssumeCapacity(.{ @@ -791,8 +799,7 @@ fn addOptionalResolvedTarget( optional_resolved_target: ?std.Build.ResolvedTarget, ) !Configuration.ResolvedTarget.OptionalIndex { const resolved_target = optional_resolved_target orelse return .none; - log.debug("TODO deduplicate resolved targets", .{}); - return @enumFromInt(try wc.addExtra(@as(Configuration.ResolvedTarget, .{ + return @enumFromInt(try wc.addDeduped(@as(Configuration.ResolvedTarget, .{ .query = try wc.addTargetQuery(resolved_target.query), .result = try wc.addTarget(resolved_target.result), }))); diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index abb89bc21839c012c0ceab2d9dd273875e754f33..0654894c4b5c209958d34105a8a6e8e81a066f83 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -43,6 +43,7 @@ export_memory: bool = false, /// For WebAssembly targets, this will allow for undefined symbols to /// be imported from the host environment. import_symbols: bool = false, +/// (WebAssembly) import function table from the host environment import_table: bool = false, export_table: bool = false, initial_memory: ?u64 = null, diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index b768d722655fe1b22a5a9092d90f2a159a3dd615..c1af30eb325ac34a76fdebb5f08a3adcf546f3c3 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -33,9 +33,8 @@ pub const Header = extern struct { pub const Wip = struct { gpa: Allocator, string_table: StringTable = .empty, - /// De-duplicates an array inside `extra` that has first element length - /// followed by length elements. - length_prefixed_table: LengthPrefixedTable = .empty, + /// De-duplicates an array inside `extra`. + dedupe_table: DedupeTable = .empty, targets_table: TargetsTable = .empty, string_bytes: std.ArrayList(u8) = .empty, @@ -46,25 +45,27 @@ pub const Wip = struct { path_deps: std.MultiArrayList(Path) = .empty, extra: std.ArrayList(u32) = .empty, - const LengthPrefixedTable = std.HashMapUnmanaged(u32, void, LengthPrefixedContext, std.hash_map.default_max_load_percentage); + const DedupeTable = std.HashMapUnmanaged(ExtraSlice, void, ExtraSlice.Context, std.hash_map.default_max_load_percentage); const TargetsTable = std.HashMapUnmanaged(TargetQuery.Index, void, TargetsTableContext, std.hash_map.default_max_load_percentage); - const LengthPrefixedContext = struct { - extra: []const u32, + const ExtraSlice = struct { + index: u32, + len: u32, - pub fn eql(ctx: @This(), a: u32, b: u32) bool { - const len_a = ctx.extra[a]; - const len_b = ctx.extra[b]; - const slice_a = ctx.extra[a + 1 ..][0..len_a]; - const slice_b = ctx.extra[b + 1 ..][0..len_b]; - return std.mem.eql(u32, slice_a, slice_b); - } + const Context = struct { + extra: []const u32, - pub fn hash(ctx: @This(), key: u32) u64 { - const len = ctx.extra[key]; - const slice = ctx.extra[key + 1 ..][0..len]; - return std.hash_map.hashString(@ptrCast(slice)); - } + pub fn eql(ctx: @This(), a: ExtraSlice, b: ExtraSlice) bool { + const slice_a = ctx.extra[a.index..][0..a.len]; + const slice_b = ctx.extra[b.index..][0..b.len]; + return std.mem.eql(u32, slice_a, slice_b); + } + + pub fn hash(ctx: @This(), key: ExtraSlice) u64 { + const slice = ctx.extra[key.index..][0..key.len]; + return std.hash_map.hashString(@ptrCast(slice)); + } + }; }; const TargetsTableContext = struct { @@ -323,36 +324,35 @@ pub const Wip = struct { } } - pub fn reserveLengthPrefixed(wip: *Wip, n: usize) Allocator.Error![]u32 { - const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1); - slice[0] = @intCast(n); - return slice[1..]; - } - - pub fn dedupeLengthPrefixed(wip: *Wip, index: u32) Allocator.Error!u32 { - assert(wip.extra.items.len == index + wip.extra.items[index] + 1); - const gpa = wip.gpa; - const gop = try wip.length_prefixed_table.getOrPutContext(gpa, index, @as(LengthPrefixedContext, .{ - .extra = wip.extra.items, - })); - if (gop.found_existing) { - wip.extra.items.len = index; - return gop.key_ptr.*; - } else { - return index; - } - } - - pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps { - return @enumFromInt(try dedupeLengthPrefixed(wip, @intFromEnum(deps))); - } - pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { const extra_len = Storage.extraLen(extra); try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len); return addExtraAssumeCapacity(wip, extra); } + /// Same as `addExtra` but uses a hash map to possibly return an already + /// existing index instead of appending to `extra`. + pub fn addDeduped(wip: *Wip, extra: anytype) Allocator.Error!u32 { + const gpa = wip.gpa; + const revert_index = wip.extra.items.len; + const extra_len = Storage.extraLen(extra); + try wip.extra.ensureUnusedCapacity(gpa, extra_len); + const new_index = addExtraAssumeCapacity(wip, extra); + const len: u32 = @intCast(wip.extra.items.len - new_index); + + const gop = try wip.dedupe_table.getOrPutContext(gpa, .{ + .index = new_index, + .len = len, + }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items })); + + if (gop.found_existing) { + wip.extra.items.len = revert_index; + return gop.key_ptr.index; + } + + return new_index; + } + pub fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 { const result: u32 = @intCast(wip.extra.items.len); wip.extra.items.len = Storage.setExtra(wip.extra.allocatedSlice(), result, extra); @@ -399,7 +399,7 @@ pub const AvailableOption = extern struct { pub const Step = extern struct { name: String, owner: Package.Index, - deps: Deps, + deps: Deps.Index, max_rss: MaxRss, extended: Storage.Extended(Flags, union(Tag) { check_file: CheckFile, @@ -1074,14 +1074,12 @@ pub const Package = struct { }; }; -/// Trailing: -/// * frameworks: FlagsPrefixedList(FrameworkFlags), // if flag is set pub const Module = struct { flags: Flags, flags2: Flags2, + import_table: ImportTable.Index, owner: Package.Index, root_source_file: OptionalLazyPath, - import_table: ImportTable, resolved_target: ResolvedTarget.OptionalIndex, c_macros: Storage.FlagLengthPrefixedList(.flags, .c_macros, String), lib_paths: Storage.FlagLengthPrefixedList(.flags, .lib_paths, LazyPath), @@ -1089,6 +1087,7 @@ pub const Module = struct { include_dirs: Storage.UnionList(.flags, .include_dirs, IncludeDir), rpaths: Storage.UnionList(.flags, .rpaths, RPath), link_objects: Storage.UnionList(.flags, .link_objects, LinkObject), + frameworks: Storage.FlagLengthPrefixedList(.flags, .frameworks, Framework), pub const Optimize = enum(u3) { debug, @@ -1220,28 +1219,47 @@ pub const Module = struct { win32_resource_file: RcSourceFile.Index, }; - pub const FrameworkFlags = packed struct(u2) { - needed: bool, - weak: bool, + pub const Framework = struct { + flags: @This().Flags, + name: String, + + pub const Flags = packed struct(u32) { + needed: bool, + weak: bool, + _: u30 = 0, + }; }; }; -/// Points into `extra`, first element is len, then: -/// * import_name: String, // for each len -/// * Module.Index, // for each len -pub const ImportTable = enum(u32) { - _, +pub const ImportTable = struct { + imports: Storage.MultiList(Import), + + pub const Import = struct { + name: String, + module: Module.Index, + }; + + /// Points into `extra`. + pub const Index = enum(u32) { + invalid = maxInt(u32), + _, + }; }; -/// Points into `extra`, where the first element is count of deps, following -/// elements is `Step.Index` per count. -pub const Deps = enum(u32) { - _, +pub const Deps = struct { + steps: Storage.LengthPrefixedList(Step.Index), + + pub const Index = enum(u32) { + _, - pub fn slice(deps: Deps, c: *const Configuration) []Step.Index { - const len = c.extra[@intFromEnum(deps)]; - return @ptrCast(c.extra[@intFromEnum(deps) + 1 ..][0..len]); - } + pub fn get(this: @This(), c: *const Configuration) Deps { + return extraData(c, Deps, @intFromEnum(this)); + } + + pub fn slice(this: @This(), c: *const Configuration) []const Step.Index { + return get(this, c).steps.slice; + } + }; }; /// Points into `extra`, where the first element is count of strings, following @@ -1760,6 +1778,7 @@ pub const Storage = enum { flag_length_prefixed_list, union_list, flag_union, + multi_list, /// The presence of the field is determined by a boolean within a packed /// struct. @@ -1853,7 +1872,8 @@ pub const Storage = enum { }; } - /// The field contains a u32 length followed by that many items. + /// The field contains a u32 length followed by that many items, each + /// element bitcastable to u32. pub fn LengthPrefixedList(comptime ElemArg: type) type { return struct { slice: []const Elem, @@ -1867,6 +1887,17 @@ pub const Storage = enum { }; } + /// The field contains a u32 length followed by that many items for the + /// first field, that many items for the second field, etc. + pub fn MultiList(comptime ElemArg: type) type { + return struct { + mal: std.MultiArrayList(Elem), + + pub const storage: Storage = .multi_list; + pub const Elem = ElemArg; + }; + } + /// `UnionArg` is a tagged union with a small integer for the enum tag. /// /// A field in flags determines whether the metadata is present. @@ -2028,6 +2059,16 @@ pub const Storage = enum { defer i.* = data_start + len; return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; }, + .multi_list => { + const data_start = i.* + 1; + const len = buffer[data_start - 1]; + defer i.* = data_start + len * @typeInfo(Field.Elem).@"struct".fields.len; + return .{ .mal = .{ + .bytes = @ptrCast(buffer[data_start..][0..len]), + .len = len, + .capacity = len, + } }; + }, .union_list => { const flags = @field(container, @tagName(Field.flags)); const flag = @field(flags, @tagName(Field.flag)); @@ -2082,6 +2123,7 @@ pub const Storage = enum { .auto => switch (Field.storage) { .flag_optional, .enum_optional, .extended => 1, .length_prefixed_list, .flag_length_prefixed_list => field.slice.len + 1, + .multi_list => 1 + field.mal.len * @typeInfo(Field.Elem).@"struct".fields.len, .union_list => Field.extraLen(field.len), .flag_union => switch (field.u) { inline else => |v| extraFieldLen(v), @@ -2153,6 +2195,17 @@ pub const Storage = enum { @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); return len + 1; }, + .multi_list => { + const len: u32 = @intCast(value.mal.len); + if (len == 0) return 0; + buffer[i] = len; + const fields = @typeInfo(Field.Elem).@"struct".fields; + inline for (0..fields.len) |field_i| @memcpy( + buffer[i + 1 + field_i * len ..][0..len], + @as([]const u32, @ptrCast(value.mal.items(@enumFromInt(field_i)))), + ); + return 1 + fields.len * len; + }, .union_list => { if (value.len == 0) return 0; const Tag = @typeInfo(Field.Union).@"union".tag_type.?; -- 2.54.0 From 6c61803d8b7281ffb4517cfbc335e4e0dfcdea66 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 2 Mar 2026 20:56:48 -0800 Subject: [PATCH 036/179] maker: implement module printing --- lib/compiler/Maker.zig | 18 +++++-- lib/compiler/Maker/ScannedConfig.zig | 70 +++++++++++++++++++--------- lib/std/zig/Configuration.zig | 4 +- 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index fa701001390e005a926ba17d1a6cc5bf62da7199..0faafceccffba312100eb48490088e8801910991 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -429,19 +429,29 @@ pub fn main(init: process.Init.Minimal) !void { break :c Configuration.loadFile(arena, io, file) catch |err| fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); }; + const c = &configuration; var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; + var modules: std.AutoArrayHashMapUnmanaged(Configuration.Module.Index, void) = .empty; for (configuration.steps, 0..) |*conf_step, step_index_usize| { if (conf_step.owner != .root) continue; const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); - const flags = conf_step.flags(&configuration); - if (flags.tag == .top_level) { - const name = step_index.ptr(&configuration).name.slice(&configuration); - try top_level_steps.put(arena, name, step_index); + const flags = conf_step.flags(c); + switch (flags.tag) { + .top_level => { + const name = step_index.ptr(c).name.slice(c); + try top_level_steps.put(arena, name, step_index); + }, + .compile => { + const root_module = step_index.ptr(c).extended.get(configuration.extra).compile.root_module; + try modules.put(arena, root_module, {}); + }, + else => {}, } } break :sc .{ .configuration = configuration, .top_level_steps = top_level_steps, + .modules = modules, }; }; diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index e5ed99506d937f5c0bc0c7d9b5a4019fd6e7ca4a..d9184d77a683a44db0c06713190e66b139844222 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -9,13 +9,13 @@ const Graph = @import("Graph.zig"); configuration: Configuration, top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), +modules: std.AutoArrayHashMapUnmanaged(Configuration.Module.Index, void), pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { std.log.err("TODO also print paths", .{}); std.log.err("TODO also print unlazy deps", .{}); std.log.err("TODO also print system integrations", .{}); std.log.err("TODO also print available options", .{}); - std.log.err("TODO also print modules", .{}); const c = &sc.configuration; var serializer: Serializer = .{ .writer = w }; var s = try serializer.beginStruct(.{}); @@ -31,7 +31,7 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { { var tf = try s.beginTupleField("steps", .{}); - for (c.steps) |*step| { + for (c.steps) |step| { var step_field = try tf.beginStructField(.{}); try printStruct(sc, &step_field, Configuration.Step, step); try step_field.end(); @@ -39,10 +39,23 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { try tf.end(); } + { + var sf = try s.beginStructField("modules", .{}); + + for (sc.modules.keys()) |module_index| { + var int_buf: [50]u8 = undefined; + const int_str = std.fmt.bufPrint(&int_buf, "{d}", .{module_index}) catch unreachable; + var step_field = try sf.beginStructField(int_str, .{}); + try printStruct(sc, &step_field, Configuration.Module, module_index.get(c)); + try step_field.end(); + } + try sf.end(); + } + try s.end(); } -fn printStruct(sc: *const ScannedConfig, s: *Serializer.Struct, comptime S: type, v: *const S) !void { +fn printStruct(sc: *const ScannedConfig, s: *Serializer.Struct, comptime S: type, v: S) !void { inline for (@typeInfo(S).@"struct".fields) |field| { try s.fieldPrefix(field.name); try printValue(sc, s.container.serializer, field.type, @field(v, field.name)); @@ -78,7 +91,11 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi if (@hasDecl(Field, "storage")) switch (Field.storage) { .extended => { var sub_struct = try s.beginStruct(.{}); - try printTaggedUnion(sc, &sub_struct, field_value.get(c.extra)); + switch (field_value.get(c.extra)) { + inline else => |u| { + try printStruct(sc, &sub_struct, @TypeOf(u), u); + }, + } try sub_struct.end(); }, .flag_optional => comptime unreachable, @@ -98,6 +115,11 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi .@"packed" => { try s.value(field_value, .{}); }, + .@"extern" => { + var sub_struct = try s.beginStruct(.{}); + try printStruct(sc, &sub_struct, Field, field_value); + try sub_struct.end(); + }, .auto => switch (Field.storage) { .flag_optional, .enum_optional => { if (field_value.value) |some| { @@ -110,35 +132,41 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try printValue(sc, s, @TypeOf(field_value.slice), field_value.slice); }, .extended => @compileError("TODO"), - .union_list => @compileError("TODO"), + .union_list => { + var slice_field = try s.beginTuple(.{}); + for (field_value.get(c.extra), 0..) |elem, i| switch (field_value.tag(c.extra, i)) { + inline else => |tag| { + var sub_struct = try s.beginStruct(.{}); + try sub_struct.fieldPrefix(@tagName(tag)); + try printValue(sc, s, @FieldType(Field.Union, @tagName(tag)), @enumFromInt(elem)); + try sub_struct.end(); + }, + }; + try slice_field.end(); + }, .flag_union => try printValue(sc, s, Field.Union, field_value.u), .multi_list => @compileError("TODO"), }, - else => @compileError("not implemented: " ++ @typeName(Field)), }, .@"union" => { - switch (field_value) { - inline else => |u, tag| { - if (@TypeOf(u) == void) { - try s.ident(@tagName(tag)); - } else { - var sub_struct = try s.beginStruct(.{}); - try sub_struct.fieldPrefix(@tagName(tag)); - try printValue(sc, s, @TypeOf(u), u); - try sub_struct.end(); - } - }, - } + try printTaggedUnion(sc, s, field_value); }, else => @compileError("not implemented: " ++ @typeName(Field)), }, } } -fn printTaggedUnion(sc: *const ScannedConfig, s: *Serializer.Struct, value: anytype) !void { +fn printTaggedUnion(sc: *const ScannedConfig, s: *Serializer, value: anytype) !void { switch (value) { - inline else => |*u| { - try printStruct(sc, s, @TypeOf(u.*), u); + inline else => |u, tag| { + if (@TypeOf(u) == void) { + try s.ident(@tagName(tag)); + } else { + var sub_struct = try s.beginStruct(.{}); + try sub_struct.fieldPrefix(@tagName(tag)); + try printValue(sc, s, @TypeOf(u), u); + try sub_struct.end(); + } }, } } diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index c1af30eb325ac34a76fdebb5f08a3adcf546f3c3..a0d63a868d76294e3113fd530f6b646d030f345e 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -1219,7 +1219,7 @@ pub const Module = struct { win32_resource_file: RcSourceFile.Index, }; - pub const Framework = struct { + pub const Framework = extern struct { flags: @This().Flags, name: String, @@ -2122,7 +2122,7 @@ pub const Storage = enum { }, .auto => switch (Field.storage) { .flag_optional, .enum_optional, .extended => 1, - .length_prefixed_list, .flag_length_prefixed_list => field.slice.len + 1, + .length_prefixed_list, .flag_length_prefixed_list => 1 + @divExact(@sizeOf(Field.Elem), @sizeOf(u32)) * field.slice.len, .multi_list => 1 + field.mal.len * @typeInfo(Field.Elem).@"struct".fields.len, .union_list => Field.extraLen(field.len), .flag_union => switch (field.u) { -- 2.54.0 From 8f36a83b45eedef92b3fcda605e0ccb3828ddbd3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 4 Mar 2026 17:40:17 -0800 Subject: [PATCH 037/179] Configuration: serialize remaining CSourceFiles information --- lib/compiler/Maker/ScannedConfig.zig | 3 +- lib/compiler/configurer.zig | 34 +++++++++++----- lib/std/zig/Configuration.zig | 61 ++++++++++++++++++++++------ 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index d9184d77a683a44db0c06713190e66b139844222..7a0f96d577bd228661b7d5ba41d015391ea1f292 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -103,6 +103,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi .enum_optional => comptime unreachable, .union_list => comptime unreachable, .length_prefixed_list => comptime unreachable, + .flag_list => comptime unreachable, .flag_union => comptime unreachable, .multi_list => comptime unreachable, } else if (std.enums.tagName(Field, field_value)) |name| { @@ -128,7 +129,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try s.value(null, .{}); } }, - .length_prefixed_list, .flag_length_prefixed_list => { + .length_prefixed_list, .flag_length_prefixed_list, .flag_list => { try printValue(sc, s, @TypeOf(field_value.slice), field_value.slice); }, .extended => @compileError("TODO"), diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index e8548d5aa9b7a26330c34ea20dc140e0d17cafa6..d1178901ef48917c37b778402b1dffe1a47e8728 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -308,40 +308,54 @@ const Serialize = struct { } fn addCSourceFile(s: *Serialize, csf: *const std.Build.Module.CSourceFile) !Configuration.CSourceFile.Index { - log.err("TODO addCSourceFile trailing data", .{}); const wc = s.wc; + const args = try initStringList(s, csf.flags); return @enumFromInt(try wc.addExtra(@as(Configuration.CSourceFile, .{ .flags = .{ - .args_len = @intCast(csf.flags.len), + .args_len = @intCast(args.len), .lang = .init(csf.language), }, .file = try addLazyPath(s, csf.file), + .args = .{ .slice = args }, }))); } fn addCSourceFiles(s: *Serialize, csf: *const std.Build.Module.CSourceFiles) !Configuration.CSourceFiles.Index { - log.err("TODO addCSourceFiles trailing data", .{}); const wc = s.wc; + const sub_paths = try initStringList(s, csf.files); + const args = try initStringList(s, csf.flags); return @enumFromInt(try wc.addExtra(@as(Configuration.CSourceFiles, .{ .flags = .{ - .args_len = @intCast(csf.flags.len), + .args_len = @intCast(args.len), .lang = .init(csf.language), }, .root = try addLazyPath(s, csf.root), - .files_len = @intCast(csf.files.len), + .sub_paths = .{ .slice = sub_paths }, + .args = .{ .slice = args }, }))); } fn addRcSourceFile(s: *Serialize, rsf: *const std.Build.Module.RcSourceFile) !Configuration.RcSourceFile.Index { - log.err("TODO addRcSourceFile trailing data", .{}); const wc = s.wc; + const include_paths = try initLazyPathList(s, rsf.include_paths); + const args = try initStringList(s, rsf.flags); return @enumFromInt(try wc.addExtra(@as(Configuration.RcSourceFile, .{ + .flags = .{ + .args_len = @intCast(args.len), + .include_paths = include_paths.len != 0, + }, .file = try addLazyPath(s, rsf.file), - .args_len = @intCast(rsf.flags.len), - .include_paths_len = @intCast(rsf.include_paths.len), + .include_paths = .{ .slice = include_paths }, + .args = .{ .slice = args }, }))); } + fn initLazyPathList(s: *Serialize, list: []const std.Build.LazyPath) ![]const Configuration.LazyPath { + const result = try s.arena.alloc(Configuration.LazyPath, list.len); + for (result, list) |*dest, src| dest.* = try addLazyPath(s, src); + return result; + } + fn initStringList(s: *Serialize, list: []const []const u8) ![]const Configuration.String { const wc = s.wc; const result = try s.arena.alloc(Configuration.String, list.len); @@ -400,9 +414,7 @@ const Serialize = struct { .name = try wc.addString(name), }; - const lib_paths = try arena.alloc(Configuration.LazyPath, m.lib_paths.items.len); - for (lib_paths, m.lib_paths.items) |*dest, src| dest.* = try addLazyPath(s, src); - + const lib_paths = try initLazyPathList(s, m.lib_paths.items); const c_macros = try initStringList(s, m.c_macros.items); const export_symbol_names = try initStringList(s, m.export_symbol_names); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index a0d63a868d76294e3113fd530f6b646d030f345e..f739dab11c9c3b57604934ed59c1772b51db2b4a 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -1388,13 +1388,11 @@ pub const SystemLib = struct { pub const SearchStrategy = enum(u2) { paths_first, mode_first, no_fallback }; }; -/// Trailing: -/// * arg: String, // for each args_len -/// * sub_path: String, // for each files_len pub const CSourceFiles = struct { flags: Flags, root: LazyPath, - files_len: u32, + args: Storage.FlagList(.flags, .args_len, String), + sub_paths: Storage.LengthPrefixedList(String), pub const Index = enum(u32) { _, @@ -1407,11 +1405,10 @@ pub const CSourceFiles = struct { }; }; -/// Trailing: -/// * arg: String, // for each args_len pub const CSourceFile = struct { flags: Flags, file: LazyPath, + args: Storage.FlagList(.flags, .args_len, String), pub const Index = enum(u32) { _, @@ -1424,17 +1421,21 @@ pub const CSourceFile = struct { }; }; -/// Trailing: -/// * arg: String, // for each args_len -/// * include_path: String, // for each include_paths_len pub const RcSourceFile = struct { + flags: Flags, file: LazyPath, - args_len: u32, - include_paths_len: u32, + args: Storage.FlagList(.flags, .args_len, String), + include_paths: Storage.FlagLengthPrefixedList(.flags, .include_paths, LazyPath), pub const Index = enum(u32) { _, }; + + pub const Flags = packed struct(u32) { + /// C compiler CLI flags. + args_len: u31, + include_paths: bool, + }; }; pub const OptionalCSourceLanguage = enum(u3) { @@ -1779,6 +1780,7 @@ pub const Storage = enum { union_list, flag_union, multi_list, + flag_list, /// The presence of the field is determined by a boolean within a packed /// struct. @@ -1887,6 +1889,26 @@ pub const Storage = enum { }; } + /// The field is a list whose length is an integer inside flags. + pub fn FlagList( + comptime flags_arg: @EnumLiteral(), + comptime flag_arg: @EnumLiteral(), + comptime ElemArg: type, + ) type { + return struct { + slice: []const Elem, + + pub const storage: Storage = .flag_list; + pub const flags = flags_arg; + pub const flag = flag_arg; + pub const Elem = ElemArg; + + pub fn initErased(s: []const u32) @This() { + return .{ .slice = @ptrCast(s) }; + } + }; + } + /// The field contains a u32 length followed by that many items for the /// first field, that many items for the second field, etc. pub fn MultiList(comptime ElemArg: type) type { @@ -2059,6 +2081,13 @@ pub const Storage = enum { defer i.* = data_start + len; return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; }, + .flag_list => { + const flags = @field(container, @tagName(Field.flags)); + const len: u32 = @field(flags, @tagName(Field.flag)); + const data_start = i.*; + defer i.* = data_start + len; + return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; + }, .multi_list => { const data_start = i.* + 1; const len = buffer[data_start - 1]; @@ -2122,7 +2151,10 @@ pub const Storage = enum { }, .auto => switch (Field.storage) { .flag_optional, .enum_optional, .extended => 1, - .length_prefixed_list, .flag_length_prefixed_list => 1 + @divExact(@sizeOf(Field.Elem), @sizeOf(u32)) * field.slice.len, + .length_prefixed_list, + .flag_length_prefixed_list, + .flag_list, + => 1 + @divExact(@sizeOf(Field.Elem), @sizeOf(u32)) * field.slice.len, .multi_list => 1 + field.mal.len * @typeInfo(Field.Elem).@"struct".fields.len, .union_list => Field.extraLen(field.len), .flag_union => switch (field.u) { @@ -2195,6 +2227,11 @@ pub const Storage = enum { @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); return len + 1; }, + .flag_list => { + const len: u32 = @intCast(value.slice.len); + @memcpy(buffer[i..][0..len], @as([]const u32, @ptrCast(value.slice))); + return len; + }, .multi_list => { const len: u32 = @intCast(value.mal.len); if (len == 0) return 0; -- 2.54.0 From eaffd5551349be6132ab33827e307f28ca8ac051 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Mar 2026 22:42:41 -0700 Subject: [PATCH 038/179] maker: progress towards lowering Compile Step CLI args next thing to do is figure out how LazyPath is supposed to work now. something like this: * each Step that provides LazyPath objects has a setLazyPath and getLazyPath function which takes a tagged union identifying which one to access * steps that fulfill LazyPath objects can freely call setLazyPath without obtaining a lock because the dependency graph prevents simultaneous access. * similarly, steps that access LazyPath results can freely call getLazyPath without obtaining a lock, because after modification, there may be simultaneous reads from dependencies but they will all be read-only * a fulfilled LazyPath object is a read-only std.Build.Cache.Path. --- lib/compiler/Maker/ScannedConfig.zig | 2 +- lib/compiler/Maker/Step.zig | 9 +- lib/compiler/Maker/Step/Compile.zig | 688 +++++++++++++++------------ lib/compiler/configurer.zig | 1 + lib/std/Build/Module.zig | 8 +- lib/std/zig/Configuration.zig | 31 +- 6 files changed, 433 insertions(+), 306 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 7a0f96d577bd228661b7d5ba41d015391ea1f292..30a227ab965575b4fb583946bf5e22d3107c2911 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -135,7 +135,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi .extended => @compileError("TODO"), .union_list => { var slice_field = try s.beginTuple(.{}); - for (field_value.get(c.extra), 0..) |elem, i| switch (field_value.tag(c.extra, i)) { + for (field_value.slice(c.extra), 0..) |elem, i| switch (field_value.tag(c.extra, i)) { inline else => |tag| { var sub_struct = try s.beginStruct(.{}); try sub_struct.fieldPrefix(@tagName(tag)); diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 8eda9e1e042ac03a84d66735d775dacb234bded0..c3c064aead143b5a4dd2a3667847929bacecfa3d 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -316,13 +316,14 @@ pub fn captureChildProcess( return result; } -pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { - try step.addError(fmt, args); +pub fn fail(step: *Step, maker: *const Maker, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { + try step.addError(maker, fmt, args); return error.MakeFailed; } -pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { - const arena = step.owner.allocator; +pub fn addError(step: *Step, maker: *const Maker, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process_arena const msg = try std.fmt.allocPrint(arena, fmt, args); try step.result_error_msgs.append(arena, msg); } diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 798d3c05e143931c10eb27e0ff34339d29a13431..f55cf46d006197f29a9b3cc39d08440bb91f2534 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -23,14 +23,17 @@ zig_args: std.ArrayList([]const u8) = .empty, pub fn make( compile: *Compile, - step_index: Configuration.Step.Index, + compile_index: Configuration.Step.Index, maker: *Maker, progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { const graph = maker.graph; - const step = maker.stepByIndex(step_index); + const step = maker.stepByIndex(compile_index); + + // Reset / repopulate persistent state. compile.zig_args.clearRetainingCapacity(); - try lowerZigArgs(compile, step_index, maker, &compile.zig_args, false); + + try lowerZigArgs(compile, compile_index, maker, &compile.zig_args, false); if (true) @panic("TODO implement compile.make()"); const process_arena = graph.arena; // TODO don't leak into the process_arena @@ -42,7 +45,7 @@ pub fn make( ) catch |err| switch (err) { error.NeedCompileErrorCheck => { assert(compile.expect_errors != null); - try checkCompileErrors(compile); + try checkCompileErrors(compile, maker); return; }, else => |e| return e, @@ -81,19 +84,50 @@ pub fn make( } } +/// List of importable modules in a compilation's module graph, including +/// the root module. The root module is guaranteed to be first. +const ModuleList = std.AutoArrayHashMapUnmanaged(Configuration.Module.Index, Configuration.String); +/// Keyed on the first key in the module list. +const ModuleGraph = std.ArrayHashMapUnmanaged(ModuleList, void, ModuleListContext, false); + +const ModuleListContext = struct { + pub fn eql(ctx: @This(), a: ModuleList, b: ModuleList) bool { + _ = ctx; + return a.keys()[0] == b.keys()[0]; + } + + pub fn hash(ctx: @This(), key: ModuleList) u32 { + _ = ctx; + return std.hash.int(@intFromEnum(key.keys()[0])); + } + + const Adapter = struct { + pub fn eql(ctx: @This(), a: Configuration.Module.Index, b: ModuleList, b_index: usize) bool { + _ = ctx; + _ = b_index; + return a == b.keys()[0]; + } + + pub fn hash(ctx: @This(), key: Configuration.Module.Index) u32 { + _ = ctx; + return std.hash.int(@intFromEnum(key)); + } + }; +}; + fn lowerZigArgs( - compile: *Compile, - step_index: Configuration.Step.Index, - maker: *Maker, + compile: *const Compile, + compile_index: Configuration.Step.Index, + maker: *const Maker, zig_args: *std.ArrayList([]const u8), fuzz: bool, -) Allocator.Error!void { - const step = maker.stepByIndex(step_index); +) error{ OutOfMemory, MakeFailed }!void { + const step = maker.stepByIndex(compile_index); const graph = maker.graph; const arena = graph.arena; // TODO don't leak into the process arena const gpa = maker.gpa; const conf = &maker.scanned_config.configuration; - const conf_step = step_index.ptr(conf); + const conf_step = compile_index.ptr(conf); const conf_comp = conf_step.extended.get(conf.extra).compile; try zig_args.append(gpa, graph.zig_exe); @@ -144,21 +178,21 @@ fn lowerZigArgs( try addBool(gpa, zig_args, "-ffuzz", fuzz); - if (true) @panic("TODO"); - - var is_linking_libc = conf_comp.flags3.is_linking_libc; - var is_linking_libcpp = conf_comp.flags3.is_linking_libcpp; - { + var is_linking_libc = conf_comp.flags3.is_linking_libc; + var is_linking_libcpp = conf_comp.flags3.is_linking_libcpp; + // Stores system libraries that have already been seen for at least one - // module, along with any arguments that need to be passed to the - // compiler for each module individually. - var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; - var frameworks: std.StringArrayHashMapUnmanaged(Module.FrameworkFlags) = .empty; + // module, along with any C compiler arguments that need to be passed + // to the compiler for each module individually as reported by + // pkg-config. + var seen_system_libs: std.AutoArrayHashMapUnmanaged(Configuration.String, []const []const u8) = .empty; + var frameworks: std.AutoArrayHashMapUnmanaged(Configuration.String, Configuration.Module.Framework.Flags) = .empty; + var module_graph: ModuleGraph = .empty; var prev_has_cflags = false; var prev_has_rcflags = false; - var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; + var prev_search_strategy: Configuration.SystemLib.SearchStrategy = .paths_first; var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; // Track the number of positional arguments so that a nice error can be // emitted if there is nothing to link. @@ -166,250 +200,256 @@ fn lowerZigArgs( // Fully recursive iteration including dynamic libraries to detect // libc and libc++ linkage. - for (getCompileDependencies(true)) |some_compile| { - for (some_compile.root_module.getGraph().modules) |mod| { - if (mod.link_libc == true) is_linking_libc = true; - if (mod.link_libcpp == true) is_linking_libcpp = true; + for (try getCompileDependencies(arena, &module_graph, conf, compile_index, true)) |some_compile_index| { + const some_compile = some_compile_index.ptr(conf).extended.get(conf.extra).compile; + const modules = try getModuleList(arena, &module_graph, some_compile.root_module, conf); + for (modules.keys()) |mod_index| { + const mod = mod_index.get(conf); + is_linking_libc = is_linking_libc or mod.flags2.link_libc == .true; + is_linking_libcpp = is_linking_libcpp or mod.flags2.link_libcpp == .true; } } - var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); + var cli_named_modules = try CliNamedModules.init(arena, &module_graph, compile_index, maker); // For this loop, don't chase dynamic libraries because their link // objects are already linked. - for (getCompileDependencies(false)) |dep_compile| { - for (dep_compile.root_module.getGraph().modules) |mod| { + for (try getCompileDependencies(arena, &module_graph, conf, compile_index, false)) |dep_compile_index| { + const dep_compile = dep_compile_index.ptr(conf).extended.get(conf.extra).compile; + const modules = try getModuleList(arena, &module_graph, dep_compile.root_module, conf); + for (modules.keys()) |mod_index| { + const mod = mod_index.get(conf); // While walking transitive dependencies, if a given link object is // already included in a library, it should not redundantly be // placed on the linker line of the dependee. - const my_responsibility = dep_compile == compile; + const my_responsibility = dep_compile_index == compile_index; const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); // Inherit dependencies on darwin frameworks. if (!already_linked) { - for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { - try frameworks.put(arena, name, info); + for (mod.frameworks.slice) |framework| { + try frameworks.put(arena, framework.name, framework.flags); } } + if (true) @panic("TODO"); + // Inherit dependencies on system libraries and static libraries. - for (mod.link_objects.items) |link_object| { - switch (link_object) { - .static_path => |static_path| { - if (my_responsibility) { - try zig_args.append(gpa, static_path.getPath2(step)); - total_linker_objects += 1; - } - }, - .system_lib => |system_lib| { - const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); - if (system_lib_gop.found_existing) { - try zig_args.appendSlice(gpa, system_lib_gop.value_ptr.*); - continue; - } else { - system_lib_gop.value_ptr.* = &.{}; - } + for (0..mod.link_objects.len) |lo_i| switch (mod.link_objects.get(conf.extra, lo_i)) { + .static_path => |static_path| { + if (my_responsibility) { + try zig_args.append(gpa, static_path.getPath2(step)); + total_linker_objects += 1; + } + }, + .system_lib => |system_lib| { + const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); + if (system_lib_gop.found_existing) { + try zig_args.appendSlice(gpa, system_lib_gop.value_ptr.*); + continue; + } else { + system_lib_gop.value_ptr.* = &.{}; + } - if (already_linked) - continue; + if (already_linked) + continue; - if ((system_lib.search_strategy != prev_search_strategy or - system_lib.preferred_link_mode != prev_preferred_link_mode) and - compile.linkage != .static) - { - switch (system_lib.search_strategy) { - .no_fallback => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append(gpa, "-search_dylibs_only"), - .static => try zig_args.append(gpa, "-search_static_only"), - }, - .paths_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append(gpa, "-search_paths_first"), - .static => try zig_args.append(gpa, "-search_paths_first_static"), - }, - .mode_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append(gpa, "-search_dylibs_first"), - .static => try zig_args.append(gpa, "-search_static_first"), + if ((system_lib.search_strategy != prev_search_strategy or + system_lib.preferred_link_mode != prev_preferred_link_mode) and + compile.linkage != .static) + { + switch (system_lib.search_strategy) { + .no_fallback => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append(gpa, "-search_dylibs_only"), + .static => try zig_args.append(gpa, "-search_static_only"), + }, + .paths_first => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append(gpa, "-search_paths_first"), + .static => try zig_args.append(gpa, "-search_paths_first_static"), + }, + .mode_first => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append(gpa, "-search_dylibs_first"), + .static => try zig_args.append(gpa, "-search_static_first"), + }, + } + prev_search_strategy = system_lib.search_strategy; + prev_preferred_link_mode = system_lib.preferred_link_mode; + } + + const prefix: []const u8 = prefix: { + if (system_lib.needed) break :prefix "-needed-l"; + if (system_lib.weak) break :prefix "-weak-l"; + break :prefix "-l"; + }; + switch (system_lib.use_pkg_config) { + .no => try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ prefix, system_lib.name })), + .yes, .force => { + if (compile.runPkgConfig(maker, system_lib.name)) |result| { + try zig_args.appendSlice(gpa, result.cflags); + try zig_args.appendSlice(gpa, result.libs); + try seen_system_libs.put(arena, system_lib.name, result.cflags); + } else |err| switch (err) { + error.PkgConfigInvalidOutput, + error.PkgConfigCrashed, + error.PkgConfigFailed, + error.PkgConfigNotInstalled, + error.PackageNotFound, + => switch (system_lib.use_pkg_config) { + .yes => { + // pkg-config failed, so fall back to linking the library + // by name directly. + try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ + prefix, + system_lib.name, + })); + }, + .force => { + return step.fail(maker, "pkg-config failed for library {s}", .{system_lib.name}); + }, + .no => unreachable, }, + + else => |e| return e, } - prev_search_strategy = system_lib.search_strategy; - prev_preferred_link_mode = system_lib.preferred_link_mode; - } - - const prefix: []const u8 = prefix: { - if (system_lib.needed) break :prefix "-needed-l"; - if (system_lib.weak) break :prefix "-weak-l"; - break :prefix "-l"; - }; - switch (system_lib.use_pkg_config) { - .no => try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ prefix, system_lib.name })), - .yes, .force => { - if (compile.runPkgConfig(maker, system_lib.name)) |result| { - try zig_args.appendSlice(gpa, result.cflags); - try zig_args.appendSlice(gpa, result.libs); - try seen_system_libs.put(arena, system_lib.name, result.cflags); - } else |err| switch (err) { - error.PkgConfigInvalidOutput, - error.PkgConfigCrashed, - error.PkgConfigFailed, - error.PkgConfigNotInstalled, - error.PackageNotFound, - => switch (system_lib.use_pkg_config) { - .yes => { - // pkg-config failed, so fall back to linking the library - // by name directly. - try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ - prefix, - system_lib.name, - })); - }, - .force => { - return step.fail("pkg-config failed for library {s}", .{system_lib.name}); - }, - .no => unreachable, - }, - - else => |e| return e, - } - }, - } - }, - .other_step => |other| { - switch (other.kind) { - .exe => return step.fail("cannot link with an executable build artifact", .{}), - .@"test" => return step.fail("cannot link with a test", .{}), - .obj, .test_obj => { - const included_in_lib_or_obj = !my_responsibility and - (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); - if (!already_linked and !included_in_lib_or_obj) { - try zig_args.append(gpa, other.getEmittedBin().getPath2(step)); - total_linker_objects += 1; - } - }, - .lib => l: { - const other_produces_implib = other.producesImplib(); - const other_is_static = other_produces_implib or other.isStaticLibrary(); - - if (compile.isStaticLibrary() and other_is_static) { - // Avoid putting a static library inside a static library. - break :l; - } - - // For DLLs, we must link against the implib. - // For everything else, we directly link - // against the library file. - const full_path_lib = if (other_produces_implib) - try other.getGeneratedFilePath("generated_implib", &compile.step) - else - try other.getGeneratedFilePath("generated_bin", &compile.step); - - try zig_args.append(gpa, full_path_lib); + }, + } + }, + .other_step => |other| { + switch (other.kind) { + .exe => return step.fail(maker, "cannot link with an executable build artifact", .{}), + .@"test" => return step.fail(maker, "cannot link with a test", .{}), + .obj, .test_obj => { + const included_in_lib_or_obj = !my_responsibility and + (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); + if (!already_linked and !included_in_lib_or_obj) { + try zig_args.append(gpa, other.getEmittedBin().getPath2(step)); total_linker_objects += 1; + } + }, + .lib => l: { + const other_produces_implib = other.producesImplib(); + const other_is_static = other_produces_implib or other.isStaticLibrary(); - if (other.linkage == .dynamic and - compile.rootModuleTarget().os.tag != .windows) - { - if (Dir.path.dirname(full_path_lib)) |dirname| { - try zig_args.append(gpa, "-rpath"); - try zig_args.append(gpa, dirname); - } + if (compile.isStaticLibrary() and other_is_static) { + // Avoid putting a static library inside a static library. + break :l; + } + + // For DLLs, we must link against the implib. + // For everything else, we directly link + // against the library file. + const full_path_lib = if (other_produces_implib) + try other.getGeneratedFilePath("generated_implib", &compile.step) + else + try other.getGeneratedFilePath("generated_bin", &compile.step); + + try zig_args.append(gpa, full_path_lib); + total_linker_objects += 1; + + if (other.linkage == .dynamic and + compile.rootModuleTarget().os.tag != .windows) + { + if (Dir.path.dirname(full_path_lib)) |dirname| { + try zig_args.append(gpa, "-rpath"); + try zig_args.append(gpa, dirname); } - }, - } - }, - .assembly_file => |asm_file| l: { - if (!my_responsibility) break :l; + } + }, + } + }, + .assembly_file => |asm_file| l: { + if (!my_responsibility) break :l; - if (prev_has_cflags) { - try zig_args.append(gpa, "-cflags"); - try zig_args.append(gpa, "--"); - prev_has_cflags = false; - } - try zig_args.append(gpa, asm_file.getPath2(mod.owner, step)); - total_linker_objects += 1; - }, + if (prev_has_cflags) { + try zig_args.append(gpa, "-cflags"); + try zig_args.append(gpa, "--"); + prev_has_cflags = false; + } + try zig_args.append(gpa, asm_file.getPath2(mod.owner, step)); + total_linker_objects += 1; + }, - .c_source_file => |c_source_file| l: { - if (!my_responsibility) break :l; + .c_source_file => |c_source_file| l: { + if (!my_responsibility) break :l; - if (prev_has_cflags or c_source_file.flags.len != 0) { - try zig_args.append(gpa, "-cflags"); - for (c_source_file.flags) |arg| { - try zig_args.append(gpa, arg); - } - try zig_args.append(gpa, "--"); + if (prev_has_cflags or c_source_file.flags.len != 0) { + try zig_args.append(gpa, "-cflags"); + for (c_source_file.flags) |arg| { + try zig_args.append(gpa, arg); } - prev_has_cflags = (c_source_file.flags.len != 0); + try zig_args.append(gpa, "--"); + } + prev_has_cflags = (c_source_file.flags.len != 0); - if (c_source_file.language) |lang| { - try zig_args.append(gpa, "-x"); - try zig_args.append(gpa, lang.internalIdentifier()); - } + if (c_source_file.language) |lang| { + try zig_args.append(gpa, "-x"); + try zig_args.append(gpa, lang.internalIdentifier()); + } - try zig_args.append(gpa, c_source_file.file.getPath2(mod.owner, step)); + try zig_args.append(gpa, c_source_file.file.getPath2(mod.owner, step)); - if (c_source_file.language != null) { - try zig_args.append(gpa, "-x"); - try zig_args.append(gpa, "none"); - } - total_linker_objects += 1; - }, + if (c_source_file.language != null) { + try zig_args.append(gpa, "-x"); + try zig_args.append(gpa, "none"); + } + total_linker_objects += 1; + }, - .c_source_files => |c_source_files| l: { - if (!my_responsibility) break :l; + .c_source_files => |c_source_files| l: { + if (!my_responsibility) break :l; - if (prev_has_cflags or c_source_files.flags.len != 0) { - try zig_args.append(gpa, "-cflags"); - for (c_source_files.flags) |arg| { - try zig_args.append(gpa, arg); - } - try zig_args.append(gpa, "--"); + if (prev_has_cflags or c_source_files.flags.len != 0) { + try zig_args.append(gpa, "-cflags"); + for (c_source_files.flags) |arg| { + try zig_args.append(gpa, arg); } - prev_has_cflags = (c_source_files.flags.len != 0); + try zig_args.append(gpa, "--"); + } + prev_has_cflags = (c_source_files.flags.len != 0); - if (c_source_files.language) |lang| { - try zig_args.append(gpa, "-x"); - try zig_args.append(gpa, lang.internalIdentifier()); - } + if (c_source_files.language) |lang| { + try zig_args.append(gpa, "-x"); + try zig_args.append(gpa, lang.internalIdentifier()); + } - const root_path = c_source_files.root.getPath2(mod.owner, step); - for (c_source_files.files) |file| { - try zig_args.append(gpa, try Dir.path.join(arena, &.{ root_path, file })); - } + const root_path = c_source_files.root.getPath2(mod.owner, step); + for (c_source_files.files) |file| { + try zig_args.append(gpa, try Dir.path.join(arena, &.{ root_path, file })); + } - if (c_source_files.language != null) { - try zig_args.append(gpa, "-x"); - try zig_args.append(gpa, "none"); - } + if (c_source_files.language != null) { + try zig_args.append(gpa, "-x"); + try zig_args.append(gpa, "none"); + } - total_linker_objects += c_source_files.files.len; - }, + total_linker_objects += c_source_files.files.len; + }, - .win32_resource_file => |rc_source_file| l: { - if (!my_responsibility) break :l; + .win32_resource_file => |rc_source_file| l: { + if (!my_responsibility) break :l; - if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { - if (prev_has_rcflags) { - try zig_args.append(gpa, "-rcflags"); - try zig_args.append(gpa, "--"); - prev_has_rcflags = false; - } - } else { + if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { + if (prev_has_rcflags) { try zig_args.append(gpa, "-rcflags"); - for (rc_source_file.flags) |arg| { - try zig_args.append(gpa, arg); - } - for (rc_source_file.include_paths) |include_path| { - try zig_args.append(gpa, "/I"); - try zig_args.append(gpa, include_path.getPath2(mod.owner, step)); - } try zig_args.append(gpa, "--"); - prev_has_rcflags = true; + prev_has_rcflags = false; } - try zig_args.append(gpa, rc_source_file.file.getPath2(mod.owner, step)); - total_linker_objects += 1; - }, - } - } + } else { + try zig_args.append(gpa, "-rcflags"); + for (rc_source_file.flags) |arg| { + try zig_args.append(gpa, arg); + } + for (rc_source_file.include_paths) |include_path| { + try zig_args.append(gpa, "/I"); + try zig_args.append(gpa, include_path.getPath2(mod.owner, step)); + } + try zig_args.append(gpa, "--"); + prev_has_rcflags = true; + } + try zig_args.append(gpa, rc_source_file.file.getPath2(mod.owner, step)); + total_linker_objects += 1; + }, + }; // We need to emit the --mod argument here so that the above link objects // have the correct parent module, but only if the module is part of @@ -450,48 +490,47 @@ fn lowerZigArgs( } if (total_linker_objects == 0) { - return step.fail("the linker needs one or more objects to link", .{}); + return step.fail(maker, "the linker needs one or more objects to link", .{}); } for (frameworks.keys(), frameworks.values()) |name, info| { + try zig_args.ensureUnusedCapacity(gpa, 2); if (info.needed) { - try zig_args.append(gpa, "-needed_framework"); + zig_args.appendAssumeCapacity("-needed_framework"); } else if (info.weak) { - try zig_args.append(gpa, "-weak_framework"); + zig_args.appendAssumeCapacity("-weak_framework"); } else { - try zig_args.append(gpa, "-framework"); + zig_args.appendAssumeCapacity("-framework"); } - try zig_args.append(gpa, name); + zig_args.appendAssumeCapacity(name.slice(conf)); } - if (is_linking_libcpp) { - try zig_args.append(gpa, "-lc++"); - } - - if (is_linking_libc) { - try zig_args.append(gpa, "-lc"); - } + try zig_args.ensureUnusedCapacity(gpa, 2); + if (is_linking_libcpp) zig_args.appendAssumeCapacity("-lc++"); + if (is_linking_libc) zig_args.appendAssumeCapacity("-lc"); } - if (compile.win32_manifest) |manifest_file| { + if (true) @panic("TODO"); + + if (conf_comp.win32_manifest) |manifest_file| { try zig_args.append(gpa, manifest_file.getPath2(step)); } - if (compile.win32_module_definition) |module_file| { + if (conf_comp.win32_module_definition) |module_file| { try zig_args.append(gpa, module_file.getPath2(step)); } - if (compile.image_base) |image_base| { + if (conf_comp.image_base) |image_base| { try zig_args.appendSlice(gpa, &.{ "--image-base", try allocPrint(arena, "0x{x}", .{image_base}), }); } - for (compile.filters) |filter| { + for (conf_comp.filters) |filter| { try zig_args.appendSlice(gpa, &.{ "--test-filter", filter }); } - if (compile.test_runner) |test_runner| { + if (conf_comp.test_runner) |test_runner| { try zig_args.appendSlice(gpa, &.{ "--test-runner", test_runner.path.getPath2(step) }); } @@ -503,8 +542,8 @@ fn lowerZigArgs( try addBool(gpa, zig_args, "--debug-incremental", graph.debug_incremental); try addBool(gpa, zig_args, "--verbose-air", graph.verbose_air); try addBool(gpa, zig_args, "--verbose-llvm-ir", graph.verbose_llvm_ir); - try addBool(gpa, zig_args, "--verbose-link", graph.verbose_link or compile.verbose_link); - try addBool(gpa, zig_args, "--verbose-cc", graph.verbose_cc or compile.verbose_cc); + try addBool(gpa, zig_args, "--verbose-link", graph.verbose_link or conf_comp.flags.verbose_link); + try addBool(gpa, zig_args, "--verbose-cc", graph.verbose_cc or conf_comp.flags.verbose_cc); try addBool(gpa, zig_args, "--verbose-llvm-cpu-features", graph.verbose_llvm_cpu_features); try addBool(gpa, zig_args, "--time-report", graph.time_report); @@ -516,49 +555,49 @@ fn lowerZigArgs( if (compile.generated_llvm_ir != null) try zig_args.append(gpa, "-femit-llvm-ir"); if (compile.generated_h != null) try zig_args.append(gpa, "-femit-h"); - try addFlag(gpa, zig_args, "formatted-panics", compile.formatted_panics); + try addFlag(gpa, zig_args, "formatted-panics", conf_comp.flags.formatted_panics); - switch (compile.compress_debug_sections) { + switch (conf_comp.compress_debug_sections) { .none => {}, .zlib => try zig_args.append(gpa, "--compress-debug-sections=zlib"), .zstd => try zig_args.append(gpa, "--compress-debug-sections=zstd"), } - if (compile.link_eh_frame_hdr) { + if (conf_comp.flags.link_eh_frame_hdr) { try zig_args.append(gpa, "--eh-frame-hdr"); } - if (compile.link_emit_relocs) { + if (conf_comp.flags.link_emit_relocs) { try zig_args.append(gpa, "--emit-relocs"); } - if (compile.link_function_sections) { + if (conf_comp.flags.link_function_sections) { try zig_args.append(gpa, "-ffunction-sections"); } - if (compile.link_data_sections) { + if (conf_comp.flags.link_data_sections) { try zig_args.append(gpa, "-fdata-sections"); } - if (compile.link_gc_sections) |x| { + if (conf_comp.flags.link_gc_sections) |x| { try zig_args.append(gpa, if (x) "--gc-sections" else "--no-gc-sections"); } - if (!compile.linker_dynamicbase) { + if (!conf_comp.flags.linker_dynamicbase) { try zig_args.append(gpa, "--no-dynamicbase"); } - if (compile.linker_allow_shlib_undefined) |x| { + if (conf_comp.flags.linker_allow_shlib_undefined) |x| { try zig_args.append(gpa, if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); } - if (compile.link_z_notext) try zig_args.appendSlice(gpa, &.{ "-z", "notext" }); - if (!compile.link_z_relro) try zig_args.appendSlice(gpa, &.{ "-z", "norelro" }); - if (compile.link_z_lazy) try zig_args.appendSlice(gpa, &.{ "-z", "lazy" }); - if (compile.link_z_common_page_size) |size| try zig_args.appendSlice(gpa, &.{ + if (conf_comp.flags.link_z_notext) try zig_args.appendSlice(gpa, &.{ "-z", "notext" }); + if (!conf_comp.flags.link_z_relro) try zig_args.appendSlice(gpa, &.{ "-z", "norelro" }); + if (conf_comp.flags.link_z_lazy) try zig_args.appendSlice(gpa, &.{ "-z", "lazy" }); + if (conf_comp.flags.link_z_common_page_size) |size| try zig_args.appendSlice(gpa, &.{ "-z", try allocPrint(arena, "common-page-size={d}", .{size}), }); - if (compile.link_z_max_page_size) |size| try zig_args.appendSlice(gpa, &.{ + if (conf_comp.flags.link_z_max_page_size) |size| try zig_args.appendSlice(gpa, &.{ "-z", try allocPrint(arena, "max-page-size={d}", .{size}), }); - if (compile.link_z_defs) try zig_args.appendSlice(gpa, &.{ "-z", "defs" }); + if (conf_comp.flags.link_z_defs) try zig_args.appendSlice(gpa, &.{ "-z", "defs" }); - if (compile.libc_file) |libc_file| { + if (conf_comp.flags.libc_file) |libc_file| { try zig_args.appendSlice(gpa, &.{ "--libc", libc_file.getPath2(step) }); } else if (graph.libc_file) |libc_file| { try zig_args.appendSlice(gpa, &.{ "--libc", libc_file }); @@ -573,18 +612,16 @@ fn lowerZigArgs( if (graph.debug_compiler_runtime_libs) |mode| try zig_args.append(gpa, try allocPrint(arena, "--debug-rt={t}", .{mode})); - try zig_args.append(gpa, "--name"); - try zig_args.append(gpa, compile.name); + try zig_args.appendSlice(gpa, &.{ "--name", conf_comp.root_name.slice(conf) }); if (compile.linkage) |some| switch (some) { .dynamic => try zig_args.append(gpa, "-dynamic"), .static => try zig_args.append(gpa, "-static"), }; if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { - if (compile.version) |version| { - try zig_args.append(gpa, "--version"); - try zig_args.append(gpa, try allocPrint(arena, "{f}", .{version})); - } + if (compile.version) |version| try zig_args.appendSlice(gpa, &.{ + "--version", try allocPrint(arena, "{f}", .{version}), + }); if (compile.rootModuleTarget().os.tag.isDarwin()) { const install_name = compile.install_name orelse try allocPrint(arena, "@rpath/{s}{s}{s}", .{ @@ -696,7 +733,7 @@ fn lowerZigArgs( for (graph.search_prefixes.items) |search_prefix| { var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { - return step.fail("unable to open prefix directory '{s}': {t}", .{ search_prefix, err }); + return step.fail(maker, "unable to open prefix directory '{s}': {t}", .{ search_prefix, err }); }; defer prefix_dir.close(io); @@ -710,7 +747,7 @@ fn lowerZigArgs( }); } else |err| switch (err) { error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/lib' directory: {t}", .{ search_prefix, e }), + else => |e| return step.fail(maker, "unable to access '{s}/lib' directory: {t}", .{ search_prefix, e }), } if (prefix_dir.access(io, "include", .{})) |_| { @@ -719,7 +756,7 @@ fn lowerZigArgs( }); } else |err| switch (err) { error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/include' directory: {t}", .{ search_prefix, e }), + else => |e| return step.fail(maker, "unable to access '{s}/include' directory: {t}", .{ search_prefix, e }), } } @@ -825,13 +862,13 @@ fn lowerZigArgs( var af = graph.cache_root.handle.createFileAtomic(io, args_file, .{ .replace = false, .make_path = true, - }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ + }) catch |e| return step.fail(maker, "failed creating tmp args file {f}{s}: {t}", .{ graph.cache_root, args_file, e, }); defer af.deinit(io); af.file.writeStreamingAll(io, args) catch |e| { - return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ + return step.fail(maker, "failed writing args data to tmp file {f}{s}: {t}", .{ graph.cache_root, args_file, e, }); }; @@ -842,7 +879,7 @@ fn lowerZigArgs( error.PathAlreadyExists => { // The args file was created by another concurrent build process. }, - else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ + else => |other_err| return step.fail(maker, "failed linking tmp file {f}{s}: {t}", .{ graph.cache_root, args_file, other_err, }), }; @@ -899,14 +936,14 @@ pub fn doAtomicSymLinks( const major_only_path = try Dir.path.join(arena, &.{ out_dir, filename_major_only }); const cwd: Io.Dir = .cwd(); cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { - return step.fail("unable to symlink {s} -> {s}: {t}", .{ + return step.fail(maker, "unable to symlink {s} -> {s}: {t}", .{ major_only_path, out_basename, err, }); }; // sym link for libfoo.so to libfoo.so.1 const name_only_path = try Dir.path.join(arena, &.{ out_dir, filename_name_only }); cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { - return step.fail("unable to symlink {s} -> {s}: {t}", .{ + return step.fail(maker, "unable to symlink {s} -> {s}: {t}", .{ name_only_path, filename_major_only, err, }); }; @@ -1080,7 +1117,7 @@ fn runPkgConfig(compile: *Compile, maker: *Maker, lib_name: []const u8) !PkgConf } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); } else if (b.debug_pkg_config) { - return compile.step.fail("unknown pkg-config flag '{s}'", .{arg}); + return compile.step.fail(maker, "unknown pkg-config flag '{s}'", .{arg}); } } @@ -1093,7 +1130,7 @@ fn runPkgConfig(compile: *Compile, maker: *Maker, lib_name: []const u8) !PkgConf }; } -fn checkCompileErrors(compile: *Compile) !void { +fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { // Clear this field so that it does not get printed by the build runner. const actual_eb = compile.step.result_error_bundle; compile.step.result_error_bundle = .empty; @@ -1120,7 +1157,7 @@ fn checkCompileErrors(compile: *Compile) !void { switch (expect_errors) { .starts_with => |expect_starts_with| { if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return; - return compile.step.fail( + return compile.step.fail(maker, \\ \\========= should start with: ============ \\{s} @@ -1135,7 +1172,7 @@ fn checkCompileErrors(compile: *Compile) !void { return; } - return compile.step.fail( + return compile.step.fail(maker, \\ \\========= should contain: =============== \\{s} @@ -1158,7 +1195,7 @@ fn checkCompileErrors(compile: *Compile) !void { return; } - return compile.step.fail( + return compile.step.fail(maker, \\ \\========= should contain: =============== \\{s} @@ -1185,7 +1222,7 @@ fn checkCompileErrors(compile: *Compile) !void { if (mem.eql(u8, expected_generated.items, actual_errors)) return; - return compile.step.fail( + return compile.step.fail(maker, \\ \\========= expected: ===================== \\{s} @@ -1222,42 +1259,109 @@ fn moduleNeedsCliArg(mod: *const Module) bool { } const CliNamedModules = struct { - modules: std.AutoArrayHashMapUnmanaged(*Module, void), + modules: std.AutoArrayHashMapUnmanaged(Configuration.Module.Index, void), names: std.StringArrayHashMapUnmanaged(void), /// Traverse the whole dependency graph and give every module a unique /// name, ideally one named after what it's called somewhere in the graph. /// It will help here to have both a mapping from module to name and a set /// of all the currently-used names. - fn init(arena: Allocator, root_module: *Module) Allocator.Error!CliNamedModules { - var compile: CliNamedModules = .{ + fn init( + arena: Allocator, + module_graph: *ModuleGraph, + compile_index: Configuration.Step.Index, + maker: *const Maker, + ) Allocator.Error!CliNamedModules { + const conf = &maker.scanned_config.configuration; + const conf_compile = compile_index.ptr(conf).extended.get(conf.extra).compile; + + var result: CliNamedModules = .{ .modules = .{}, .names = .{}, }; - const graph = root_module.getGraph(); + const modules = try getModuleList(arena, module_graph, conf_compile.root_module, conf); { - assert(graph.modules[0] == root_module); - try compile.modules.put(arena, root_module, {}); - try compile.names.put(arena, "root", {}); + assert(conf_compile.root_module == modules.keys()[0]); + try result.modules.put(arena, conf_compile.root_module, {}); + try result.names.put(arena, "root", {}); } - for (graph.modules[1..], graph.names[1..]) |mod, orig_name| { - var name = orig_name; + for (modules.keys()[1..], modules.values()[1..]) |mod, orig_name| { + const orig_name_slice = orig_name.slice(conf); + var name: []const u8 = orig_name_slice; var n: usize = 0; while (true) { - const gop = try compile.names.getOrPut(arena, name); + const gop = try result.names.getOrPut(arena, name); if (!gop.found_existing) { - try compile.modules.putNoClobber(arena, mod, {}); + try result.modules.putNoClobber(arena, mod, {}); break; } - name = try allocPrint(arena, "{s}{d}", .{ orig_name, n }); + name = try allocPrint(arena, "{s}{d}", .{ orig_name_slice, n }); n += 1; } } - return compile; + return result; } }; -fn getCompileDependencies(chase_dynamic: bool) void { - _ = chase_dynamic; - @panic("TODO"); +fn getCompileDependencies( + arena: Allocator, + module_graph: *ModuleGraph, + conf: *const Configuration, + start: Configuration.Step.Index, + chase_dynamic: bool, +) ![]const Configuration.Step.Index { + var compiles: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void) = .empty; + var compiles_i: usize = 0; + + try compiles.putNoClobber(arena, start, {}); + + while (compiles_i < compiles.count()) : (compiles_i += 1) { + const step = compiles.keys()[compiles_i].ptr(conf); + const compile = step.extended.get(conf.extra).compile; + const modules = try getModuleList(arena, module_graph, compile.root_module, conf); + + for (modules.keys()) |mod_index| { + const mod = mod_index.get(conf); + for (0..mod.link_objects.len) |i| { + switch (mod.link_objects.get(conf.extra, i)) { + .other_step => |other_compile_index| { + const other_compile = other_compile_index.ptr(conf).extended.get(conf.extra).compile; + if (!chase_dynamic and other_compile.isDynamicLibrary()) continue; + try compiles.put(arena, other_compile_index, {}); + }, + else => {}, + } + } + } + } + + return compiles.keys(); +} + +/// Returned pointer expires upon next call to `getModuleList`. +fn getModuleList( + arena: Allocator, + module_graph: *ModuleGraph, + root_module: Configuration.Module.Index, + conf: *const Configuration, +) !*ModuleList { + const gop = try module_graph.getOrPutAdapted(arena, root_module, @as(ModuleListContext.Adapter, .{})); + const modules = gop.key_ptr; + + if (gop.found_existing) return modules; + modules.* = .empty; + try modules.putNoClobber(arena, root_module, .root); + + var i: usize = 0; + + while (i < modules.entries.len) : (i += 1) { + const dep_index = modules.keys()[i]; + const dep = dep_index.get(conf); + const imports = dep.import_table.get(conf).imports; + try modules.ensureUnusedCapacity(arena, imports.mal.len); + for (imports.mal.items(.name), imports.mal.items(.module)) |import_name, other_mod| + modules.putAssumeCapacity(other_mod, import_name); + } + + return modules; } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index d1178901ef48917c37b778402b1dffe1a47e8728..bea788fba183691ae505857a21daf7e78e7488bc 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -202,6 +202,7 @@ pub fn main(init: process.Init.Minimal) !void { var wc: Configuration.Wip = .init(gpa); defer wc.deinit(); assert(try wc.addString("") == .empty); + assert(try wc.addString("root") == .root); try serializeSystemIntegrationOptions(&graph, &wc); diff --git a/lib/std/Build/Module.zig b/lib/std/Build/Module.zig index dd3b6b0251aa02dce746ac68280bc7eab8a4a10b..2074eb85db2786cb9271b3c8638521676a548bde 100644 --- a/lib/std/Build/Module.zig +++ b/lib/std/Build/Module.zig @@ -668,11 +668,9 @@ pub const Graph = struct { names: []const []const u8, }; -/// Intended to be used during the make phase only. -/// -/// Given that `root` is the root `Module` of a compilation, return all `Module`s -/// in the module graph, including `root` itself. `root` is guaranteed to be the -/// first module in the returned slice. +/// Given that `root` is the root `Module` of a compilation, return all +/// `Module` in the module graph, including `root` itself. `root` is guaranteed +/// to be the first module in the returned slice. pub fn getGraph(root: *Module) Graph { if (root.cached_graph.modules.len != 0) { return root.cached_graph; diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index f739dab11c9c3b57604934ed59c1772b51db2b4a..23d027e77715586f981171d85c68ed7effd9566c 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -859,6 +859,10 @@ pub const Step = extern struct { version_script: bool, _: u18 = 0, }; + + pub fn isDynamicLibrary(compile: *const Compile) bool { + return compile.flags3.kind == .lib and compile.flags2.linkage == .dynamic; + } }; pub const CheckFile = struct { @@ -1243,6 +1247,13 @@ pub const ImportTable = struct { pub const Index = enum(u32) { invalid = maxInt(u32), _, + + pub fn get(this: @This(), c: *const Configuration) ImportTable { + return switch (this) { + .invalid => unreachable, + _ => extraData(c, ImportTable, @intFromEnum(this)), + }; + } }; }; @@ -1313,6 +1324,8 @@ pub const InstallDestDir = enum(u32) { /// Points into `string_bytes`, null-terminated. pub const OptionalString = enum(u32) { empty = 0, + /// The string "root". + root = 1, none = maxInt(u32), _, @@ -1326,6 +1339,8 @@ pub const OptionalString = enum(u32) { /// Points into `string_bytes`, null-terminated. pub const String = enum(u32) { empty = 0, + /// The string "root". + root = 1, _, pub fn slice(index: String, c: *const Configuration) [:0]const u8 { @@ -1954,15 +1969,23 @@ pub const Storage = enum { }; /// Valid to call only when serializing. - pub fn init(slice: []const Union) @This() { - return .{ .data = slice.ptr, .len = slice.len }; + pub fn init(s: []const Union) @This() { + return .{ .data = s.ptr, .len = s.len }; } /// Valid to call only when deserializing. - pub fn get(this: *const @This(), extra: []const u32) []const u32 { + pub fn slice(this: *const @This(), extra: []const u32) []const u32 { return extra[@intFromPtr(this.data)..][0..this.len]; } + /// Valid to call only when deserializing. + pub fn get(this: *const @This(), extra: []const u32, i: usize) Union { + const elem = slice(this, extra)[i]; + return switch (this.tag(extra, i)) { + inline else => |comptime_tag| @unionInit(Union, @tagName(comptime_tag), @enumFromInt(elem)), + }; + } + /// Valid to call only when deserializing. pub fn tag(this: *const @This(), extra: []const u32, i: usize) Tag { _ = this; @@ -2093,7 +2116,7 @@ pub const Storage = enum { const len = buffer[data_start - 1]; defer i.* = data_start + len * @typeInfo(Field.Elem).@"struct".fields.len; return .{ .mal = .{ - .bytes = @ptrCast(buffer[data_start..][0..len]), + .bytes = @ptrCast(@constCast(buffer[data_start..][0..len])), .len = len, .capacity = len, } }; -- 2.54.0 From 71ac3f15b3740974e1bac091f32fc56933134ca2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Mar 2026 22:09:12 -0700 Subject: [PATCH 039/179] build system: implement LazyPath Number of generated files is recorded in serialized Configuration. Maker preallocates array of generated files so that loads and stores can be synchronization-free (protected by the dependency tree ordering). More progress on Compile Step Zig CLI lowering. --- BRANCH_TODO | 12 + lib/compiler/Maker.zig | 104 ++++++++ lib/compiler/Maker/Graph.zig | 3 + lib/compiler/Maker/Step/Compile.zig | 356 ++++++++++++++++++++-------- lib/compiler/configurer.zig | 32 ++- lib/std/Build.zig | 32 ++- lib/std/Build/Module.zig | 141 +---------- lib/std/Build/Step/Compile.zig | 103 ++------ lib/std/Build/Step/ConfigHeader.zig | 17 +- lib/std/Build/Step/ObjCopy.zig | 21 +- lib/std/Build/Step/Options.zig | 15 +- lib/std/Build/Step/Run.zig | 43 ++-- lib/std/Build/Step/TranslateC.zig | 19 +- lib/std/Build/Step/WriteFile.zig | 18 +- lib/std/zig/Configuration.zig | 268 +++++++++++++++------ 15 files changed, 726 insertions(+), 458 deletions(-) create mode 100644 BRANCH_TODO diff --git a/BRANCH_TODO b/BRANCH_TODO new file mode 100644 index 0000000000000000000000000000000000000000..90eec2c8c8840eae47fce9964ddedde0511ddb10 --- /dev/null +++ b/BRANCH_TODO @@ -0,0 +1,12 @@ +* rename std.zig.Configuration to std.Build.Configuration +* replace union(@This().Tag) +* replace b.dupe() with string internment +* don't forget to add -listen arg back +* get zig init template working +* finish migrating the rest of the build steps +* make zig-pkg path root configurable in maker (make sure --system still works) +* eliminate calls to getPath, getPath2, getPath3 +* solve the TODOs added in this branch +* get zig tests passing +* test a bunch of third party projects / help people migrate +* refactor with DefaultingEnum diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 0faafceccffba312100eb48490088e8801910991..24818d4815e429c9f04baccdb44e5c2bba6c8218 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -33,6 +33,7 @@ graph: *Graph, install_paths: InstallPaths, scanned_config: *const ScannedConfig, steps: []Step, +generated_files: []Path, available_rss: usize, max_rss_is_default: bool, @@ -115,7 +116,13 @@ pub fn main(init: process.Init.Minimal) !void { .zig_exe = zig_exe, .environ_map = try init.environ.createMap(arena), .global_cache_root = global_cache_directory, + .local_cache_root = local_cache_directory, .zig_lib_directory = zig_lib_directory, + .build_root_directory = build_root_directory, + .pkg_root = .{ + .root_dir = build_root_directory, + .sub_path = "zig-pkg", + }, }; graph.cache.addPrefix(.{ .path = null, .handle = cwd }); @@ -525,6 +532,7 @@ pub fn main(init: process.Init.Minimal) !void { .include = install_include_path, }, .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), + .generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len), .available_rss = max_rss, .max_rss_is_default = false, @@ -1679,3 +1687,99 @@ fn initStdoutWriter(io: Io) *Writer { stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); return &stdout_writer_allocation.interface; } + +/// `asking_step` is only used for debugging purposes; it's the step being run +/// that is asking for the path. +pub fn resolveLazyPath( + maker: *const Maker, + arena: Allocator, + lazy_path: Configuration.LazyPath, + asking_step_index: Configuration.Step.Index, +) Allocator.Error!Path { + _ = asking_step_index; // TODO use this to enhance debugability when this function fails + const c = &maker.scanned_config.configuration; + const graph = maker.graph; + return switch (lazy_path) { + .source_path => |sp| try packagePath(maker, arena, sp.owner, sp.sub_path.slice(c)), + .relative => |relative| switch (relative.flags.base) { + .cwd => .{ + .root_dir = .cwd(), + .sub_path = relative.sub_path.slice(c), + }, + .local_cache => .{ + .root_dir = graph.local_cache_root, + }, + .global_cache => .{ + .root_dir = graph.global_cache_root, + }, + .build_root => .{ + .root_dir = graph.build_root_directory, + }, + }, + .generated => |gen| { + const base = maker.generated_files[@intFromEnum(gen.index)]; + var file_path = base; + for (0..gen.flags.up) |_| { + file_path.sub_path = Io.Dir.path.dirname(file_path.sub_path) orelse + fatal("invalid LazyPath traversal: up {d} times from {f}", .{ gen.flags.up, base }); + } + return file_path.join(arena, gen.sub_path.slice(c)); + }, + }; +} + +pub fn resolveLazyPathIndex( + maker: *const Maker, + arena: Allocator, + lazy_path_index: Configuration.LazyPath.Index, + asking_step_index: Configuration.Step.Index, +) Allocator.Error!Path { + const c = &maker.scanned_config.configuration; + return resolveLazyPath(maker, arena, lazy_path_index.get(c), asking_step_index); +} + +/// `resolveLazyPath` is preferred, but this can be necessary when passing Path +/// objects to child processes. +pub fn resolveLazyPathAbs( + maker: *const Maker, + arena: Allocator, + lazy_path: Configuration.LazyPath, + asking_step_index: Configuration.Step.Index, +) Allocator.Error![]const u8 { + const p = try resolveLazyPath(maker, arena, lazy_path, asking_step_index); + const root_dir_path = p.root_dir.path orelse return p.subPathOrDot(); + if (p.sub_path.len == 0) return root_dir_path; + return Io.Dir.path.join(arena, &.{ root_dir_path, p.sub_path }); +} + +/// `resolveLazyPath` is preferred, but this can be necessary when passing Path +/// objects to child processes. +pub fn resolveLazyPathIndexAbs( + maker: *const Maker, + arena: Allocator, + lazy_path_index: Configuration.LazyPath.Index, + asking_step_index: Configuration.Step.Index, +) Allocator.Error![]const u8 { + const c = &maker.scanned_config.configuration; + return resolveLazyPathAbs(maker, arena, lazy_path_index.get(c), asking_step_index); +} + +fn packagePath( + maker: *const Maker, + arena: Allocator, + package_index: Configuration.Package.Index, + sub_path: []const u8, +) Allocator.Error!Path { + const c = &maker.scanned_config.configuration; + const graph = maker.graph; + const package = package_index.get(c) orelse return .{ + .root_dir = graph.build_root_directory, + .sub_path = sub_path, + }; + const hash = package.hash.slice(c); + const pkg_root = graph.pkg_root; + return .{ + .root_dir = pkg_root.root_dir, + .sub_path = try Io.Dir.path.join(arena, &.{ pkg_root.sub_path, hash, sub_path }), + }; +} diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index e1b9ef58757c7f31e9e94e547b877e58b22c6b41..c2ed939b54af0c89e3da1cebe955797a88be8d2d 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -13,7 +13,10 @@ cache: std.Build.Cache, zig_exe: []const u8, environ_map: std.process.Environ.Map, global_cache_root: std.Build.Cache.Directory, +local_cache_root: std.Build.Cache.Directory, zig_lib_directory: std.Build.Cache.Directory, +build_root_directory: std.Build.Cache.Directory, +pkg_root: std.Build.Cache.Path, debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, incremental: ?bool = null, diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index f55cf46d006197f29a9b3cc39d08440bb91f2534..77d1f45b3175b639721548a14a8127e94a31240f 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -129,6 +129,7 @@ fn lowerZigArgs( const conf = &maker.scanned_config.configuration; const conf_step = compile_index.ptr(conf); const conf_comp = conf_step.extended.get(conf.extra).compile; + const root_module_target = conf_comp.rootModuleTarget(conf); try zig_args.append(gpa, graph.zig_exe); @@ -232,17 +233,17 @@ fn lowerZigArgs( } } - if (true) @panic("TODO"); - // Inherit dependencies on system libraries and static libraries. for (0..mod.link_objects.len) |lo_i| switch (mod.link_objects.get(conf.extra, lo_i)) { .static_path => |static_path| { if (my_responsibility) { - try zig_args.append(gpa, static_path.getPath2(step)); + try zig_args.append(gpa, try maker.resolveLazyPathIndexAbs(arena, static_path, compile_index)); total_linker_objects += 1; } }, - .system_lib => |system_lib| { + .system_lib => |system_lib_index| { + const system_lib = system_lib_index.get(conf); + const system_lib_name = system_lib.name.slice(conf); const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); if (system_lib_gop.found_existing) { try zig_args.appendSlice(gpa, system_lib_gop.value_ptr.*); @@ -254,37 +255,39 @@ fn lowerZigArgs( if (already_linked) continue; - if ((system_lib.search_strategy != prev_search_strategy or - system_lib.preferred_link_mode != prev_preferred_link_mode) and - compile.linkage != .static) + if ((system_lib.flags.search_strategy != prev_search_strategy or + system_lib.flags.preferred_link_mode != prev_preferred_link_mode) and + conf_comp.flags2.linkage != .static) { - switch (system_lib.search_strategy) { - .no_fallback => switch (system_lib.preferred_link_mode) { + switch (system_lib.flags.search_strategy) { + .no_fallback => switch (system_lib.flags.preferred_link_mode) { .dynamic => try zig_args.append(gpa, "-search_dylibs_only"), .static => try zig_args.append(gpa, "-search_static_only"), }, - .paths_first => switch (system_lib.preferred_link_mode) { + .paths_first => switch (system_lib.flags.preferred_link_mode) { .dynamic => try zig_args.append(gpa, "-search_paths_first"), .static => try zig_args.append(gpa, "-search_paths_first_static"), }, - .mode_first => switch (system_lib.preferred_link_mode) { + .mode_first => switch (system_lib.flags.preferred_link_mode) { .dynamic => try zig_args.append(gpa, "-search_dylibs_first"), .static => try zig_args.append(gpa, "-search_static_first"), }, } - prev_search_strategy = system_lib.search_strategy; - prev_preferred_link_mode = system_lib.preferred_link_mode; + prev_search_strategy = system_lib.flags.search_strategy; + prev_preferred_link_mode = system_lib.flags.preferred_link_mode; } const prefix: []const u8 = prefix: { - if (system_lib.needed) break :prefix "-needed-l"; - if (system_lib.weak) break :prefix "-weak-l"; + if (system_lib.flags.needed) break :prefix "-needed-l"; + if (system_lib.flags.weak) break :prefix "-weak-l"; break :prefix "-l"; }; - switch (system_lib.use_pkg_config) { - .no => try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ prefix, system_lib.name })), + switch (system_lib.flags.use_pkg_config) { + .no => try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ + prefix, system_lib_name, + })), .yes, .force => { - if (compile.runPkgConfig(maker, system_lib.name)) |result| { + if (compile.runPkgConfig(maker, system_lib_name)) |result| { try zig_args.appendSlice(gpa, result.cflags); try zig_args.appendSlice(gpa, result.libs); try seen_system_libs.put(arena, system_lib.name, result.cflags); @@ -294,17 +297,18 @@ fn lowerZigArgs( error.PkgConfigFailed, error.PkgConfigNotInstalled, error.PackageNotFound, - => switch (system_lib.use_pkg_config) { + => switch (system_lib.flags.use_pkg_config) { .yes => { // pkg-config failed, so fall back to linking the library // by name directly. try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ - prefix, - system_lib.name, + prefix, system_lib_name, })); }, .force => { - return step.fail(maker, "pkg-config failed for library {s}", .{system_lib.name}); + return step.fail(maker, "pkg-config failed for library {s}", .{ + system_lib_name, + }); }, .no => unreachable, }, @@ -314,23 +318,31 @@ fn lowerZigArgs( }, } }, - .other_step => |other| { - switch (other.kind) { + .other_step => |other_step_index| { + const other = other_step_index.ptr(conf); + const other_compile = other.extended.get(conf.extra).compile; + switch (other_compile.flags3.kind) { .exe => return step.fail(maker, "cannot link with an executable build artifact", .{}), .@"test" => return step.fail(maker, "cannot link with a test", .{}), .obj, .test_obj => { - const included_in_lib_or_obj = !my_responsibility and - (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); + const included_in_lib_or_obj = switch (dep_compile.flags3.kind) { + .lib, .obj, .test_obj => !my_responsibility, + else => false, + }; if (!already_linked and !included_in_lib_or_obj) { - try zig_args.append(gpa, other.getEmittedBin().getPath2(step)); + try zig_args.append(gpa, try maker.resolveLazyPathAbs( + arena, + .{ .generated = .{ .index = other_compile.generated_bin.value.? } }, + compile_index, + )); total_linker_objects += 1; } }, .lib => l: { - const other_produces_implib = other.producesImplib(); - const other_is_static = other_produces_implib or other.isStaticLibrary(); + const other_produces_implib = other_compile.producesImplib(conf); + const other_is_static = other_produces_implib or other_compile.isStaticLibrary(); - if (compile.isStaticLibrary() and other_is_static) { + if (conf_comp.isStaticLibrary() and other_is_static) { // Avoid putting a static library inside a static library. break :l; } @@ -338,20 +350,25 @@ fn lowerZigArgs( // For DLLs, we must link against the implib. // For everything else, we directly link // against the library file. - const full_path_lib = if (other_produces_implib) - try other.getGeneratedFilePath("generated_implib", &compile.step) - else - try other.getGeneratedFilePath("generated_bin", &compile.step); + const full_path_lib = try maker.resolveLazyPathAbs( + arena, + .{ .generated = .{ + .index = if (other_produces_implib) + other_compile.generated_implib.value.? + else + other_compile.generated_bin.value.?, + } }, + compile_index, + ); try zig_args.append(gpa, full_path_lib); total_linker_objects += 1; - if (other.linkage == .dynamic and - compile.rootModuleTarget().os.tag != .windows) + if (other_compile.flags2.linkage == .dynamic and + root_module_target.flags.os_tag != .windows) { if (Dir.path.dirname(full_path_lib)) |dirname| { - try zig_args.append(gpa, "-rpath"); - try zig_args.append(gpa, dirname); + try zig_args.appendSlice(gpa, &.{ "-rpath", dirname }); } } }, @@ -361,92 +378,96 @@ fn lowerZigArgs( if (!my_responsibility) break :l; if (prev_has_cflags) { - try zig_args.append(gpa, "-cflags"); - try zig_args.append(gpa, "--"); + try zig_args.appendSlice(gpa, &.{ "-cflags", "--" }); prev_has_cflags = false; } - try zig_args.append(gpa, asm_file.getPath2(mod.owner, step)); + try zig_args.append(gpa, try maker.resolveLazyPathIndexAbs(arena, asm_file, compile_index)); total_linker_objects += 1; }, - .c_source_file => |c_source_file| l: { + .c_source_file => |c_source_file_index| l: { if (!my_responsibility) break :l; - if (prev_has_cflags or c_source_file.flags.len != 0) { - try zig_args.append(gpa, "-cflags"); - for (c_source_file.flags) |arg| { - try zig_args.append(gpa, arg); + const c_source_file = c_source_file_index.get(conf); + + if (prev_has_cflags or c_source_file.args.slice.len != 0) { + try zig_args.ensureUnusedCapacity(gpa, 2 + c_source_file.args.slice.len); + zig_args.appendAssumeCapacity("-cflags"); + for (c_source_file.args.slice) |arg| { + zig_args.appendAssumeCapacity(arg.slice(conf)); } - try zig_args.append(gpa, "--"); + zig_args.appendAssumeCapacity("--"); } - prev_has_cflags = (c_source_file.flags.len != 0); + prev_has_cflags = (c_source_file.args.slice.len != 0); - if (c_source_file.language) |lang| { - try zig_args.append(gpa, "-x"); - try zig_args.append(gpa, lang.internalIdentifier()); - } + if (c_source_file.flags.lang.get()) |lang| + (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-x", lang.clangIdentifier() }; - try zig_args.append(gpa, c_source_file.file.getPath2(mod.owner, step)); + try zig_args.append(gpa, try maker.resolveLazyPathIndexAbs(arena, c_source_file.file, compile_index)); + + if (c_source_file.flags.lang != .default) + (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-x", "none" }; - if (c_source_file.language != null) { - try zig_args.append(gpa, "-x"); - try zig_args.append(gpa, "none"); - } total_linker_objects += 1; }, - .c_source_files => |c_source_files| l: { + .c_source_files => |c_source_files_index| l: { if (!my_responsibility) break :l; - if (prev_has_cflags or c_source_files.flags.len != 0) { - try zig_args.append(gpa, "-cflags"); - for (c_source_files.flags) |arg| { - try zig_args.append(gpa, arg); + const c_source_files = c_source_files_index.get(conf); + + if (prev_has_cflags or c_source_files.args.slice.len != 0) { + try zig_args.ensureUnusedCapacity(gpa, 2 + c_source_files.args.slice.len); + zig_args.appendAssumeCapacity("-cflags"); + for (c_source_files.args.slice) |arg| { + zig_args.appendAssumeCapacity(arg.slice(conf)); } - try zig_args.append(gpa, "--"); + zig_args.appendAssumeCapacity("--"); } - prev_has_cflags = (c_source_files.flags.len != 0); + prev_has_cflags = (c_source_files.args.slice.len != 0); - if (c_source_files.language) |lang| { - try zig_args.append(gpa, "-x"); - try zig_args.append(gpa, lang.internalIdentifier()); - } + if (c_source_files.flags.lang.get()) |lang| + (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-x", lang.clangIdentifier() }; - const root_path = c_source_files.root.getPath2(mod.owner, step); - for (c_source_files.files) |file| { - try zig_args.append(gpa, try Dir.path.join(arena, &.{ root_path, file })); + const root_path = try maker.resolveLazyPathIndexAbs(arena, c_source_files.root, compile_index); + try zig_args.ensureUnusedCapacity(gpa, c_source_files.sub_paths.slice.len); + for (c_source_files.sub_paths.slice) |sub_path| { + zig_args.appendAssumeCapacity(try Dir.path.join(arena, &.{ + root_path, sub_path.slice(conf), + })); } - if (c_source_files.language != null) { - try zig_args.append(gpa, "-x"); - try zig_args.append(gpa, "none"); - } + if (c_source_files.flags.lang != .default) + (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-x", "none" }; - total_linker_objects += c_source_files.files.len; + total_linker_objects += c_source_files.sub_paths.slice.len; }, - .win32_resource_file => |rc_source_file| l: { + .win32_resource_file => |rc_source_file_index| l: { if (!my_responsibility) break :l; - if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { + const rc_source_file = rc_source_file_index.get(conf); + + if (rc_source_file.args.slice.len == 0 and rc_source_file.include_paths.slice.len == 0) { if (prev_has_rcflags) { - try zig_args.append(gpa, "-rcflags"); - try zig_args.append(gpa, "--"); + (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-rcflags", "--" }; prev_has_rcflags = false; } } else { - try zig_args.append(gpa, "-rcflags"); - for (rc_source_file.flags) |arg| { - try zig_args.append(gpa, arg); + try zig_args.ensureUnusedCapacity(gpa, 1 + rc_source_file.args.slice.len); + zig_args.appendAssumeCapacity("-rcflags"); + for (rc_source_file.args.slice) |arg| { + zig_args.appendAssumeCapacity(arg.slice(conf)); } - for (rc_source_file.include_paths) |include_path| { - try zig_args.append(gpa, "/I"); - try zig_args.append(gpa, include_path.getPath2(mod.owner, step)); + try zig_args.ensureUnusedCapacity(gpa, 1 + 2 * rc_source_file.include_paths.slice.len); + for (rc_source_file.include_paths.slice) |include_path| { + zig_args.appendAssumeCapacity("/I"); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, include_path, compile_index)); } - try zig_args.append(gpa, "--"); + zig_args.appendAssumeCapacity("--"); prev_has_rcflags = true; } - try zig_args.append(gpa, rc_source_file.file.getPath2(mod.owner, step)); + try zig_args.append(gpa, try maker.resolveLazyPathIndexAbs(arena, rc_source_file.file, compile_index)); total_linker_objects += 1; }, }; @@ -455,9 +476,10 @@ fn lowerZigArgs( // have the correct parent module, but only if the module is part of // this compilation. if (!my_responsibility) continue; - if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { + if (cli_named_modules.modules.getIndex(mod_index)) |module_cli_index| { const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; - try mod.appendZigProcessFlags(zig_args, step); + if (true) @panic("TODO"); + try appendModuleFlags(zig_args, step); // --dep arguments try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); @@ -510,12 +532,12 @@ fn lowerZigArgs( if (is_linking_libc) zig_args.appendAssumeCapacity("-lc"); } - if (true) @panic("TODO"); - - if (conf_comp.win32_manifest) |manifest_file| { - try zig_args.append(gpa, manifest_file.getPath2(step)); + if (conf_comp.win32_manifest.value) |manifest_file| { + try zig_args.append(gpa, try maker.resolveLazyPathIndexAbs(arena, manifest_file, compile_index)); } + if (true) @panic("TODO"); + if (conf_comp.win32_module_definition) |module_file| { try zig_args.append(gpa, module_file.getPath2(step)); } @@ -623,27 +645,26 @@ fn lowerZigArgs( "--version", try allocPrint(arena, "{f}", .{version}), }); - if (compile.rootModuleTarget().os.tag.isDarwin()) { + if (root_module_target.flags.os_tag.isDarwin()) { const install_name = compile.install_name orelse try allocPrint(arena, "@rpath/{s}{s}{s}", .{ - compile.rootModuleTarget().libPrefix(), + root_module_target.libPrefix(), compile.name, - compile.rootModuleTarget().dynamicLibSuffix(), + root_module_target.dynamicLibSuffix(), }); - try zig_args.append(gpa, "-install_name"); - try zig_args.append(gpa, install_name); + try zig_args.appendSlice(gpa, &.{ "-install_name", install_name }); } } if (compile.entitlements) |entitlements| { - try zig_args.appendSlice(gpa, &[_][]const u8{ "--entitlements", entitlements }); + try zig_args.appendSlice(gpa, &.{ "--entitlements", entitlements }); } if (compile.pagezero_size) |pagezero_size| { const size = try allocPrint(arena, "{x}", .{pagezero_size}); - try zig_args.appendSlice(gpa, &[_][]const u8{ "-pagezero_size", size }); + try zig_args.appendSlice(gpa, &.{ "-pagezero_size", size }); } if (compile.headerpad_size) |headerpad_size| { const size = try allocPrint(arena, "{x}", .{headerpad_size}); - try zig_args.appendSlice(gpa, &[_][]const u8{ "-headerpad", size }); + try zig_args.appendSlice(gpa, &.{ "-headerpad", size }); } if (compile.headerpad_max_install_names) { try zig_args.append(gpa, "-headerpad_max_install_names"); @@ -1019,7 +1040,8 @@ const PkgConfigResult = struct { /// Run pkg-config for the given library name and parse the output, returning the arguments /// that should be passed to zig to link the given library. -fn runPkgConfig(compile: *Compile, maker: *Maker, lib_name: []const u8) !PkgConfigResult { +fn runPkgConfig(compile: *const Compile, maker: *const Maker, lib_name: []const u8) !PkgConfigResult { + if (true) @panic("TODO"); const graph = maker.graph; const wl_rpath_prefix = "-Wl,-rpath,"; @@ -1365,3 +1387,131 @@ fn getModuleList( return modules; } + +fn appendModuleFlags( + m: *Module, + zig_args: *std.array_list.Managed([]const u8), + asking_step: ?*Step, +) !void { + const b = m.owner; + + try addFlag(zig_args, m.strip, "-fstrip", "-fno-strip"); + try addFlag(zig_args, m.single_threaded, "-fsingle-threaded", "-fno-single-threaded"); + try addFlag(zig_args, m.stack_check, "-fstack-check", "-fno-stack-check"); + try addFlag(zig_args, m.stack_protector, "-fstack-protector", "-fno-stack-protector"); + try addFlag(zig_args, m.omit_frame_pointer, "-fomit-frame-pointer", "-fno-omit-frame-pointer"); + try addFlag(zig_args, m.error_tracing, "-ferror-tracing", "-fno-error-tracing"); + try addFlag(zig_args, m.sanitize_thread, "-fsanitize-thread", "-fno-sanitize-thread"); + try addFlag(zig_args, m.fuzz, "-ffuzz", "-fno-fuzz"); + try addFlag(zig_args, m.valgrind, "-fvalgrind", "-fno-valgrind"); + try addFlag(zig_args, m.pic, "-fPIC", "-fno-PIC"); + try addFlag(zig_args, m.red_zone, "-mred-zone", "-mno-red-zone"); + try addFlag(zig_args, m.no_builtin, "-fno-builtin", "-fbuiltin"); + + if (m.sanitize_c) |sc| switch (sc) { + .off => try zig_args.append("-fno-sanitize-c"), + .trap => try zig_args.append("-fsanitize-c=trap"), + .full => try zig_args.append("-fsanitize-c=full"), + }; + + if (m.dwarf_format) |dwarf_format| { + try zig_args.append(switch (dwarf_format) { + .@"32" => "-gdwarf32", + .@"64" => "-gdwarf64", + }); + } + + if (m.unwind_tables) |unwind_tables| { + try zig_args.append(switch (unwind_tables) { + .none => "-fno-unwind-tables", + .sync => "-funwind-tables", + .async => "-fasync-unwind-tables", + }); + } + + try zig_args.ensureUnusedCapacity(1); + if (m.optimize) |optimize| switch (optimize) { + .Debug => zig_args.appendAssumeCapacity("-ODebug"), + .ReleaseSmall => zig_args.appendAssumeCapacity("-OReleaseSmall"), + .ReleaseFast => zig_args.appendAssumeCapacity("-OReleaseFast"), + .ReleaseSafe => zig_args.appendAssumeCapacity("-OReleaseSafe"), + }; + + if (m.code_model != .default) { + try zig_args.append("-mcmodel"); + try zig_args.append(@tagName(m.code_model)); + } + + if (m.resolved_target) |*target| { + // Communicate the query via CLI since it's more compact. + if (!target.query.isNative()) { + try zig_args.appendSlice(&.{ + "-target", try target.query.zigTriple(b.allocator), + "-mcpu", try target.query.serializeCpuAlloc(b.allocator), + }); + if (target.query.dynamic_linker) |*dynamic_linker| { + if (dynamic_linker.get()) |dynamic_linker_path| { + try zig_args.append("--dynamic-linker"); + try zig_args.append(dynamic_linker_path); + } else { + try zig_args.append("--no-dynamic-linker"); + } + } + } + } + + for (m.export_symbol_names) |symbol_name| { + try zig_args.append(b.fmt("--export={s}", .{symbol_name})); + } + + for (m.include_dirs.items) |include_dir| { + try appendIncludeDirFlags(include_dir, b, zig_args, asking_step); + } + + try zig_args.appendSlice(m.c_macros.items); + + try zig_args.ensureUnusedCapacity(2 * m.lib_paths.items.len); + for (m.lib_paths.items) |lib_path| { + zig_args.appendAssumeCapacity("-L"); + zig_args.appendAssumeCapacity(lib_path.getPath2(b, asking_step)); + } + + try zig_args.ensureUnusedCapacity(2 * m.rpaths.items.len); + for (m.rpaths.items) |rpath| switch (rpath) { + .lazy_path => |lp| { + zig_args.appendAssumeCapacity("-rpath"); + zig_args.appendAssumeCapacity(lp.getPath2(b, asking_step)); + }, + .special => |bytes| { + zig_args.appendAssumeCapacity("-rpath"); + zig_args.appendAssumeCapacity(bytes); + }, + }; +} + +fn appendIncludeDirFlags( + include_dir: Configuration.Module.IncludeDir, + b: *std.Build, + zig_args: *std.array_list.Managed([]const u8), + asking_step: ?*Step, +) !void { + const flag: []const u8, const lazy_path: Configuration.LazyPath = switch (include_dir) { + // zig fmt: off + .path => |lp| .{ "-I", lp }, + .path_system => |lp| .{ "-isystem", lp }, + .path_after => |lp| .{ "-idirafter", lp }, + .framework_path => |lp| .{ "-F", lp }, + .framework_path_system => |lp| .{ "-iframework", lp }, + .config_header_step => |ch| .{ "-I", ch.getOutputDir() }, + .other_step => |comp| .{ "-I", comp.installed_headers_include_tree.?.getDirectory() }, + // zig fmt: on + .embed_path => |lazy_path| { + // Special case: this is a single arg. + const resolved = lazy_path.getPath3(b, asking_step); + const arg = b.fmt("--embed-dir={f}", .{resolved}); + return zig_args.append(arg); + }, + }; + const resolved_str = try lazy_path.getPath3(b, asking_step).toString(b.graph.arena); + return zig_args.appendSlice(&.{ flag, resolved_str }); +} diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index bea788fba183691ae505857a21daf7e78e7488bc..f6b52f057ea1520dd102d9407d78ce670af7f876 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -96,6 +96,7 @@ pub fn main(init: process.Init.Minimal) !void { .query = .{}, .result = try std.zig.system.resolveTargetQuery(io, .{}), }, + .generated_files = .empty, }; graph.cache.addPrefix(.{ .path = null, .handle = cwd }); @@ -242,7 +243,7 @@ const Serialize = struct { return gop.value_ptr.*; } - fn addOptionalLazyPathEnum(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.OptionalLazyPath { + fn addOptionalLazyPathEnum(s: *Serialize, lp: ?std.Build.LazyPath) !Configuration.LazyPath.OptionalIndex { const wc = s.wc; return @enumFromInt(switch (lp orelse return .none) { .src_path => |src_path| i: { @@ -257,6 +258,7 @@ const Serialize = struct { const sub_path = try wc.addString(generated.sub_path); break :i try wc.addExtra(@as(Configuration.LazyPath.Generated, .{ .flags = .{ .up = @intCast(generated.up) }, + .index = generated.index, .sub_path = sub_path, })); }, @@ -278,11 +280,11 @@ const Serialize = struct { }); } - fn addOptionalLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !?Configuration.LazyPath { + fn addOptionalLazyPath(s: *Serialize, lp: ?std.Build.LazyPath) !?Configuration.LazyPath.Index { return (try addOptionalLazyPathEnum(s, lp)).unwrap(); } - fn addLazyPath(s: *Serialize, lp: std.Build.LazyPath) !Configuration.LazyPath { + fn addLazyPath(s: *Serialize, lp: std.Build.LazyPath) !Configuration.LazyPath.Index { return @enumFromInt(@intFromEnum(try addOptionalLazyPathEnum(s, lp))); } @@ -351,8 +353,8 @@ const Serialize = struct { }))); } - fn initLazyPathList(s: *Serialize, list: []const std.Build.LazyPath) ![]const Configuration.LazyPath { - const result = try s.arena.alloc(Configuration.LazyPath, list.len); + fn initLazyPathList(s: *Serialize, list: []const std.Build.LazyPath) ![]const Configuration.LazyPath.Index { + const result = try s.arena.alloc(Configuration.LazyPath.Index, list.len); for (result, list) |*dest, src| dest.* = try addLazyPath(s, src); return result; } @@ -665,6 +667,15 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { } else .none, .linker_script = c.linker_script != null, .version_script = c.version_script != null, + .emit_directory = c.emit_directory != .none, + .generated_docs = c.generated_docs != .none, + .generated_asm = c.generated_asm != .none, + .generated_bin = c.generated_bin != .none, + .generated_pdb = c.generated_pdb != .none, + .generated_implib = c.generated_implib != .none, + .generated_llvm_bc = c.generated_llvm_bc != .none, + .generated_llvm_ir = c.generated_llvm_ir != .none, + .generated_h = c.generated_h != .none, }, .root_module = try s.addModule(c.root_module), .root_name = try wc.addString(c.name), @@ -709,6 +720,16 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .simple => .{ .simple = try s.addLazyPath(tr.path) }, .server => .{ .server = try s.addLazyPath(tr.path) }, } else .default }, + + .emit_directory = .{ .value = c.emit_directory.unwrap() }, + .generated_docs = .{ .value = c.generated_docs.unwrap() }, + .generated_asm = .{ .value = c.generated_asm.unwrap() }, + .generated_bin = .{ .value = c.generated_bin.unwrap() }, + .generated_pdb = .{ .value = c.generated_pdb.unwrap() }, + .generated_implib = .{ .value = c.generated_implib.unwrap() }, + .generated_llvm_bc = .{ .value = c.generated_llvm_bc.unwrap() }, + .generated_llvm_ir = .{ .value = c.generated_llvm_ir.unwrap() }, + .generated_h = .{ .value = c.generated_h.unwrap() }, })); break :e @enumFromInt(extra_index); @@ -804,6 +825,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { try wc.write(writer, .{ .default_step = s.stepIndex(b.default_step), + .generated_files_len = @intCast(graph.generated_files.items.len), }); } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 284a2da346f17edba2100a5a5b93f40986b3c9a9..8b3b8ec5fd5a3d561a58c7c3c949704f7b04a1d4 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1,4 +1,5 @@ const Build = @This(); + const builtin = @import("builtin"); const std = @import("std.zig"); @@ -111,6 +112,14 @@ pub const Graph = struct { /// process via `Step.Run` API but cannot be observed in the configure /// phase. have_run_args: bool = false, + + /// Indexes correspond to `Configuration.GeneratedFileIndex`. + generated_files: std.ArrayList(*Step), + + pub fn addGeneratedFile(graph: *Graph, owner: *Step) Configuration.GeneratedFileIndex { + graph.generated_files.append(graph.arena, owner) catch @panic("OOM"); + return @enumFromInt(graph.generated_files.items.len - 1); + } }; const AvailableDeps = []const struct { []const u8, []const u8 }; @@ -865,7 +874,7 @@ pub fn dupe(b: *Build, bytes: []const u8) []u8 { return dupeInner(b.allocator, bytes); } -pub fn dupeInner(allocator: std.mem.Allocator, bytes: []const u8) []u8 { +pub fn dupeInner(allocator: Allocator, bytes: []const u8) []u8 { return allocator.dupe(u8, bytes) catch @panic("OOM"); } @@ -881,7 +890,7 @@ pub fn dupePath(b: *Build, bytes: []const u8) []u8 { return dupePathInner(b.allocator, bytes); } -fn dupePathInner(allocator: std.mem.Allocator, bytes: []const u8) []u8 { +fn dupePathInner(allocator: Allocator, bytes: []const u8) []u8 { const the_copy = dupeInner(allocator, bytes); for (the_copy) |*byte| { switch (byte.*) { @@ -2068,13 +2077,6 @@ pub fn runBuild(b: *Build, build_zig: anytype) anyerror!void { } } -/// A file that is generated by a build step. -/// This struct is an interface that is meant to be used with `@fieldParentPtr` to implement the actual path logic. -pub const GeneratedFile = struct { - /// The step that generates the file. - step: *Step, -}; - // dirnameAllowEmpty is a variant of fs.path.dirname // that allows "" to refer to the root for relative paths. // @@ -2114,7 +2116,7 @@ pub const LazyPath = union(enum) { }, generated: struct { - file: *const GeneratedFile, + index: Configuration.GeneratedFileIndex, /// The number of parent directories to go up. /// 0 means the generated file itself. @@ -2242,7 +2244,11 @@ pub const LazyPath = union(enum) { pub fn addStepDependencies(lazy_path: LazyPath, other_step: *Step) void { switch (lazy_path) { .src_path, .cwd_relative, .dependency => {}, - .generated => |gen| other_step.dependOn(gen.file.step), + .generated => |gen| { + const graph = other_step.owner.graph; + const generated_owner_step = graph.generated_files.items[@intFromEnum(gen.index)]; + other_step.dependOn(generated_owner_step); + }, } } @@ -2266,7 +2272,7 @@ pub const LazyPath = union(enum) { return lazy_path.dupeInner(b.allocator); } - fn dupeInner(lazy_path: LazyPath, allocator: std.mem.Allocator) LazyPath { + fn dupeInner(lazy_path: LazyPath, allocator: Allocator) LazyPath { return switch (lazy_path) { .src_path => |sp| .{ .src_path = .{ .owner = sp.owner, @@ -2274,7 +2280,7 @@ pub const LazyPath = union(enum) { } }, .cwd_relative => |p| .{ .cwd_relative = dupePathInner(allocator, p) }, .generated => |gen| .{ .generated = .{ - .file = gen.file, + .index = gen.index, .up = gen.up, .sub_path = dupePathInner(allocator, gen.sub_path), } }, diff --git a/lib/std/Build/Module.zig b/lib/std/Build/Module.zig index 2074eb85db2786cb9271b3c8638521676a548bde..22100e1b2e4ea08c9972eeaee78c555e24b0f43f 100644 --- a/lib/std/Build/Module.zig +++ b/lib/std/Build/Module.zig @@ -89,7 +89,8 @@ pub const CSourceLanguage = enum { /// Assembly with the C preprocessor assembly_with_preprocessor, - pub fn internalIdentifier(self: CSourceLanguage) []const u8 { + /// The value passed to "-x" CLI flag of Clang. + pub fn clangIdentifier(self: CSourceLanguage) [:0]const u8 { return switch (self) { .c => "c", .cpp => "c++", @@ -164,33 +165,6 @@ pub const IncludeDir = union(enum) { other_step: *Step.Compile, config_header_step: *Step.ConfigHeader, embed_path: LazyPath, - - pub fn appendZigProcessFlags( - include_dir: IncludeDir, - b: *std.Build, - zig_args: *std.array_list.Managed([]const u8), - asking_step: ?*Step, - ) !void { - const flag: []const u8, const lazy_path: LazyPath = switch (include_dir) { - // zig fmt: off - .path => |lp| .{ "-I", lp }, - .path_system => |lp| .{ "-isystem", lp }, - .path_after => |lp| .{ "-idirafter", lp }, - .framework_path => |lp| .{ "-F", lp }, - .framework_path_system => |lp| .{ "-iframework", lp }, - .config_header_step => |ch| .{ "-I", ch.getOutputDir() }, - .other_step => |comp| .{ "-I", comp.installed_headers_include_tree.?.getDirectory() }, - // zig fmt: on - .embed_path => |lazy_path| { - // Special case: this is a single arg. - const resolved = lazy_path.getPath3(b, asking_step); - const arg = b.fmt("--embed-dir={f}", .{resolved}); - return zig_args.append(arg); - }, - }; - const resolved_str = try lazy_path.getPath3(b, asking_step).toString(b.graph.arena); - return zig_args.appendSlice(&.{ flag, resolved_str }); - } }; pub const LinkFrameworkOptions = struct { @@ -533,117 +507,6 @@ pub fn addCMacro(m: *Module, name: []const u8, value: []const u8) void { m.c_macros.append(b.allocator, b.fmt("-D{s}={s}", .{ name, value })) catch @panic("OOM"); } -pub fn appendZigProcessFlags( - m: *Module, - zig_args: *std.array_list.Managed([]const u8), - asking_step: ?*Step, -) !void { - const b = m.owner; - - try addFlag(zig_args, m.strip, "-fstrip", "-fno-strip"); - try addFlag(zig_args, m.single_threaded, "-fsingle-threaded", "-fno-single-threaded"); - try addFlag(zig_args, m.stack_check, "-fstack-check", "-fno-stack-check"); - try addFlag(zig_args, m.stack_protector, "-fstack-protector", "-fno-stack-protector"); - try addFlag(zig_args, m.omit_frame_pointer, "-fomit-frame-pointer", "-fno-omit-frame-pointer"); - try addFlag(zig_args, m.error_tracing, "-ferror-tracing", "-fno-error-tracing"); - try addFlag(zig_args, m.sanitize_thread, "-fsanitize-thread", "-fno-sanitize-thread"); - try addFlag(zig_args, m.fuzz, "-ffuzz", "-fno-fuzz"); - try addFlag(zig_args, m.valgrind, "-fvalgrind", "-fno-valgrind"); - try addFlag(zig_args, m.pic, "-fPIC", "-fno-PIC"); - try addFlag(zig_args, m.red_zone, "-mred-zone", "-mno-red-zone"); - try addFlag(zig_args, m.no_builtin, "-fno-builtin", "-fbuiltin"); - - if (m.sanitize_c) |sc| switch (sc) { - .off => try zig_args.append("-fno-sanitize-c"), - .trap => try zig_args.append("-fsanitize-c=trap"), - .full => try zig_args.append("-fsanitize-c=full"), - }; - - if (m.dwarf_format) |dwarf_format| { - try zig_args.append(switch (dwarf_format) { - .@"32" => "-gdwarf32", - .@"64" => "-gdwarf64", - }); - } - - if (m.unwind_tables) |unwind_tables| { - try zig_args.append(switch (unwind_tables) { - .none => "-fno-unwind-tables", - .sync => "-funwind-tables", - .async => "-fasync-unwind-tables", - }); - } - - try zig_args.ensureUnusedCapacity(1); - if (m.optimize) |optimize| switch (optimize) { - .Debug => zig_args.appendAssumeCapacity("-ODebug"), - .ReleaseSmall => zig_args.appendAssumeCapacity("-OReleaseSmall"), - .ReleaseFast => zig_args.appendAssumeCapacity("-OReleaseFast"), - .ReleaseSafe => zig_args.appendAssumeCapacity("-OReleaseSafe"), - }; - - if (m.code_model != .default) { - try zig_args.append("-mcmodel"); - try zig_args.append(@tagName(m.code_model)); - } - - if (m.resolved_target) |*target| { - // Communicate the query via CLI since it's more compact. - if (!target.query.isNative()) { - try zig_args.appendSlice(&.{ - "-target", try target.query.zigTriple(b.allocator), - "-mcpu", try target.query.serializeCpuAlloc(b.allocator), - }); - if (target.query.dynamic_linker) |*dynamic_linker| { - if (dynamic_linker.get()) |dynamic_linker_path| { - try zig_args.append("--dynamic-linker"); - try zig_args.append(dynamic_linker_path); - } else { - try zig_args.append("--no-dynamic-linker"); - } - } - } - } - - for (m.export_symbol_names) |symbol_name| { - try zig_args.append(b.fmt("--export={s}", .{symbol_name})); - } - - for (m.include_dirs.items) |include_dir| { - try include_dir.appendZigProcessFlags(b, zig_args, asking_step); - } - - try zig_args.appendSlice(m.c_macros.items); - - try zig_args.ensureUnusedCapacity(2 * m.lib_paths.items.len); - for (m.lib_paths.items) |lib_path| { - zig_args.appendAssumeCapacity("-L"); - zig_args.appendAssumeCapacity(lib_path.getPath2(b, asking_step)); - } - - try zig_args.ensureUnusedCapacity(2 * m.rpaths.items.len); - for (m.rpaths.items) |rpath| switch (rpath) { - .lazy_path => |lp| { - zig_args.appendAssumeCapacity("-rpath"); - zig_args.appendAssumeCapacity(lp.getPath2(b, asking_step)); - }, - .special => |bytes| { - zig_args.appendAssumeCapacity("-rpath"); - zig_args.appendAssumeCapacity(bytes); - }, - }; -} - -fn addFlag( - args: *std.array_list.Managed([]const u8), - opt: ?bool, - then_name: []const u8, - else_name: []const u8, -) !void { - const cond = opt orelse return; - return args.append(if (cond) then_name else else_name); -} - fn linkLibraryOrObject(m: *Module, other: *Step.Compile) void { const allocator = m.owner.allocator; _ = other.getEmittedBin(); // Indicate there is a dependency on the outputted binary. diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 0654894c4b5c209958d34105a8a6e8e81a066f83..81e07f317d8a0f86c974c2023827cbeffa1a7e45 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1,4 +1,5 @@ const Compile = @This(); + const builtin = @import("builtin"); const std = @import("std"); @@ -13,8 +14,8 @@ const Step = std.Build.Step; const LazyPath = std.Build.LazyPath; const Module = std.Build.Module; const InstallDir = std.Build.InstallDir; -const GeneratedFile = std.Build.GeneratedFile; const Path = std.Build.Cache.Path; +const Configuration = std.Build.Configuration; pub const base_tag: Step.Tag = .compile; @@ -212,19 +213,6 @@ allow_so_scripts: ?bool = null, /// otherwise. expect_errors: ?ExpectedCompileErrors = null, -emit_directory: ?*GeneratedFile, - -generated_docs: ?*GeneratedFile, -generated_asm: ?*GeneratedFile, -generated_bin: ?*GeneratedFile, -generated_pdb: ?*GeneratedFile, -// hack for stage2_x86_64 + coff -generated_compiler_rt_dyn_lib: ?*GeneratedFile, -generated_implib: ?*GeneratedFile, -generated_llvm_bc: ?*GeneratedFile, -generated_llvm_ir: ?*GeneratedFile, -generated_h: ?*GeneratedFile, - /// The maximum number of distinct errors within a compilation step Defaults to /// `std.math.maxInt(u16)`. Overrides the argument passed to `zig build`. error_limit: ?u32 = null, @@ -248,6 +236,16 @@ is_linking_libcpp: bool = false, /// builtin fuzzer, see the `fuzz` flag in `Module`. sanitize_coverage_trace_pc_guard: ?bool = null, +emit_directory: Configuration.OptionalGeneratedFileIndex = .none, +generated_docs: Configuration.OptionalGeneratedFileIndex = .none, +generated_asm: Configuration.OptionalGeneratedFileIndex = .none, +generated_bin: Configuration.OptionalGeneratedFileIndex = .none, +generated_pdb: Configuration.OptionalGeneratedFileIndex = .none, +generated_implib: Configuration.OptionalGeneratedFileIndex = .none, +generated_llvm_bc: Configuration.OptionalGeneratedFileIndex = .none, +generated_llvm_ir: Configuration.OptionalGeneratedFileIndex = .none, +generated_h: Configuration.OptionalGeneratedFileIndex = .none, + pub const ExpectedCompileErrors = union(enum) { contains: []const u8, exact: []const []const u8, @@ -291,7 +289,7 @@ pub const Options = struct { entitlements: ?LazyPath = null, }; -pub const Kind = std.Build.Configuration.Step.Compile.Kind; +pub const Kind = Configuration.Step.Compile.Kind; pub const HeaderInstallation = union(enum) { file: File, @@ -362,6 +360,9 @@ pub const TestRunner = struct { }; pub fn create(owner: *std.Build, options: Options) *Compile { + const graph = owner.graph; + const arena = graph.arena; + const name = owner.dupe(options.name); if (mem.find(u8, name, "/") != null or mem.find(u8, name, "\\") != null) { panic("invalid name: '{s}'. It looks like a file path, but it is supposed to be the library or application name.", .{name}); @@ -376,12 +377,12 @@ pub fn create(owner: *std.Build, options: Options) *Compile { if (options.kind.isTest() and mem.eql(u8, name, "test")) @tagName(options.kind) else - owner.fmt("{s} {s}", .{ @tagName(options.kind), name }), + owner.fmt("{t} {s}", .{ options.kind, name }), @tagName(options.root_module.optimize orelse .Debug), - resolved_target.query.zigTriple(owner.allocator) catch @panic("OOM"), + resolved_target.query.zigTriple(arena) catch @panic("OOM"), }); - const out_filename = std.zig.binNameAlloc(owner.allocator, .{ + const out_filename = std.zig.binNameAlloc(arena, .{ .root_name = name, .target = target, .output_mode = switch (options.kind) { @@ -393,7 +394,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .version = options.version, }) catch @panic("OOM"); - const compile = owner.allocator.create(Compile) catch @panic("OOM"); + const compile = arena.create(Compile) catch @panic("OOM"); compile.* = .{ .root_module = options.root_module, .verbose_link = false, @@ -420,17 +421,6 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .rdynamic = false, .force_undefined_symbols = .empty, - .emit_directory = null, - .generated_docs = null, - .generated_asm = null, - .generated_bin = null, - .generated_pdb = null, - .generated_compiler_rt_dyn_lib = null, - .generated_implib = null, - .generated_llvm_bc = null, - .generated_llvm_ir = null, - .generated_h = null, - .use_llvm = options.use_llvm, .use_lld = options.use_lld, .use_new_linker = null, @@ -706,13 +696,12 @@ pub fn setLibCFile(compile: *Compile, libc_file: ?LazyPath) void { } } -fn getEmittedFileGeneric(compile: *Compile, output_file: *?*GeneratedFile) LazyPath { - if (output_file.*) |file| return .{ .generated = .{ .file = file } }; - const arena = compile.step.owner.allocator; - const generated_file = arena.create(GeneratedFile) catch @panic("OOM"); - generated_file.* = .{ .step = &compile.step }; - output_file.* = generated_file; - return .{ .generated = .{ .file = generated_file } }; +fn getEmittedFileGeneric(compile: *Compile, output_file: *Configuration.OptionalGeneratedFileIndex) LazyPath { + if (output_file.unwrap()) |index| return .{ .generated = .{ .index = index } }; + const graph = compile.step.owner.graph; + const index = graph.addGeneratedFile(&compile.step); + output_file.* = .init(index); + return .{ .generated = .{ .index = index } }; } /// Returns the path to the directory that contains the emitted binary file. @@ -785,46 +774,6 @@ pub fn setExecCmd(compile: *Compile, args: []const ?[]const u8) void { compile.exec_cmd_args = duped_args; } -fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking_step: ?*Step) ![]const u8 { - const step = &compile.step; - const b = step.owner; - const graph = b.graph; - const io = graph.io; - const maybe_path: ?*GeneratedFile = @field(compile, tag_name); - - const generated_file = maybe_path orelse { - const stderr = try io.lockStderr(&.{}, graph.stderr_mode); - std.Build.dumpBadGetPathHelp(&compile.step, stderr.terminal(), compile.step.owner, asking_step) catch {}; - io.unlockStderr(); - @panic("missing emit option for " ++ tag_name); - }; - - const path = generated_file.path orelse { - const stderr = try io.lockStderr(&.{}, graph.stderr_mode); - std.Build.dumpBadGetPathHelp(&compile.step, stderr.terminal(), compile.step.owner, asking_step) catch {}; - io.unlockStderr(); - @panic(tag_name ++ " is null. Is there a missing step dependency?"); - }; - - return path; -} - -fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifact) []const u8 { - const arena = c.step.owner.graph.arena; - const name = ea.cacheName(arena, .{ - .root_name = c.name, - .target = &c.root_module.resolved_target.?.result, - .output_mode = switch (c.kind) { - .lib => .Lib, - .obj, .test_obj => .Obj, - .exe, .@"test" => .Exe, - }, - .link_mode = c.linkage, - .version = c.version, - }) catch @panic("OOM"); - return out_dir.joinString(arena, name) catch @panic("OOM"); -} - pub fn rootModuleTarget(c: *Compile) std.Target { // The root module is always given a target, so we know this to be non-null. return c.root_module.resolved_target.?.result; diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index 2406250b4764d59fed65b8186e820f32b7078ecb..a1f613e1cb4471f39dfd7311db6d8b14cf52d433 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -5,6 +5,7 @@ const Io = std.Io; const Step = std.Build.Step; const Allocator = std.mem.Allocator; const Writer = std.Io.Writer; +const Configuration = std.Build.Configuration; pub const Style = union(enum) { /// A configure format supported by autotools that uses `#undef foo` to @@ -40,7 +41,7 @@ pub const Value = union(enum) { step: Step, values: std.array_hash_map.String(Value), /// This directory contains the generated file under the name `include_path`. -generated_dir: std.Build.GeneratedFile, +generated_dir: Configuration.GeneratedFileIndex, style: Style, max_bytes: usize, @@ -58,7 +59,9 @@ pub const Options = struct { }; pub fn create(owner: *std.Build, options: Options) *ConfigHeader { - const config_header = owner.allocator.create(ConfigHeader) catch @panic("OOM"); + const graph = owner.graph; + const arena = graph.arena; + const config_header = arena.create(ConfigHeader) catch @panic("OOM"); var include_path: []const u8 = "config.h"; @@ -80,11 +83,9 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { } const name = if (options.style.getPath()) |s| - owner.fmt("configure {s} header {s} to {s}", .{ - @tagName(options.style), s.getDisplayName(), include_path, - }) + owner.fmt("configure {t} header {s} to {s}", .{ options.style, s.getDisplayName(), include_path }) else - owner.fmt("configure {s} header to {s}", .{ @tagName(options.style), include_path }); + owner.fmt("configure {t} header to {s}", .{ options.style, include_path }); config_header.* = .{ .step = .init(.{ @@ -100,7 +101,7 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { .max_bytes = options.max_bytes, .include_path = include_path, .include_guard_override = options.include_guard_override, - .generated_dir = .{ .step = &config_header.step }, + .generated_dir = graph.addGeneratedFile(&config_header.step), }; if (options.style.getPath()) |s| { @@ -125,7 +126,7 @@ pub fn addValues(config_header: *ConfigHeader, values: anytype) void { } pub fn getOutputDir(ch: *ConfigHeader) std.Build.LazyPath { - return .{ .generated = .{ .file = &ch.generated_dir } }; + return .{ .generated = .{ .index = &ch.generated_dir } }; } pub fn getOutputFile(ch: *ConfigHeader) std.Build.LazyPath { return ch.getOutputDir().path(ch.step.owner, ch.include_path); diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index 5ab21c3bcc835254226d27e66c5049be1b972ad3..d7de00c0ef28400009daf1f521fec43ba90403dc 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -9,6 +9,7 @@ const Step = std.Build.Step; const elf = std.elf; const fs = std.fs; const sort = std.sort; +const Configuration = std.Build.Configuration; pub const base_tag: Step.Tag = .objcopy; @@ -71,8 +72,8 @@ pub const SetSectionFlags = struct { step: Step, input_file: std.Build.LazyPath, basename: []const u8, -output_file: std.Build.GeneratedFile, -output_file_debug: ?std.Build.GeneratedFile, +output_file: Configuration.GeneratedFileIndex, +output_file_debug: Configuration.OptionalGeneratedFileIndex, format: ?RawFormat, only_section: ?[]const u8, @@ -108,7 +109,10 @@ pub fn create( input_file: std.Build.LazyPath, options: Options, ) *ObjCopy { - const objcopy = owner.allocator.create(ObjCopy) catch @panic("OOM"); + const graph = owner.graph; + const arena = graph.arena; + + const objcopy = arena.create(ObjCopy) catch @panic("OOM"); objcopy.* = ObjCopy{ .step = Step.init(.{ .tag = base_tag, @@ -118,8 +122,11 @@ pub fn create( }), .input_file = input_file, .basename = options.basename orelse input_file.getDisplayName(), - .output_file = std.Build.GeneratedFile{ .step = &objcopy.step }, - .output_file_debug = if (options.strip != .none and options.extract_to_separate_file) std.Build.GeneratedFile{ .step = &objcopy.step } else null, + .output_file = graph.addGeneratedFile(&objcopy.step), + .output_file_debug = if (options.strip != .none and options.extract_to_separate_file) + .init(graph.addGeneratedFile(&objcopy.step)) + else + .none, .format = options.format, .only_section = options.only_section, .pad_to = options.pad_to, @@ -134,10 +141,10 @@ pub fn create( } pub fn getOutput(objcopy: *const ObjCopy) std.Build.LazyPath { - return .{ .generated = .{ .file = &objcopy.output_file } }; + return .{ .generated = .{ .index = objcopy.output_file } }; } pub fn getOutputSeparatedDebug(objcopy: *const ObjCopy) ?std.Build.LazyPath { - return if (objcopy.output_file_debug) |*file| .{ .generated = .{ .file = file } } else null; + return if (objcopy.output_file_debug.unwrap()) |index| .{ .generated = .{ .index = index } } else null; } fn make(step: *Step, options: Step.MakeOptions) !void { diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 21df380b18ed05dd0ad181116370b4c38b533c84..1bf8266ec0841c97c282fc310356fc4aaf7b6608 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -1,24 +1,28 @@ const Options = @This(); + const builtin = @import("builtin"); const std = @import("std"); const Io = std.Io; const fs = std.fs; const Step = std.Build.Step; -const GeneratedFile = std.Build.GeneratedFile; const LazyPath = std.Build.LazyPath; +const Configuration = std.Build.Configuration; pub const base_tag: Step.Tag = .options; step: Step, -generated_file: GeneratedFile, +generated_file: Configuration.GeneratedFileIndex, contents: std.ArrayList(u8), args: std.ArrayList(Arg), encountered_types: std.StringHashMapUnmanaged(void), pub fn create(owner: *std.Build) *Options { - const options = owner.allocator.create(Options) catch @panic("OOM"); + const graph = owner.graph; + const arena = graph.arena; + + const options = arena.create(Options) catch @panic("OOM"); options.* = .{ .step = .init(.{ .tag = base_tag, @@ -26,12 +30,11 @@ pub fn create(owner: *std.Build) *Options { .owner = owner, .makeFn = make, }), - .generated_file = undefined, + .generated_file = graph.addGeneratedFile(&options.step), .contents = .empty, .args = .empty, .encountered_types = .empty, }; - options.generated_file = .{ .step = &options.step }; return options; } @@ -434,7 +437,7 @@ pub fn createModule(options: *Options) *std.Build.Module { /// Returns the main artifact of this Build Step which is a Zig source file /// generated from the key-value pairs of the Options. pub fn getOutput(options: *Options) LazyPath { - return .{ .generated = .{ .file = &options.generated_file } }; + return .{ .generated = .{ .index = options.generated_file } }; } fn make(step: *Step, make_options: Step.MakeOptions) !void { diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 73aeba93218ee65d9cc5d69cc5d4035907054d30..e51be09242ca074ecfaa5bec5369620c32a03ae2 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -11,6 +11,7 @@ const process = std.process; const EnvMap = std.process.Environ.Map; const assert = std.debug.assert; const Path = std.Build.Cache.Path; +const Configuration = std.Build.Configuration; pub const base_tag: Step.Tag = .run; @@ -162,7 +163,7 @@ pub const DecoratedLazyPath = struct { }; pub const Output = struct { - generated_file: std.Build.GeneratedFile, + generated_file: Configuration.GeneratedFileIndex, prefix: []const u8, basename: []const u8, }; @@ -272,21 +273,23 @@ pub fn addPrefixedOutputFileArg( basename: []const u8, ) std.Build.LazyPath { const b = run.step.owner; + const graph = b.graph; + const arena = graph.arena; if (basename.len == 0) @panic("basename must not be empty"); - const output = b.allocator.create(Output) catch @panic("OOM"); + const output = arena.create(Output) catch @panic("OOM"); output.* = .{ .prefix = b.dupe(prefix), .basename = b.dupe(basename), - .generated_file = .{ .step = &run.step }, + .generated_file = graph.addGeneratedFile(&run.step), }; - run.argv.append(b.allocator, .{ .output_file = output }) catch @panic("OOM"); + run.argv.append(arena, .{ .output_file = output }) catch @panic("OOM"); if (run.rename_step_with_output_arg) { run.setName(b.fmt("{s} ({s})", .{ run.step.name, basename })); } - return .{ .generated = .{ .file = &output.generated_file } }; + return .{ .generated = .{ .index = output.generated_file } }; } /// Appends an input file to the command line arguments. @@ -470,20 +473,22 @@ pub fn addDepFileOutputArg(run: *Run, basename: []const u8) std.Build.LazyPath { /// Only one dep file argument is allowed by instance. pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []const u8) std.Build.LazyPath { const b = run.step.owner; + const graph = b.graph; + const arena = graph.arena; assert(run.dep_output_file == null); - const dep_file = b.allocator.create(Output) catch @panic("OOM"); + const dep_file = arena.create(Output) catch @panic("OOM"); dep_file.* = .{ .prefix = b.dupe(prefix), .basename = b.dupe(basename), - .generated_file = .{ .step = &run.step }, + .generated_file = graph.addGeneratedFile(&run.step), }; run.dep_output_file = dep_file; - run.argv.append(b.allocator, .{ .output_file = dep_file }) catch @panic("OOM"); + run.argv.append(arena, .{ .output_file = dep_file }) catch @panic("OOM"); - return .{ .generated = .{ .file = &dep_file.generated_file } }; + return .{ .generated = .{ .index = dep_file.generated_file } }; } pub fn addArg(run: *Run, arg: []const u8) void { @@ -627,20 +632,22 @@ pub fn captureStdErr(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPa assert(run.stdio != .zig_test); const b = run.step.owner; + const graph = b.graph; + const arena = graph.arena; - if (run.captured_stderr) |captured| return .{ .generated = .{ .file = &captured.output.generated_file } }; + if (run.captured_stderr) |captured| return .{ .generated = .{ .index = captured.output.generated_file } }; - const captured = b.allocator.create(CapturedStdIo) catch @panic("OOM"); + const captured = arena.create(CapturedStdIo) catch @panic("OOM"); captured.* = .{ .output = .{ .prefix = "", .basename = if (options.basename) |basename| b.dupe(basename) else "stderr", - .generated_file = .{ .step = &run.step }, + .generated_file = graph.addGeneratedFile(&run.step), }, .trim_whitespace = options.trim_whitespace, }; run.captured_stderr = captured; - return .{ .generated = .{ .file = &captured.output.generated_file } }; + return .{ .generated = .{ .index = captured.output.generated_file } }; } pub fn captureStdOut(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPath { @@ -648,20 +655,22 @@ pub fn captureStdOut(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPa assert(run.stdio != .zig_test); const b = run.step.owner; + const graph = b.graph; + const arena = graph.arena; - if (run.captured_stdout) |captured| return .{ .generated = .{ .file = &captured.output.generated_file } }; + if (run.captured_stdout) |captured| return .{ .generated = .{ .index = captured.output.generated_file } }; - const captured = b.allocator.create(CapturedStdIo) catch @panic("OOM"); + const captured = arena.create(CapturedStdIo) catch @panic("OOM"); captured.* = .{ .output = .{ .prefix = "", .basename = if (options.basename) |basename| b.dupe(basename) else "stdout", - .generated_file = .{ .step = &run.step }, + .generated_file = graph.addGeneratedFile(&run.step), }, .trim_whitespace = options.trim_whitespace, }; run.captured_stdout = captured; - return .{ .generated = .{ .file = &captured.output.generated_file } }; + return .{ .generated = .{ .index = captured.output.generated_file } }; } /// Adds an additional input files that, when modified, indicates that this Run diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index 90d9e28155fabf1a853808d19bd3b81097fa745e..0c0844d070f09ba8907f3a4af8f0e5e23a676818 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -1,10 +1,11 @@ +const TranslateC = @This(); + const std = @import("std"); const Step = std.Build.Step; const LazyPath = std.Build.LazyPath; const fs = std.fs; const mem = std.mem; - -const TranslateC = @This(); +const Configuration = std.Build.Configuration; pub const base_tag: Step.Tag = .translate_c; @@ -16,7 +17,7 @@ c_macros: std.array_list.Managed([]const u8), out_basename: []const u8, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, -output_file: std.Build.GeneratedFile, +output_file: Configuration.GeneratedFileIndex, link_libc: bool, pub const Options = struct { @@ -27,7 +28,9 @@ pub const Options = struct { }; pub fn create(owner: *std.Build, options: Options) *TranslateC { - const translate_c = owner.allocator.create(TranslateC) catch @panic("OOM"); + const graph = owner.graph; + const arena = graph.arena; + const translate_c = arena.create(TranslateC) catch @panic("OOM"); const source = options.root_source_file.dupe(owner); translate_c.* = .{ .step = Step.init(.{ @@ -37,12 +40,12 @@ pub fn create(owner: *std.Build, options: Options) *TranslateC { .makeFn = make, }), .source = source, - .include_dirs = std.array_list.Managed(std.Build.Module.IncludeDir).init(owner.allocator), - .c_macros = std.array_list.Managed([]const u8).init(owner.allocator), + .include_dirs = std.array_list.Managed(std.Build.Module.IncludeDir).init(arena), + .c_macros = std.array_list.Managed([]const u8).init(arena), .out_basename = undefined, .target = options.target, .optimize = options.optimize, - .output_file = .{ .step = &translate_c.step }, + .output_file = graph.addGeneratedFile(&translate_c.step), .link_libc = options.link_libc, .system_libs = .empty, }; @@ -59,7 +62,7 @@ pub const AddExecutableOptions = struct { }; pub fn getOutput(translate_c: *TranslateC) std.Build.LazyPath { - return .{ .generated = .{ .file = &translate_c.output_file } }; + return .{ .generated = .{ .index = translate_c.output_file } }; } /// Creates a module from the translated source and adds it to the package's diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 06f030efbc8003fb31334141be311e2487bb0c52..14097a76edff48beab3a94510f888f6e5ad0bc0e 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -9,13 +9,13 @@ const Dir = std.Io.Dir; const Step = std.Build.Step; const ArrayList = std.ArrayList; const assert = std.debug.assert; +const Configuration = std.Build.Configuration; step: Step, -/// The elements here are pointers because we need stable pointers for the GeneratedFile field. files: std.ArrayList(File), directories: std.ArrayList(Directory), -generated_directory: std.Build.GeneratedFile, +generated_directory: Configuration.GeneratedFileIndex, mode: Mode = .whole_cached, pub const base_tag: Step.Tag = .write_file; @@ -86,7 +86,9 @@ pub const Contents = union(enum) { }; pub fn create(owner: *std.Build) *WriteFile { - const write_file = owner.allocator.create(WriteFile) catch @panic("OOM"); + const graph = owner.graph; + const arena = graph.arena; + const write_file = arena.create(WriteFile) catch @panic("OOM"); write_file.* = .{ .step = Step.init(.{ .tag = base_tag, @@ -95,7 +97,7 @@ pub fn create(owner: *std.Build) *WriteFile { }), .files = .empty, .directories = .empty, - .generated_directory = .{ .step = &write_file.step }, + .generated_directory = graph.addGeneratedFile(&write_file.step), }; return write_file; } @@ -111,7 +113,7 @@ pub fn add(write_file: *WriteFile, sub_path: []const u8, bytes: []const u8) std. write_file.maybeUpdateName(); return .{ .generated = .{ - .file = &write_file.generated_directory, + .index = write_file.generated_directory, .sub_path = file.sub_path, }, }; @@ -137,7 +139,7 @@ pub fn addCopyFile(write_file: *WriteFile, source: std.Build.LazyPath, sub_path: source.addStepDependencies(&write_file.step); return .{ .generated = .{ - .file = &write_file.generated_directory, + .index = write_file.generated_directory, .sub_path = file.sub_path, }, }; @@ -165,7 +167,7 @@ pub fn addCopyDirectory( source.addStepDependencies(&write_file.step); return .{ .generated = .{ - .file = &write_file.generated_directory, + .index = write_file.generated_directory, .sub_path = dir.sub_path, }, }; @@ -174,7 +176,7 @@ pub fn addCopyDirectory( /// Returns a `LazyPath` representing the base directory that contains all the /// files from this `WriteFile`. pub fn getDirectory(write_file: *WriteFile) std.Build.LazyPath { - return .{ .generated = .{ .file = &write_file.generated_directory } }; + return .{ .generated = .{ .index = write_file.generated_directory } }; } fn maybeUpdateName(write_file: *WriteFile) void { diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 23d027e77715586f981171d85c68ed7effd9566c..b01c6cd8bc59c77b912561696031b3316506d9e2 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -15,6 +15,7 @@ system_integrations: []SystemIntegration, available_options: []AvailableOption, extra: []u32, default_step: Step.Index, +generated_files_len: u32, /// The field order here matches `Configuration` which documents the order in /// the serialized format. @@ -28,6 +29,9 @@ pub const Header = extern struct { extra_len: u32, default_step: Step.Index, + /// There is not actually any data stored for this - it just provides a way + /// for maker process to preallocate an array for these. + generated_files_len: u32, }; pub const Wip = struct { @@ -44,6 +48,7 @@ pub const Wip = struct { steps: std.ArrayList(Step) = .empty, path_deps: std.MultiArrayList(Path) = .empty, extra: std.ArrayList(u32) = .empty, + next_generated_file_index: u32 = 0, const DedupeTable = std.HashMapUnmanaged(ExtraSlice, void, ExtraSlice.Context, std.hash_map.default_max_load_percentage); const TargetsTable = std.HashMapUnmanaged(TargetQuery.Index, void, TargetsTableContext, std.hash_map.default_max_load_percentage); @@ -127,6 +132,7 @@ pub const Wip = struct { pub const Static = struct { default_step: Step.Index, + generated_files_len: u32, }; pub fn write(wip: *Wip, w: *Io.Writer, static: Static) Io.Writer.Error!void { @@ -140,6 +146,7 @@ pub const Wip = struct { .extra_len = @intCast(wip.extra.items.len), .default_step = static.default_step, + .generated_files_len = static.generated_files_len, }; var buffers = [_][]const u8{ @ptrCast(&header), @@ -363,6 +370,11 @@ pub const Wip = struct { const string = optional_string orelse return; wip.extra.appendAssumeCapacity(@intFromEnum(string)); } + + pub fn addGeneratedFile(wip: *Wip) GeneratedFileIndex { + defer wip.next_generated_file_index += 1; + return @enumFromInt(wip.next_generated_file_index); + } }; pub const SystemIntegration = extern struct { @@ -471,16 +483,16 @@ pub const Step = extern struct { dest_dir: InstallDestDir, dest_sub_path: String, - emitted_bin: OptionalLazyPath, + emitted_bin: LazyPath.OptionalIndex, implib_dir: InstallDestDir, - emitted_implib: OptionalLazyPath, + emitted_implib: LazyPath.OptionalIndex, pdb_dir: InstallDestDir, - emitted_pdb: OptionalLazyPath, + emitted_pdb: LazyPath.OptionalIndex, h_dir: InstallDestDir, - emitted_h: OptionalLazyPath, + emitted_h: LazyPath.OptionalIndex, /// Always a compile step. artifact: Step.Index, @@ -493,11 +505,11 @@ pub const Step = extern struct { }; /// Trailing: - /// * LazyPath for each file_inputs_len + /// * LazyPath.Index for each file_inputs_len /// * Arg for each args_len /// * environ_map if corresponding flag is set /// * stdin: Bytes, // if StdIn.bytes is chosen - /// * stdin: LazyPath, // if StdIn.lazy_path is chosen + /// * stdin: LazyPath.Index, // if StdIn.lazy_path is chosen /// * checks: Checks, // if StdIo.check is chosen /// * stdio_limit: u64, // if stdio_limit is set /// * producer: Step.Index, // if producer is set. always compile step @@ -505,7 +517,7 @@ pub const Step = extern struct { flags: @This().Flags, file_inputs_len: u32, args_len: u32, - cwd: OptionalLazyPath, + cwd: LazyPath.OptionalIndex, captured_stdout: OptionalString, // basename captured_stderr: OptionalString, // basename @@ -514,7 +526,7 @@ pub const Step = extern struct { /// * String if suffix set /// * String if basename set /// * Step.Index which is always a compile step if tag is artifact - /// * LazyPath if tag is path_file, path_directory, or file_content + /// * LazyPath.Index if tag is path_file, path_directory, or file_content pub const Arg = struct { flags: Arg.Flags, @@ -591,13 +603,13 @@ pub const Step = extern struct { installed_headers: Storage.FlagLengthPrefixedList(.flags, .installed_headers_len, Storage.Extended(InstalledHeader.Flags, InstalledHeader)), force_undefined_symbols: Storage.FlagLengthPrefixedList(.flags, .force_undefined_symbols_len, String), expect_errors: Storage.FlagUnion(.flags4, .expect_errors, ExpectErrors), - linker_script: Storage.FlagOptional(.flags4, .linker_script, LazyPath), - version_script: Storage.FlagOptional(.flags4, .version_script, LazyPath), - zig_lib_dir: Storage.FlagOptional(.flags3, .zig_lib_dir, LazyPath), - libc_file: Storage.FlagOptional(.flags4, .libc_file, LazyPath), - win32_manifest: Storage.FlagOptional(.flags3, .win32_manifest, LazyPath), - win32_module_definition: Storage.FlagOptional(.flags3, .win32_module_definition, LazyPath), - entitlements: Storage.FlagOptional(.flags4, .entitlements, LazyPath), + linker_script: Storage.FlagOptional(.flags4, .linker_script, LazyPath.Index), + version_script: Storage.FlagOptional(.flags4, .version_script, LazyPath.Index), + zig_lib_dir: Storage.FlagOptional(.flags3, .zig_lib_dir, LazyPath.Index), + libc_file: Storage.FlagOptional(.flags4, .libc_file, LazyPath.Index), + win32_manifest: Storage.FlagOptional(.flags3, .win32_manifest, LazyPath.Index), + win32_module_definition: Storage.FlagOptional(.flags3, .win32_module_definition, LazyPath.Index), + entitlements: Storage.FlagOptional(.flags4, .entitlements, LazyPath.Index), version: Storage.FlagOptional(.flags3, .version, String), // semantic version string entry: Storage.EnumOptional(.flags3, .entry, .symbol_name, String), install_name: Storage.FlagOptional(.flags4, .install_name, String), @@ -614,6 +626,16 @@ pub const Step = extern struct { build_id: Storage.EnumOptional(.flags3, .build_id, .hexstring, String), test_runner: Storage.FlagUnion(.flags3, .test_runner, TestRunner), + emit_directory: Storage.FlagOptional(.flags4, .emit_directory, GeneratedFileIndex), + generated_docs: Storage.FlagOptional(.flags4, .generated_docs, GeneratedFileIndex), + generated_asm: Storage.FlagOptional(.flags4, .generated_asm, GeneratedFileIndex), + generated_bin: Storage.FlagOptional(.flags4, .generated_bin, GeneratedFileIndex), + generated_pdb: Storage.FlagOptional(.flags4, .generated_pdb, GeneratedFileIndex), + generated_implib: Storage.FlagOptional(.flags4, .generated_implib, GeneratedFileIndex), + generated_llvm_bc: Storage.FlagOptional(.flags4, .generated_llvm_bc, GeneratedFileIndex), + generated_llvm_ir: Storage.FlagOptional(.flags4, .generated_llvm_ir, GeneratedFileIndex), + generated_h: Storage.FlagOptional(.flags4, .generated_h, GeneratedFileIndex), + pub const InstalledHeader = union(@This().Tag) { file: File, directory: Directory, @@ -630,7 +652,7 @@ pub const Step = extern struct { pub const File = struct { flags: @This().Flags = .{}, - source: LazyPath, + source: LazyPath.Index, dest_sub_path: String, pub const Flags = packed struct(u32) { @@ -641,7 +663,7 @@ pub const Step = extern struct { pub const Directory = struct { flags: @This().Flags, - source: LazyPath, + source: LazyPath.Index, dest_sub_path: String, exclude_extensions: Storage.FlagLengthPrefixedList(.flags, .exclude_extensions, String), include_extensions: Storage.FlagLengthPrefixedList(.flags, .include_extensions, String), @@ -667,8 +689,8 @@ pub const Step = extern struct { pub const Tag = enum(u2) { default, simple, server }; default: void, - simple: LazyPath, - server: LazyPath, + simple: LazyPath.Index, + server: LazyPath.Index, }; pub const Entry = enum(u2) { default, disabled, enabled, symbol_name }; @@ -857,12 +879,37 @@ pub const Step = extern struct { expect_errors: ExpectErrors.Tag, linker_script: bool, version_script: bool, - _: u18 = 0, + emit_directory: bool, + generated_docs: bool, + generated_asm: bool, + generated_bin: bool, + generated_pdb: bool, + generated_implib: bool, + generated_llvm_bc: bool, + generated_llvm_ir: bool, + generated_h: bool, + _: u9 = 0, }; pub fn isDynamicLibrary(compile: *const Compile) bool { return compile.flags3.kind == .lib and compile.flags2.linkage == .dynamic; } + + pub fn isStaticLibrary(compile: *const Compile) bool { + return compile.flags3.kind == .lib and compile.flags2.linkage != .dynamic; + } + + pub fn producesImplib(compile: *const Compile, c: *const Configuration) bool { + return isDll(compile, c); + } + + pub fn isDll(compile: *const Compile, c: *const Configuration) bool { + return isDynamicLibrary(compile) and rootModuleTarget(compile, c).flags.os_tag == .windows; + } + + pub fn rootModuleTarget(compile: *const Compile, c: *const Configuration) TargetQuery { + return compile.root_module.get(c).resolved_target.get(c).?.result.get(c); + } }; pub const CheckFile = struct { @@ -1001,32 +1048,49 @@ pub const MaxRss = enum(u32) { } }; -/// An index into `extra`, or `null`. -pub const OptionalLazyPath = enum(u32) { - none = maxInt(u32), - _, - - pub fn unwrap(this: @This()) ?LazyPath { - return switch (this) { - .none => null, - else => @enumFromInt(@intFromEnum(this)), - }; - } -}; - -/// An index into `extra`. -pub const LazyPath = enum(u32) { - _, +pub const LazyPath = union(@This().Tag) { + source_path: SourcePath, + relative: Relative, + generated: Generated, pub const Tag = enum(u8) { /// A source file path relative to build root. source_path, - generated, + /// Relative to the directory indicated in flags. relative, + /// Path is available only after it is populated by its owning step. + generated, + }; + + pub const Flags = packed struct(u32) { + tag: Tag, + _: u24 = 0, + }; + + /// An index into `extra`. + pub const Index = enum(u32) { + _, + + pub fn get(this: @This(), c: *const Configuration) LazyPath { + return extraData(c, LazyPath, @intFromEnum(this)); + } + }; + + /// An index into `extra`, or `null`. + pub const OptionalIndex = enum(u32) { + none = maxInt(u32), + _, + + pub fn unwrap(this: @This()) ?Index { + return switch (this) { + .none => null, + else => @enumFromInt(@intFromEnum(this)), + }; + } }; pub const SourcePath = struct { - flags: Flags, + flags: @This().Flags, owner: Package.Index, sub_path: String, @@ -1037,9 +1101,10 @@ pub const LazyPath = enum(u32) { }; pub const Generated = struct { - flags: Flags, + flags: @This().Flags = .{}, + index: GeneratedFileIndex, /// Applied after `up`. - sub_path: String, + sub_path: String = .empty, pub const Flags = packed struct(u32) { tag: Tag = .generated, @@ -1047,12 +1112,12 @@ pub const LazyPath = enum(u32) { /// 0 means the generated file itself. /// 1 means the directory of the generated file. /// 2 means the parent of that directory, and so on. - up: u24, + up: u24 = 0, }; }; pub const Relative = struct { - flags: Flags, + flags: @This().Flags, sub_path: String, pub const Flags = packed struct(u32) { @@ -1063,6 +1128,26 @@ pub const LazyPath = enum(u32) { }; }; +pub const GeneratedFileIndex = enum(u32) { + _, +}; + +pub const OptionalGeneratedFileIndex = enum(u32) { + none = maxInt(u32), + _, + + pub fn init(i: ?GeneratedFileIndex) OptionalGeneratedFileIndex { + return @enumFromInt(@intFromEnum(i orelse return .none)); + } + + pub fn unwrap(this: @This()) ?GeneratedFileIndex { + return switch (this) { + .none => null, + else => @enumFromInt(@intFromEnum(this)), + }; + } +}; + pub const Package = struct { dep_prefix: String, hash: String, @@ -1071,9 +1156,15 @@ pub const Package = struct { root = maxInt(u32), _, - pub fn depPrefixSlice(i: Index, c: *const Configuration) [:0]const u8 { - if (i == .root) return ""; - return extraData(c, Package, @intFromEnum(i)).dep_prefix.slice(c); + /// Returns `null` for root package. + pub fn get(i: @This(), c: *const Configuration) ?Package { + if (i == .root) return null; + return extraData(c, Package, @intFromEnum(i)); + } + + pub fn depPrefixSlice(i: @This(), c: *const Configuration) [:0]const u8 { + const package = get(i, c) orelse return ""; + return package.dep_prefix.slice(c); } }; }; @@ -1083,10 +1174,10 @@ pub const Module = struct { flags2: Flags2, import_table: ImportTable.Index, owner: Package.Index, - root_source_file: OptionalLazyPath, + root_source_file: LazyPath.OptionalIndex, resolved_target: ResolvedTarget.OptionalIndex, c_macros: Storage.FlagLengthPrefixedList(.flags, .c_macros, String), - lib_paths: Storage.FlagLengthPrefixedList(.flags, .lib_paths, LazyPath), + lib_paths: Storage.FlagLengthPrefixedList(.flags, .lib_paths, LazyPath.Index), export_symbol_names: Storage.FlagLengthPrefixedList(.flags, .export_symbol_names, String), include_dirs: Storage.UnionList(.flags, .include_dirs, IncludeDir), rpaths: Storage.UnionList(.flags, .rpaths, RPath), @@ -1195,29 +1286,29 @@ pub const Module = struct { }; pub const IncludeDir = union(enum(u3)) { - path: LazyPath, - path_system: LazyPath, - path_after: LazyPath, - framework_path: LazyPath, - framework_path_system: LazyPath, + path: LazyPath.Index, + path_system: LazyPath.Index, + path_after: LazyPath.Index, + framework_path: LazyPath.Index, + framework_path_system: LazyPath.Index, /// Always `Step.Tag.compile`. other_step: Step.Index, /// Always `Step.Tag.config_header`. config_header_step: Step.Index, - embed_path: LazyPath, + embed_path: LazyPath.Index, }; pub const RPath = union(enum(u1)) { - lazy_path: LazyPath, + lazy_path: LazyPath.Index, special: String, }; pub const LinkObject = union(enum(u3)) { - static_path: LazyPath, + static_path: LazyPath.Index, /// Always `Step.Tag.compile`. other_step: Step.Index, system_lib: SystemLib.Index, - assembly_file: LazyPath, + assembly_file: LazyPath.Index, c_source_file: CSourceFile.Index, c_source_files: CSourceFiles.Index, win32_resource_file: RcSourceFile.Index, @@ -1376,6 +1467,10 @@ pub const SystemLib = struct { pub const Index = enum(u32) { _, + + pub fn get(this: @This(), c: *const Configuration) SystemLib { + return extraData(c, SystemLib, @intFromEnum(this)); + } }; pub const UsePkgConfig = enum(u2) { @@ -1405,12 +1500,16 @@ pub const SystemLib = struct { pub const CSourceFiles = struct { flags: Flags, - root: LazyPath, + root: LazyPath.Index, args: Storage.FlagList(.flags, .args_len, String), sub_paths: Storage.LengthPrefixedList(String), pub const Index = enum(u32) { _, + + pub fn get(this: @This(), c: *const Configuration) CSourceFiles { + return extraData(c, CSourceFiles, @intFromEnum(this)); + } }; pub const Flags = packed struct(u32) { @@ -1422,11 +1521,15 @@ pub const CSourceFiles = struct { pub const CSourceFile = struct { flags: Flags, - file: LazyPath, + file: LazyPath.Index, args: Storage.FlagList(.flags, .args_len, String), pub const Index = enum(u32) { _, + + pub fn get(this: @This(), c: *const Configuration) CSourceFile { + return extraData(c, CSourceFile, @intFromEnum(this)); + } }; pub const Flags = packed struct(u32) { @@ -1438,12 +1541,16 @@ pub const CSourceFile = struct { pub const RcSourceFile = struct { flags: Flags, - file: LazyPath, + file: LazyPath.Index, args: Storage.FlagList(.flags, .args_len, String), - include_paths: Storage.FlagLengthPrefixedList(.flags, .include_paths, LazyPath), + include_paths: Storage.FlagLengthPrefixedList(.flags, .include_paths, LazyPath.Index), pub const Index = enum(u32) { _, + + pub fn get(this: @This(), c: *const Configuration) RcSourceFile { + return extraData(c, RcSourceFile, @intFromEnum(this)); + } }; pub const Flags = packed struct(u32) { @@ -1472,6 +1579,18 @@ pub const OptionalCSourceLanguage = enum(u3) { .assembly_with_preprocessor => .assembly_with_preprocessor, }; } + + pub fn get(this: @This()) ?std.Build.Module.CSourceLanguage { + return switch (this) { + .c => .c, + .cpp => .cpp, + .objective_c => .objective_c, + .objective_cpp => .objective_cpp, + .assembly => .assembly, + .assembly_with_preprocessor => .assembly_with_preprocessor, + .default => null, + }; + } }; pub const ResolvedTarget = struct { @@ -1483,7 +1602,7 @@ pub const ResolvedTarget = struct { pub const Index = enum(u32) { _, - pub fn get(this: @This(), c: *const Configuration) ?ResolvedTarget { + pub fn get(this: @This(), c: *const Configuration) ResolvedTarget { return extraData(c, ResolvedTarget, @intFromEnum(this)); } }; @@ -2006,13 +2125,27 @@ pub const Storage = enum { return end - i; } - pub fn data(buffer: []const u32, i: *usize, comptime S: type) S { - var result: S = undefined; - const fields = @typeInfo(S).@"struct".fields; - inline for (fields) |field| { - @field(result, field.name) = dataField(buffer, i, &result, field.type); + pub fn data(buffer: []const u32, i: *usize, comptime T: type) T { + switch (@typeInfo(T)) { + .@"struct" => |info| { + var result: T = undefined; + inline for (info.fields) |field| { + @field(result, field.name) = dataField(buffer, i, &result, field.type); + } + return result; + }, + .@"union" => |info| { + const flags: T.Flags = @bitCast(buffer[i.*]); + return switch (flags.tag) { + inline else => |comptime_tag| @unionInit( + T, + @tagName(comptime_tag), + data(buffer, i, info.fields[@intFromEnum(comptime_tag)].type), + ), + }; + }, + else => comptime unreachable, } - return result; } fn dataField(buffer: []const u32, i: *usize, container: anytype, comptime Field: type) Field { @@ -2332,6 +2465,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { .available_options = try arena.alloc(AvailableOption, header.available_options_len), .extra = try arena.alloc(u32, header.extra_len), .default_step = header.default_step, + .generated_files_len = header.generated_files_len, }; var vecs = [_][]u8{ result.string_bytes, -- 2.54.0 From ec2b156720dec2873ca0a7958b282bfda12d5250 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2026 17:11:01 -0700 Subject: [PATCH 040/179] std: rename zig.Configuration to Build.Configuration --- BRANCH_TODO | 2 +- lib/std/Build.zig | 2 +- lib/std/{zig => Build}/Configuration.zig | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename lib/std/{zig => Build}/Configuration.zig (100%) diff --git a/BRANCH_TODO b/BRANCH_TODO index 90eec2c8c8840eae47fce9964ddedde0511ddb10..a289b85243cec7e2e58804ac3df39cb7586af758 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ -* rename std.zig.Configuration to std.Build.Configuration * replace union(@This().Tag) * replace b.dupe() with string internment * don't forget to add -listen arg back @@ -10,3 +9,4 @@ * get zig tests passing * test a bunch of third party projects / help people migrate * refactor with DefaultingEnum +* add flag for compiling maker in debug mode diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 8b3b8ec5fd5a3d561a58c7c3c949704f7b04a1d4..cfe9dfa946c2cedfa8981460673181ee807eb966 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -22,7 +22,7 @@ pub const Step = @import("Build/Step.zig"); pub const Module = @import("Build/Module.zig"); pub const abi = @import("Build/abi.zig"); /// The serialized output of configure phase ingested by make phase. -pub const Configuration = @import("zig/Configuration.zig"); +pub const Configuration = @import("Build/Configuration.zig"); /// Shared state among all Build instances. graph: *Graph, diff --git a/lib/std/zig/Configuration.zig b/lib/std/Build/Configuration.zig similarity index 100% rename from lib/std/zig/Configuration.zig rename to lib/std/Build/Configuration.zig -- 2.54.0 From c8f7e270608e3f44b5dce898a6eab8efdcf56bd0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2026 18:40:04 -0700 Subject: [PATCH 041/179] zig build: add --debug-maker CLI flag for changing the optimization mode of the maker executable --- BRANCH_TODO | 2 +- lib/compiler/Maker/ScannedConfig.zig | 2 +- src/main.zig | 22 +++++++++++++++------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index a289b85243cec7e2e58804ac3df39cb7586af758..8b299493e263c12cd7902965b0b0eb009f423d95 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -9,4 +9,4 @@ * get zig tests passing * test a bunch of third party projects / help people migrate * refactor with DefaultingEnum -* add flag for compiling maker in debug mode +* inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 30a227ab965575b4fb583946bf5e22d3107c2911..9c4585957ea808209eb30fca746a3e4ada873733 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -341,7 +341,7 @@ pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { \\ none (default) No build ID \\ --debug-log [scope] Enable debugging the compiler \\ --debug-pkg-config Fail if unknown pkg-config flags encountered - \\ --debug-rt Debug compiler runtime libraries + \\ --debug-maker[=mode] Change maker executable optimization mode \\ --verbose-link Enable compiler debug output for linking \\ --verbose-air Enable compiler debug output for Zig AIR \\ --verbose-llvm-ir Enable compiler debug output for LLVM IR diff --git a/src/main.zig b/src/main.zig index c03bbc4e101247d875d6228023e4bf10d661063c..f0590f27de92b34b8e733dd0948af6457ebe9c09 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4942,6 +4942,10 @@ fn cmdBuild( var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); var override_make_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map); + var maker_optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map)) + .Debug + else + .ReleaseSafe; var configure_argv: std.ArrayList([]const u8) = .empty; var make_argv: std.ArrayList([]const u8) = .empty; var forks: std.ArrayList(Fork) = .empty; @@ -5079,6 +5083,12 @@ fn cmdBuild( }; } else if (mem.eql(u8, arg, "-fno-reference-trace")) { reference_trace = null; + } else if (mem.eql(u8, arg, "--debug-maker")) { + maker_optimize_mode = .Debug; + continue; + } else if (mem.cutPrefix(u8, arg, "--debug-maker=")) |rest| { + maker_optimize_mode = parseOptimizeMode(rest); + continue; } else if (mem.eql(u8, arg, "--debug-log")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); try make_argv.appendSlice(arena, args[i .. i + 2]); @@ -5270,6 +5280,7 @@ fn cmdBuild( .self_exe_path = self_exe_path, .color = color, .reference_trace = reference_trace, + .optimize_mode = maker_optimize_mode, } }); defer _ = make_runner_task.cancel(io) catch {}; @@ -5779,6 +5790,7 @@ const MakeRunner = struct { thread_limit: usize, color: Color, reference_trace: ?u32, + optimize_mode: std.builtin.OptimizeMode, }; }; @@ -5786,11 +5798,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn const compile_prog_node = options.parent_prog_node.start("Compile Maker", 0); defer compile_prog_node.end(); - const optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(options.environ_map)) - .Debug - else - .ReleaseSafe; - const strip = optimize_mode != .Debug; + const strip = options.optimize_mode != .Debug; const main_mod_paths: Package.Module.CreateOptions.Paths = .{ .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"), @@ -5800,7 +5808,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn const config = try Compilation.Config.resolve(.{ .output_mode = .Exe, .root_strip = strip, - .root_optimize_mode = optimize_mode, + .root_optimize_mode = options.optimize_mode, .resolved_target = options.resolved_target, .have_zcu = true, .emit_bin = true, @@ -5813,7 +5821,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn .cc_argv = &.{}, .inherited = .{ .resolved_target = options.resolved_target, - .optimize_mode = optimize_mode, + .optimize_mode = options.optimize_mode, .strip = strip, }, .global = config, -- 2.54.0 From cc4f205fc3a55eb788b8663fa2817ccd6761a4eb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2026 21:09:22 -0700 Subject: [PATCH 042/179] maker: progress towards lowering zig cli args --- lib/compiler/Maker/Step/Compile.zig | 233 ++++++++++++++++------------ lib/std/Build/Configuration.zig | 4 + 2 files changed, 142 insertions(+), 95 deletions(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 77d1f45b3175b639721548a14a8127e94a31240f..e65d26e88a272d8a9134a5a870dd95f6e1595b94 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -478,11 +478,13 @@ fn lowerZigArgs( if (!my_responsibility) continue; if (cli_named_modules.modules.getIndex(mod_index)) |module_cli_index| { const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; + const module_index = cli_named_modules.modules.keys()[module_cli_index]; + try appendModuleFlags(module_index, zig_args, compile_index, maker); + if (true) @panic("TODO"); - try appendModuleFlags(zig_args, step); // --dep arguments - try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); + try zig_args.ensureUnusedCapacity(gpa, mod.import_table.count() * 2); for (mod.import_table.keys(), mod.import_table.values()) |name, import| { const import_index = cli_named_modules.modules.getIndex(import).?; const import_cli_name = cli_named_modules.names.keys()[import_index]; @@ -1389,129 +1391,170 @@ fn getModuleList( } fn appendModuleFlags( - m: *Module, - zig_args: *std.array_list.Managed([]const u8), - asking_step: ?*Step, + module_index: Configuration.Module.Index, + zig_args: *std.ArrayList([]const u8), + asking_step: Configuration.Step.Index, + maker: *const Maker, ) !void { - const b = m.owner; - - try addFlag(zig_args, m.strip, "-fstrip", "-fno-strip"); - try addFlag(zig_args, m.single_threaded, "-fsingle-threaded", "-fno-single-threaded"); - try addFlag(zig_args, m.stack_check, "-fstack-check", "-fno-stack-check"); - try addFlag(zig_args, m.stack_protector, "-fstack-protector", "-fno-stack-protector"); - try addFlag(zig_args, m.omit_frame_pointer, "-fomit-frame-pointer", "-fno-omit-frame-pointer"); - try addFlag(zig_args, m.error_tracing, "-ferror-tracing", "-fno-error-tracing"); - try addFlag(zig_args, m.sanitize_thread, "-fsanitize-thread", "-fno-sanitize-thread"); - try addFlag(zig_args, m.fuzz, "-ffuzz", "-fno-fuzz"); - try addFlag(zig_args, m.valgrind, "-fvalgrind", "-fno-valgrind"); - try addFlag(zig_args, m.pic, "-fPIC", "-fno-PIC"); - try addFlag(zig_args, m.red_zone, "-mred-zone", "-mno-red-zone"); - try addFlag(zig_args, m.no_builtin, "-fno-builtin", "-fbuiltin"); - - if (m.sanitize_c) |sc| switch (sc) { - .off => try zig_args.append("-fno-sanitize-c"), - .trap => try zig_args.append("-fsanitize-c=trap"), - .full => try zig_args.append("-fsanitize-c=full"), - }; - - if (m.dwarf_format) |dwarf_format| { - try zig_args.append(switch (dwarf_format) { - .@"32" => "-gdwarf32", - .@"64" => "-gdwarf64", - }); - } + const gpa = maker.gpa; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + const conf = &maker.scanned_config.configuration; + const m = module_index.get(conf); - if (m.unwind_tables) |unwind_tables| { - try zig_args.append(switch (unwind_tables) { - .none => "-fno-unwind-tables", - .sync => "-funwind-tables", - .async => "-fasync-unwind-tables", - }); - } + try addFlag(gpa, zig_args, "strip", m.flags.strip.toBool()); + try addFlag(gpa, zig_args, "single-threaded", m.flags.single_threaded.toBool()); + try addFlag(gpa, zig_args, "stack-check", m.flags.stack_check.toBool()); + try addFlag(gpa, zig_args, "stack-protector", m.flags.stack_protector.toBool()); + try addFlag(gpa, zig_args, "omit-frame-pointer", m.flags2.omit_frame_pointer.toBool()); + try addFlag(gpa, zig_args, "error-tracing", m.flags2.error_tracing.toBool()); + try addFlag(gpa, zig_args, "sanitize-thread", m.flags.sanitize_thread.toBool()); + try addFlag(gpa, zig_args, "fuzz", m.flags.fuzz.toBool()); + try addFlag(gpa, zig_args, "valgrind", m.flags2.valgrind.toBool()); + try addFlag(gpa, zig_args, "PIC", m.flags2.pic.toBool()); + try addFlag(gpa, zig_args, "red-zone", m.flags2.red_zone.toBool()); + try addFlag(gpa, zig_args, "no-builtin", m.flags2.no_builtin.toBool()); + + { + try zig_args.ensureUnusedCapacity(gpa, 6); + + switch (m.flags.sanitize_c) { + .off => zig_args.appendAssumeCapacity("-fno-sanitize-c"), + .trap => zig_args.appendAssumeCapacity("-fsanitize-c=trap"), + .full => zig_args.appendAssumeCapacity("-fsanitize-c=full"), + .default => {}, + } + + switch (m.flags.dwarf_format) { + .@"32" => zig_args.appendAssumeCapacity("-gdwarf32"), + .@"64" => zig_args.appendAssumeCapacity("-gdwarf64"), + .default => {}, + } + + switch (m.flags.unwind_tables) { + .none => zig_args.appendAssumeCapacity("-fno-unwind-tables"), + .sync => zig_args.appendAssumeCapacity("-funwind-tables"), + .async => zig_args.appendAssumeCapacity("-fasync-unwind-tables"), + .default => {}, + } + + switch (m.flags.optimize) { + .debug => zig_args.appendAssumeCapacity("-ODebug"), + .safe => zig_args.appendAssumeCapacity("-OReleaseSafe"), + .fast => zig_args.appendAssumeCapacity("-OReleaseFast"), + .small => zig_args.appendAssumeCapacity("-OReleaseSmall"), + .default => {}, + } - try zig_args.ensureUnusedCapacity(1); - if (m.optimize) |optimize| switch (optimize) { - .Debug => zig_args.appendAssumeCapacity("-ODebug"), - .ReleaseSmall => zig_args.appendAssumeCapacity("-OReleaseSmall"), - .ReleaseFast => zig_args.appendAssumeCapacity("-OReleaseFast"), - .ReleaseSafe => zig_args.appendAssumeCapacity("-OReleaseSafe"), - }; - - if (m.code_model != .default) { - try zig_args.append("-mcmodel"); - try zig_args.append(@tagName(m.code_model)); + if (m.flags.code_model != .default) { + zig_args.appendAssumeCapacity("-mcmodel"); + zig_args.appendAssumeCapacity(@tagName(m.flags.code_model)); + } } - if (m.resolved_target) |*target| { + if (m.resolved_target.get(conf)) |target| { // Communicate the query via CLI since it's more compact. - if (!target.query.isNative()) { - try zig_args.appendSlice(&.{ - "-target", try target.query.zigTriple(b.allocator), - "-mcpu", try target.query.serializeCpuAlloc(b.allocator), - }); - if (target.query.dynamic_linker) |*dynamic_linker| { - if (dynamic_linker.get()) |dynamic_linker_path| { - try zig_args.append("--dynamic-linker"); - try zig_args.append(dynamic_linker_path); + if (target.query.get(conf)) |query| { + try zig_args.ensureUnusedCapacity(gpa, 6); + + if (true) @panic("TODO"); + + zig_args.appendAssumeCapacity("-target"); + zig_args.appendAssumeCapacity(try query.zigTriple(arena)); + zig_args.appendAssumeCapacity("-mcpu"); + zig_args.appendAssumeCapacity(try query.serializeCpuAlloc(arena)); + + if (query.dynamic_linker) |dynamic_linker| { + const dynamic_linker_slice = dynamic_linker.slice(conf); + if (dynamic_linker_slice.len != 0) { + zig_args.appendAssumeCapacity("--dynamic-linker"); + zig_args.appendAssumeCapacity(dynamic_linker_slice); } else { - try zig_args.append("--no-dynamic-linker"); + zig_args.appendAssumeCapacity("--no-dynamic-linker"); } } } } - for (m.export_symbol_names) |symbol_name| { - try zig_args.append(b.fmt("--export={s}", .{symbol_name})); + for (m.export_symbol_names.slice) |symbol_name| { + try zig_args.append(gpa, try allocPrint(arena, "--export={s}", .{symbol_name.slice(conf)})); } - for (m.include_dirs.items) |include_dir| { - try appendIncludeDirFlags(include_dir, b, zig_args, asking_step); - } + for (0..m.include_dirs.len) |i| + try appendIncludeDirFlags(m.include_dirs.get(conf.extra, i), zig_args, asking_step, maker); - try zig_args.appendSlice(m.c_macros.items); + try zig_args.ensureUnusedCapacity(gpa, m.c_macros.slice.len); + for (m.c_macros.slice) |c_macro| + zig_args.appendAssumeCapacity(c_macro.slice(conf)); - try zig_args.ensureUnusedCapacity(2 * m.lib_paths.items.len); - for (m.lib_paths.items) |lib_path| { + try zig_args.ensureUnusedCapacity(gpa, 2 * m.lib_paths.slice.len); + for (m.lib_paths.slice) |lib_path| { zig_args.appendAssumeCapacity("-L"); - zig_args.appendAssumeCapacity(lib_path.getPath2(b, asking_step)); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lib_path, asking_step)); } - try zig_args.ensureUnusedCapacity(2 * m.rpaths.items.len); - for (m.rpaths.items) |rpath| switch (rpath) { + try zig_args.ensureUnusedCapacity(gpa, 2 * m.rpaths.len); + for (0..m.rpaths.len) |i| switch (m.rpaths.get(conf.extra, i)) { .lazy_path => |lp| { zig_args.appendAssumeCapacity("-rpath"); - zig_args.appendAssumeCapacity(lp.getPath2(b, asking_step)); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, asking_step)); }, - .special => |bytes| { + .special => |string| { zig_args.appendAssumeCapacity("-rpath"); - zig_args.appendAssumeCapacity(bytes); + zig_args.appendAssumeCapacity(string.slice(conf)); }, }; } fn appendIncludeDirFlags( include_dir: Configuration.Module.IncludeDir, - b: *std.Build, - zig_args: *std.array_list.Managed([]const u8), - asking_step: ?*Step, + zig_args: *std.ArrayList([]const u8), + asking_step: Configuration.Step.Index, + maker: *const Maker, ) !void { - const flag: []const u8, const lazy_path: Configuration.LazyPath = switch (include_dir) { - // zig fmt: off - .path => |lp| .{ "-I", lp }, - .path_system => |lp| .{ "-isystem", lp }, - .path_after => |lp| .{ "-idirafter", lp }, - .framework_path => |lp| .{ "-F", lp }, - .framework_path_system => |lp| .{ "-iframework", lp }, - .config_header_step => |ch| .{ "-I", ch.getOutputDir() }, - .other_step => |comp| .{ "-I", comp.installed_headers_include_tree.?.getDirectory() }, - // zig fmt: on + const gpa = maker.gpa; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + + try zig_args.ensureUnusedCapacity(gpa, 2); + switch (include_dir) { + .path => |lp| { + zig_args.appendAssumeCapacity("-I"); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, asking_step)); + }, + .path_system => |lp| { + zig_args.appendAssumeCapacity("-isystem"); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, asking_step)); + }, + .path_after => |lp| { + zig_args.appendAssumeCapacity("-idirafter"); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, asking_step)); + }, + .framework_path => |lp| { + zig_args.appendAssumeCapacity("-F"); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, asking_step)); + }, + .framework_path_system => |lp| { + zig_args.appendAssumeCapacity("-iframework"); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, asking_step)); + }, + .config_header_step => |ch| { + zig_args.appendAssumeCapacity("-I"); + if (true) @panic("TODO"); + ch.getOutputDir(); + }, + .other_step => |comp| { + zig_args.appendAssumeCapacity("-I"); + if (true) @panic("TODO"); + comp.installed_headers_include_tree.?.getDirectory(); + }, .embed_path => |lazy_path| { - // Special case: this is a single arg. - const resolved = lazy_path.getPath3(b, asking_step); - const arg = b.fmt("--embed-dir={f}", .{resolved}); - return zig_args.append(arg); + try zig_args.append( + gpa, + try allocPrint(arena, "--embed-dir={f}", .{ + try maker.resolveLazyPathIndex(arena, lazy_path, asking_step), + }), + ); }, - }; - const resolved_str = try lazy_path.getPath3(b, asking_step).toString(b.graph.arena); - return zig_args.appendSlice(&.{ flag, resolved_str }); + } } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index b01c6cd8bc59c77b912561696031b3316506d9e2..a93c9e9249b8781865d5c30284d5e549f992a233 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1668,6 +1668,10 @@ pub const TargetQuery = struct { _ => @enumFromInt(@intFromEnum(this)), }; } + + pub fn get(this: @This(), c: *const Configuration) ?TargetQuery { + return (unwrap(this) orelse return null).get(c); + } }; pub const CpuModel = enum(u2) { -- 2.54.0 From a60ffaf5b357cd85200fffba8bd0e98df5f91a16 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 12 Mar 2026 17:06:12 -0700 Subject: [PATCH 043/179] maker: finish migrating most of CLI lowering code --- BRANCH_TODO | 4 + lib/compiler/Maker/Step/Compile.zig | 427 ++++++++++++++-------------- lib/std/Build/Configuration.zig | 37 +++ 3 files changed, 248 insertions(+), 220 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 8b299493e263c12cd7902965b0b0eb009f423d95..dbd9fedf23a26edf2c9b78008e15f8229329dd01 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -10,3 +10,7 @@ * test a bunch of third party projects / help people migrate * refactor with DefaultingEnum * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) + +## Followup Issues +* link_eh_frame_hdr should be DefaultingBool +* make --foo, --no-foo CLI args uniform (make them -f args instead) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index e65d26e88a272d8a9134a5a870dd95f6e1595b94..c1d903668b48df83db0afd6fa8179ac0e9ff529c 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -15,10 +15,10 @@ const allocPrint = std.fmt.allocPrint; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); -/// Populated during the make phase when there is a long-lived compiler process. -/// Managed by the build runner, not user build script. +/// Populated when there is compiler process that lives across multiple calls +/// to `make`. zig_process: ?*Step.ZigProcess = null, -/// Persisted to reuse memory on subsequent make. +/// Persisted to reuse memory on subsequent calls to `make`. zig_args: std.ArrayList([]const u8) = .empty, pub fn make( @@ -538,28 +538,29 @@ fn lowerZigArgs( try zig_args.append(gpa, try maker.resolveLazyPathIndexAbs(arena, manifest_file, compile_index)); } - if (true) @panic("TODO"); - - if (conf_comp.win32_module_definition) |module_file| { - try zig_args.append(gpa, module_file.getPath2(step)); + if (conf_comp.win32_module_definition.value) |module_file| { + try zig_args.append(gpa, try maker.resolveLazyPathIndexAbs(arena, module_file, compile_index)); } - if (conf_comp.image_base) |image_base| { - try zig_args.appendSlice(gpa, &.{ + if (conf_comp.image_base.value) |image_base| { + (try zig_args.addManyAsArray(gpa, 2)).* = .{ "--image-base", try allocPrint(arena, "0x{x}", .{image_base}), - }); + }; } - for (conf_comp.filters) |filter| { - try zig_args.appendSlice(gpa, &.{ "--test-filter", filter }); + for (conf_comp.filters.slice) |filter| { + (try zig_args.addManyAsArray(gpa, 2)).* = .{ "--test-filter", filter.slice(conf) }; } - if (conf_comp.test_runner) |test_runner| { - try zig_args.appendSlice(gpa, &.{ "--test-runner", test_runner.path.getPath2(step) }); + switch (conf_comp.test_runner.u) { + .default => {}, + .simple, .server => |lp| (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "--test-runner", try maker.resolveLazyPathIndexAbs(arena, lp, compile_index), + }, } - for (graph.debug_log_scopes) |log_scope| { - try zig_args.appendSlice(gpa, &.{ "--debug-log", log_scope }); + for (graph.debug_log_scopes.items) |log_scope| { + (try zig_args.addManyAsArray(gpa, 2)).* = .{ "--debug-log", log_scope }; } try addBool(gpa, zig_args, "--debug-compile-errors", graph.debug_compile_errors); @@ -571,179 +572,172 @@ fn lowerZigArgs( try addBool(gpa, zig_args, "--verbose-llvm-cpu-features", graph.verbose_llvm_cpu_features); try addBool(gpa, zig_args, "--time-report", graph.time_report); - if (compile.generated_asm != null) try zig_args.append(gpa, "-femit-asm"); - if (compile.generated_bin == null) try zig_args.append(gpa, "-fno-emit-bin"); - if (compile.generated_docs != null) try zig_args.append(gpa, "-femit-docs"); - if (compile.generated_implib != null) try zig_args.append(gpa, "-femit-implib"); - if (compile.generated_llvm_bc != null) try zig_args.append(gpa, "-femit-llvm-bc"); - if (compile.generated_llvm_ir != null) try zig_args.append(gpa, "-femit-llvm-ir"); - if (compile.generated_h != null) try zig_args.append(gpa, "-femit-h"); + if (conf_comp.generated_bin.value == null) try zig_args.append(gpa, "-fno-emit-bin"); + if (conf_comp.generated_asm.value != null) try zig_args.append(gpa, "-femit-asm"); + if (conf_comp.generated_docs.value != null) try zig_args.append(gpa, "-femit-docs"); + if (conf_comp.generated_implib.value != null) try zig_args.append(gpa, "-femit-implib"); + if (conf_comp.generated_llvm_bc.value != null) try zig_args.append(gpa, "-femit-llvm-bc"); + if (conf_comp.generated_llvm_ir.value != null) try zig_args.append(gpa, "-femit-llvm-ir"); + if (conf_comp.generated_h.value != null) try zig_args.append(gpa, "-femit-h"); - try addFlag(gpa, zig_args, "formatted-panics", conf_comp.flags.formatted_panics); + try addFlag(gpa, zig_args, "formatted-panics", conf_comp.flags2.formatted_panics.toBool()); - switch (conf_comp.compress_debug_sections) { + switch (conf_comp.flags3.compress_debug_sections) { .none => {}, .zlib => try zig_args.append(gpa, "--compress-debug-sections=zlib"), .zstd => try zig_args.append(gpa, "--compress-debug-sections=zstd"), } - if (conf_comp.flags.link_eh_frame_hdr) { - try zig_args.append(gpa, "--eh-frame-hdr"); - } - if (conf_comp.flags.link_emit_relocs) { - try zig_args.append(gpa, "--emit-relocs"); - } - if (conf_comp.flags.link_function_sections) { - try zig_args.append(gpa, "-ffunction-sections"); - } - if (conf_comp.flags.link_data_sections) { - try zig_args.append(gpa, "-fdata-sections"); - } - if (conf_comp.flags.link_gc_sections) |x| { + try addBool(gpa, zig_args, "--eh-frame-hdr", conf_comp.flags.link_eh_frame_hdr); + try addBool(gpa, zig_args, "--emit-relocs", conf_comp.flags.link_emit_relocs); + try addBool(gpa, zig_args, "-ffunction-sections", conf_comp.flags.link_function_sections); + try addBool(gpa, zig_args, "-fdata-sections", conf_comp.flags.link_data_sections); + + if (conf_comp.flags2.link_gc_sections.toBool()) |x| try zig_args.append(gpa, if (x) "--gc-sections" else "--no-gc-sections"); - } - if (!conf_comp.flags.linker_dynamicbase) { + + if (!conf_comp.flags.linker_dynamicbase) try zig_args.append(gpa, "--no-dynamicbase"); - } - if (conf_comp.flags.linker_allow_shlib_undefined) |x| { - try zig_args.append(gpa, if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); - } - if (conf_comp.flags.link_z_notext) try zig_args.appendSlice(gpa, &.{ "-z", "notext" }); - if (!conf_comp.flags.link_z_relro) try zig_args.appendSlice(gpa, &.{ "-z", "norelro" }); - if (conf_comp.flags.link_z_lazy) try zig_args.appendSlice(gpa, &.{ "-z", "lazy" }); - if (conf_comp.flags.link_z_common_page_size) |size| try zig_args.appendSlice(gpa, &.{ - "-z", - try allocPrint(arena, "common-page-size={d}", .{size}), - }); - if (conf_comp.flags.link_z_max_page_size) |size| try zig_args.appendSlice(gpa, &.{ - "-z", - try allocPrint(arena, "max-page-size={d}", .{size}), - }); - if (conf_comp.flags.link_z_defs) try zig_args.appendSlice(gpa, &.{ "-z", "defs" }); - if (conf_comp.flags.libc_file) |libc_file| { - try zig_args.appendSlice(gpa, &.{ "--libc", libc_file.getPath2(step) }); + try addFlag(gpa, zig_args, "allow-shlib-undefined", conf_comp.flags2.linker_allow_shlib_undefined.toBool()); + if (conf_comp.flags.link_z_notext) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "notext" }; + if (!conf_comp.flags.link_z_relro) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "norelro" }; + if (conf_comp.flags.link_z_lazy) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "lazy" }; + if (conf_comp.link_z_common_page_size.value) |size| (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "-z", try allocPrint(arena, "common-page-size={d}", .{size}), + }; + if (conf_comp.link_z_max_page_size.value) |size| (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "-z", try allocPrint(arena, "max-page-size={d}", .{size}), + }; + if (conf_comp.flags.link_z_defs) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "defs" }; + + try zig_args.ensureUnusedCapacity(gpa, 2); + if (conf_comp.libc_file.value) |libc_file| { + zig_args.appendAssumeCapacity("--libc"); + zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, libc_file, compile_index)); } else if (graph.libc_file) |libc_file| { - try zig_args.appendSlice(gpa, &.{ "--libc", libc_file }); + zig_args.appendAssumeCapacity("--libc"); + zig_args.appendAssumeCapacity(libc_file); } - try zig_args.append(gpa, "--cache-dir"); - try zig_args.append(gpa, graph.cache_root.path orelse "."); + (try zig_args.addManyAsArray(gpa, 4)).* = .{ + "--cache-dir", graph.local_cache_root.path orelse ".", + "--global-cache-dir", graph.global_cache_root.path orelse ".", + }; - try zig_args.append(gpa, "--global-cache-dir"); - try zig_args.append(gpa, graph.global_cache_root.path orelse "."); + try zig_args.ensureUnusedCapacity(gpa, 1); + if (graph.debug_compiler_runtime_libs) |mode| switch (mode) { + .Debug => zig_args.appendAssumeCapacity("--debug-rt"), + else => zig_args.appendAssumeCapacity(try allocPrint(arena, "--debug-rt={t}", .{mode})), + }; - if (graph.debug_compiler_runtime_libs) |mode| - try zig_args.append(gpa, try allocPrint(arena, "--debug-rt={t}", .{mode})); + { + try zig_args.ensureUnusedCapacity(gpa, 7); - try zig_args.appendSlice(gpa, &.{ "--name", conf_comp.root_name.slice(conf) }); + zig_args.addManyAsArrayAssumeCapacity(2).* = .{ "--name", conf_comp.root_name.slice(conf) }; - if (compile.linkage) |some| switch (some) { - .dynamic => try zig_args.append(gpa, "-dynamic"), - .static => try zig_args.append(gpa, "-static"), - }; - if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { - if (compile.version) |version| try zig_args.appendSlice(gpa, &.{ - "--version", try allocPrint(arena, "{f}", .{version}), - }); + switch (conf_comp.flags2.linkage) { + .dynamic => zig_args.appendAssumeCapacity("-dynamic"), + .static => zig_args.appendAssumeCapacity("-static"), + .default => {}, + } - if (root_module_target.flags.os_tag.isDarwin()) { - const install_name = compile.install_name orelse try allocPrint(arena, "@rpath/{s}{s}{s}", .{ - root_module_target.libPrefix(), - compile.name, - root_module_target.dynamicLibSuffix(), - }); - try zig_args.appendSlice(gpa, &.{ "-install_name", install_name }); + if (conf_comp.flags3.kind == .lib and conf_comp.flags2.linkage == .dynamic) { + if (conf_comp.version.value) |version| zig_args.addManyAsArrayAssumeCapacity(2).* = .{ + "--version", version.slice(conf), + }; + + const os_tag = root_module_target.flags.os_tag.unwrap().?; + if (os_tag.isDarwin()) { + const abi = root_module_target.flags.abi.unwrap().?; + zig_args.addManyAsArrayAssumeCapacity(2).* = .{ + "-install_name", + if (conf_comp.install_name.value) |s| s.slice(conf) else try allocPrint( + arena, + "@rpath/{s}{s}{s}", + .{ + os_tag.libPrefix(abi), + conf_comp.root_name.slice(conf), + os_tag.dynamicLibSuffix(), + }, + ), + }; + } } } - if (compile.entitlements) |entitlements| { - try zig_args.appendSlice(gpa, &.{ "--entitlements", entitlements }); - } - if (compile.pagezero_size) |pagezero_size| { - const size = try allocPrint(arena, "{x}", .{pagezero_size}); - try zig_args.appendSlice(gpa, &.{ "-pagezero_size", size }); - } - if (compile.headerpad_size) |headerpad_size| { - const size = try allocPrint(arena, "{x}", .{headerpad_size}); - try zig_args.appendSlice(gpa, &.{ "-headerpad", size }); - } - if (compile.headerpad_max_install_names) { - try zig_args.append(gpa, "-headerpad_max_install_names"); - } - if (compile.dead_strip_dylibs) { - try zig_args.append(gpa, "-dead_strip_dylibs"); - } - if (compile.force_load_objc) { - try zig_args.append(gpa, "-ObjC"); - } - if (compile.discard_local_symbols) { - try zig_args.append(gpa, "--discard-all"); - } + if (conf_comp.entitlements.value) |entitlements| { + (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "--entitlements", try maker.resolveLazyPathIndexAbs(arena, entitlements, compile_index), + }; + } + if (conf_comp.pagezero_size.value) |pagezero_size| { + (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "-pagezero_size", try allocPrint(arena, "{x}", .{pagezero_size}), + }; + } + if (conf_comp.headerpad_size.value) |headerpad_size| { + (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "-headerpad", try allocPrint(arena, "{x}", .{headerpad_size}), + }; + } + try addBool(gpa, zig_args, "-headerpad_max_install_names", conf_comp.flags.headerpad_max_install_names); + try addBool(gpa, zig_args, "-dead_strip_dylibs", conf_comp.flags.dead_strip_dylibs); + try addBool(gpa, zig_args, "-ObjC", conf_comp.flags.force_load_objc); + try addBool(gpa, zig_args, "--discard-all", conf_comp.flags.discard_local_symbols); - try addFlag(gpa, zig_args, "compiler-rt", compile.bundle_compiler_rt); - try addFlag(gpa, zig_args, "ubsan-rt", compile.bundle_ubsan_rt); - try addFlag(gpa, zig_args, "dll-export-fns", compile.dll_export_fns); - if (compile.rdynamic) { - try zig_args.append(gpa, "-rdynamic"); - } - if (compile.import_memory) { - try zig_args.append(gpa, "--import-memory"); - } - if (compile.export_memory) { - try zig_args.append(gpa, "--export-memory"); - } - if (compile.import_symbols) { - try zig_args.append(gpa, "--import-symbols"); - } - if (compile.import_table) { - try zig_args.append(gpa, "--import-table"); - } - if (compile.export_table) { - try zig_args.append(gpa, "--export-table"); - } - if (compile.initial_memory) |initial_memory| { - try zig_args.append(gpa, try allocPrint(arena, "--initial-memory={d}", .{initial_memory})); - } - if (compile.max_memory) |max_memory| { - try zig_args.append(gpa, try allocPrint(arena, "--max-memory={d}", .{max_memory})); - } - if (compile.shared_memory) { - try zig_args.append(gpa, "--shared-memory"); - } - if (compile.global_base) |global_base| { - try zig_args.append(gpa, try allocPrint(arena, "--global-base={d}", .{global_base})); - } + try addFlag(gpa, zig_args, "compiler-rt", conf_comp.flags2.bundle_compiler_rt.toBool()); + try addFlag(gpa, zig_args, "ubsan-rt", conf_comp.flags2.bundle_ubsan_rt.toBool()); + try addFlag(gpa, zig_args, "dll-export-fns", conf_comp.flags2.dll_export_fns.toBool()); - if (compile.wasi_exec_model) |model| { - try zig_args.append(gpa, try allocPrint(arena, "-mexec-model={t}", .{model})); - } - if (compile.linker_script) |linker_script| { - try zig_args.append(gpa, "--script"); - try zig_args.append(gpa, linker_script.getPath2(step)); - } + try addBool(gpa, zig_args, "-rdynamic", conf_comp.flags.rdynamic); + try addBool(gpa, zig_args, "--import-memory", conf_comp.flags.import_memory); + try addBool(gpa, zig_args, "--export-memory", conf_comp.flags.export_memory); + try addBool(gpa, zig_args, "--import-symbols", conf_comp.flags.import_symbols); + try addBool(gpa, zig_args, "--import-table", conf_comp.flags.import_table); + try addBool(gpa, zig_args, "--export-table", conf_comp.flags.export_table); + try addBool(gpa, zig_args, "--shared-memory", conf_comp.flags.shared_memory); - if (compile.version_script) |version_script| { - try zig_args.append(gpa, "--version-script"); - try zig_args.append(gpa, version_script.getPath2(step)); + { + try zig_args.ensureUnusedCapacity(gpa, 4); + if (conf_comp.initial_memory.value) |initial_memory| { + zig_args.appendAssumeCapacity(try allocPrint(arena, "--initial-memory={d}", .{initial_memory})); + } + if (conf_comp.max_memory.value) |max_memory| { + zig_args.appendAssumeCapacity(try allocPrint(arena, "--max-memory={d}", .{max_memory})); + } + if (conf_comp.global_base.value) |global_base| { + zig_args.appendAssumeCapacity(try allocPrint(arena, "--global-base={d}", .{global_base})); + } + switch (conf_comp.flags3.wasi_exec_model) { + .default => {}, + .command => zig_args.appendAssumeCapacity("-mexec-model=command"), + .reactor => zig_args.appendAssumeCapacity("-mexec-model=reactor"), + } } - if (compile.linker_allow_undefined_version) |x| { + + if (conf_comp.linker_script.value) |linker_script| (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "--script", try maker.resolveLazyPathIndexAbs(arena, linker_script, compile_index), + }; + if (conf_comp.version_script.value) |version_script| (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "--version-script", try maker.resolveLazyPathIndexAbs(arena, version_script, compile_index), + }; + if (conf_comp.flags2.linker_allow_undefined_version.toBool()) |x| { try zig_args.append(gpa, if (x) "--undefined-version" else "--no-undefined-version"); } - if (compile.linker_enable_new_dtags) |enabled| { + if (conf_comp.flags2.linker_enable_new_dtags.toBool()) |enabled| { try zig_args.append(gpa, if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); } - if (compile.kind == .@"test") { - if (compile.exec_cmd_args) |exec_cmd_args| { - for (exec_cmd_args) |cmd_arg| { - if (cmd_arg) |arg| { - try zig_args.append(gpa, "--test-cmd"); - try zig_args.append(gpa, arg); - } else { - try zig_args.append(gpa, "--test-cmd-bin"); - } + if (conf_comp.flags3.kind == .@"test" and conf_comp.exec_cmd_args.slice.len != 0) { + for (conf_comp.exec_cmd_args.slice) |cmd_arg| { + try zig_args.ensureUnusedCapacity(gpa, 2); + if (cmd_arg.slice(conf)) |arg| { + zig_args.appendAssumeCapacity("--test-cmd"); + zig_args.appendAssumeCapacity(arg); + } else { + zig_args.appendAssumeCapacity("--test-cmd-bin"); } } } @@ -783,54 +777,52 @@ fn lowerZigArgs( } } - if (compile.rc_includes != .any) { - try zig_args.appendSlice(gpa, &.{ "-rcincludes", @tagName(compile.rc_includes) }); - } + if (conf_comp.flags3.rc_includes != .any) (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "-rcincludes", @tagName(conf_comp.flags3.rc_includes), + }; - try addFlag(gpa, zig_args, "each-lib-rpath", compile.each_lib_rpath); + try addFlag(gpa, zig_args, "each-lib-rpath", conf_comp.flags2.each_lib_rpath.toBool()); - if (compile.build_id orelse graph.build_id) |build_id| { + if (conf_comp.flags3.build_id.unwrap(conf_comp.build_id.value, conf) orelse graph.build_id) |build_id| { try zig_args.append(gpa, switch (build_id) { .hexstring => |hs| try allocPrint(arena, "--build-id=0x{x}", .{hs.toSlice()}), .none, .fast, .uuid, .sha1, .md5 => try allocPrint(arena, "--build-id={t}", .{build_id}), }); } - const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| - dir.getPath2(step) + const opt_zig_lib_dir: ?[]const u8 = if (conf_comp.zig_lib_dir.value) |dir| + try maker.resolveLazyPathIndexAbs(arena, dir, compile_index) else if (graph.zig_lib_directory.path) |_| try allocPrint(arena, "{f}", .{graph.zig_lib_directory}) else null; - if (opt_zig_lib_dir) |zig_lib_dir| { - try zig_args.append(gpa, "--zig-lib-dir"); - try zig_args.append(gpa, zig_lib_dir); - } + if (opt_zig_lib_dir) |zig_lib_dir| (try zig_args.addManyAsArray(gpa, 2)).* = .{ + "--zig-lib-dir", zig_lib_dir, + }; - try addFlag(gpa, zig_args, "PIE", compile.pie); + try addFlag(gpa, zig_args, "PIE", conf_comp.flags2.pie.toBool()); - if (compile.lto) |lto| { - try zig_args.append(gpa, switch (lto) { - .full => "-flto=full", - .thin => "-flto=thin", - .none => "-fno-lto", - }); + try zig_args.ensureUnusedCapacity(gpa, 1); + switch (conf_comp.flags3.lto) { + .full => zig_args.appendAssumeCapacity("-flto=full"), + .thin => zig_args.appendAssumeCapacity("-flto=thin"), + .none => zig_args.appendAssumeCapacity("-fno-lto"), + .default => {}, } - try addFlag(gpa, zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); + try addFlag(gpa, zig_args, "sanitize-coverage-trace-pc-guard", conf_comp.flags2.sanitize_coverage_trace_pc_guard.toBool()); - if (compile.subsystem) |subsystem| { - try zig_args.appendSlice(gpa, &.{ "--subsystem", @tagName(subsystem) }); + switch (conf_comp.flags3.subsystem) { + .default => {}, + else => |t| (try zig_args.addManyAsArray(gpa, 2)).* = .{ "--subsystem", @tagName(t) }, } - if (compile.mingw_unicode_entry_point) { - try zig_args.append(gpa, "-municode"); - } + try addBool(gpa, zig_args, "-municode", conf_comp.flags.mingw_unicode_entry_point); - if (compile.error_limit orelse graph.error_limit) |err_limit| try zig_args.appendSlice(gpa, &.{ + if (conf_comp.error_limit.value orelse graph.error_limit) |err_limit| (try zig_args.addManyAsArray(gpa, 2)).* = .{ "--error-limit", try allocPrint(arena, "{d}", .{err_limit}), - }); + }; try addFlag(gpa, zig_args, "incremental", graph.incremental); @@ -845,7 +837,10 @@ fn lowerZigArgs( args_length += arg.len + 1; // +1 to account for null terminator } if (args_length >= 30 * 1024) { - try graph.cache_root.handle.createDirPath(io, "args"); + const local_cache_root = graph.local_cache_root; + const args_path: Path = .{ .root_dir = local_cache_root, .sub_path = "args" }; + args_path.root_dir.handle.createDirPath(io, args_path.sub_path) catch |err| + return step.fail(maker, "failed creating directory {f}: {t}", .{ args_path, err }); const args_to_escape = zig_args.items[2..]; var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); @@ -875,51 +870,43 @@ fn lowerZigArgs( var args_hash: [Sha256.digest_length]u8 = undefined; Sha256.hash(args, &args_hash, .{}); var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; - _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); + _ = std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}) catch unreachable; const args_file = "args" ++ Dir.path.sep_str ++ args_hex_hash; - if (graph.cache_root.handle.access(io, args_file, .{})) |_| { - // The args file is already present from a previous run. - } else |err| switch (err) { - error.FileNotFound => { - var af = graph.cache_root.handle.createFileAtomic(io, args_file, .{ - .replace = false, - .make_path = true, - }) catch |e| return step.fail(maker, "failed creating tmp args file {f}{s}: {t}", .{ - graph.cache_root, args_file, e, + local_cache_root.handle.access(io, args_file, .{}) catch { + var af = local_cache_root.handle.createFileAtomic(io, args_file, .{ + .replace = false, + .make_path = true, + }) catch |e| return step.fail(maker, "failed creating tmp args file {f}{s}: {t}", .{ + local_cache_root, args_file, e, + }); + defer af.deinit(io); + + af.file.writeStreamingAll(io, args) catch |e| { + return step.fail(maker, "failed writing args data to tmp file {f}{s}: {t}", .{ + local_cache_root, args_file, e, }); - defer af.deinit(io); - - af.file.writeStreamingAll(io, args) catch |e| { - return step.fail(maker, "failed writing args data to tmp file {f}{s}: {t}", .{ - graph.cache_root, args_file, e, - }); - }; - // Note we can't clean up this file, not even after build - // success, because that might interfere with another build - // process that needs the same file. - af.link(io) catch |e| switch (e) { - error.PathAlreadyExists => { - // The args file was created by another concurrent build process. - }, - else => |other_err| return step.fail(maker, "failed linking tmp file {f}{s}: {t}", .{ - graph.cache_root, args_file, other_err, - }), - }; - }, - else => |other_err| return other_err, - } + }; + // Note we can't clean up this file, not even after build + // success, because that might interfere with another build + // process that needs the same file. + af.link(io) catch |e| switch (e) { + error.PathAlreadyExists => { + // The args file was created by another concurrent build process. + }, + else => |other_err| return step.fail(maker, "failed linking tmp file {f}{s}: {t}", .{ + local_cache_root, args_file, other_err, + }), + }; + }; const resolved_args_file = try mem.concat(arena, u8, &.{ - "@", - try graph.cache_root.join(arena, &.{args_file}), + "@", try local_cache_root.join(arena, &.{args_file}), }); zig_args.shrinkRetainingCapacity(2); try zig_args.append(gpa, resolved_args_file); } - - return try zig_args.toOwnedSlice(); } pub fn rebuildInFuzzMode(compile: *Compile, maker: *Maker, progress_node: std.Progress.Node) !Path { diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index a93c9e9249b8781865d5c30284d5e549f992a233..4d0aae772e4798f69e9f663066c6223becaeb05a 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -728,6 +728,22 @@ pub const Step = extern struct { .hexstring => .hexstring, }; } + + pub fn unwrap(this: @This(), hexstring: ?String, c: *const Configuration) ?std.zig.BuildId { + if (hexstring) |h| { + assert(this == .hexstring); + return .initHexString(h.slice(c)); + } + return switch (this) { + .none => .none, + .fast => .fast, + .uuid => .uuid, + .sha1 => .sha1, + .md5 => .md5, + .hexstring => unreachable, + .default => null, + }; + } }; pub const WasiExecModel = enum(u2) { default, @@ -1425,6 +1441,15 @@ pub const OptionalString = enum(u32) { assert(result != .none); return result; } + + pub fn unwrap(this: @This()) ?String { + if (this == .none) return null; + return @enumFromInt(@intFromEnum(this)); + } + + pub fn slice(this: @This(), c: *const Configuration) ?[:0]const u8 { + return (unwrap(this) orelse return null).slice(c); + } }; /// Points into `string_bytes`, null-terminated. @@ -1740,6 +1765,12 @@ pub const TargetQuery = struct { // TODO comptime assert the enums match return @enumFromInt(@intFromEnum(x orelse return .default)); } + + pub fn unwrap(this: @This()) ?std.Target.Abi { + // TODO comptime assert the enums match + if (this == .default) return null; + return @enumFromInt(@intFromEnum(this)); + } }; pub const CpuArch = enum(u6) { aarch64, @@ -1857,6 +1888,12 @@ pub const TargetQuery = struct { // TODO comptime assert the enums match return @enumFromInt(@intFromEnum(x orelse return .default)); } + + pub fn unwrap(this: @This()) ?std.Target.Os.Tag { + // TODO comptime assert the enums match + if (this == .default) return null; + return @enumFromInt(@intFromEnum(this)); + } }; pub const ObjectFormat = enum(u4) { c, -- 2.54.0 From 6ad6a58e5dcb2d127f980cfb9d3edd7cf1964852 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 13 Mar 2026 14:12:37 -0700 Subject: [PATCH 044/179] Configuration: fix bad serialization of PrefixedList and MultiList Length zero is still serialized because there is no flag bit to hide the length. --- BRANCH_TODO | 2 +- lib/compiler/Maker/ScannedConfig.zig | 4 ++-- lib/compiler/configurer.zig | 1 + lib/std/Build/Configuration.zig | 13 +++++++++---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index dbd9fedf23a26edf2c9b78008e15f8229329dd01..efe67ee3b8d60348b8134955cf38f973d55f0d95 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,4 @@ -* replace union(@This().Tag) +* maxInt(u32) -> max_u32 * replace b.dupe() with string internment * don't forget to add -listen arg back * get zig init template working diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 9c4585957ea808209eb30fca746a3e4ada873733..6115e05d629a17ec81348ae0e15770738e30295a 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -68,8 +68,8 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi Configuration.String => { try s.value(field_value.slice(c), .{}); }, - Configuration.Deps => { - try printValue(sc, s, []Configuration.Step.Index, field_value.slice(c)); + Configuration.Deps.Index => { + try printValue(sc, s, []const Configuration.Step.Index, field_value.get(c).steps.slice); }, Configuration.MaxRss => { try s.value(field_value.toBytes(), .{}); diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index f6b52f057ea1520dd102d9407d78ce670af7f876..18b7cdf1b7d12f9b6439d343790f43494c74f844 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -528,6 +528,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { const dep_steps = try arena.alloc(Configuration.Step.Index, step.dependencies.items.len); for (dep_steps, step.dependencies.items) |*dest, src| dest.* = @enumFromInt(s.step_map.getIndex(src).?); + const deps: Configuration.Deps.Index = @enumFromInt(try wc.addDeduped(@as(Configuration.Deps, .{ .steps = .{ .slice = dep_steps }, }))); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 4d0aae772e4798f69e9f663066c6223becaeb05a..ebc42dec51edce73ea45f3fad381227ab63521b6 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -346,7 +346,7 @@ pub const Wip = struct { try wip.extra.ensureUnusedCapacity(gpa, extra_len); const new_index = addExtraAssumeCapacity(wip, extra); const len: u32 = @intCast(wip.extra.items.len - new_index); - + assert(len != 0); const gop = try wip.dedupe_table.getOrPutContext(gpa, .{ .index = new_index, .len = len, @@ -2417,9 +2417,15 @@ pub const Storage = enum { inline else => |x| setExtraField(buffer, i, @TypeOf(x), x), }, .extended => @compileError("TODO"), - .flag_length_prefixed_list, .length_prefixed_list => { + .flag_length_prefixed_list => { + const len: u32 = @intCast(value.slice.len); + if (len == 0) return 0; // Flag bit hides the length prefix. + buffer[i] = len; + @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); + return len + 1; + }, + .length_prefixed_list => { const len: u32 = @intCast(value.slice.len); - if (len == 0) return 0; buffer[i] = len; @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); return len + 1; @@ -2431,7 +2437,6 @@ pub const Storage = enum { }, .multi_list => { const len: u32 = @intCast(value.mal.len); - if (len == 0) return 0; buffer[i] = len; const fields = @typeInfo(Field.Elem).@"struct".fields; inline for (0..fields.len) |field_i| @memcpy( -- 2.54.0 From c60d33f167494afea808dc0ac9c476422efe22c0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 13 Mar 2026 14:14:01 -0700 Subject: [PATCH 045/179] Configuration: refactor maxInt(u32) --- BRANCH_TODO | 1 - lib/std/Build/Configuration.zig | 28 ++++++++++++++-------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index efe67ee3b8d60348b8134955cf38f973d55f0d95..6dd57427caf43f9774f7495957b13f4a7919869f 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ -* maxInt(u32) -> max_u32 * replace b.dupe() with string internment * don't forget to add -listen arg back * get zig init template working diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index ebc42dec51edce73ea45f3fad381227ab63521b6..6f61eb31ef188c08fa8a0460b837a11e8566c7af 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -4,7 +4,7 @@ const std = @import("../std.zig"); const Io = std.Io; const Allocator = std.mem.Allocator; const assert = std.debug.assert; -const maxInt = std.math.maxInt; +const max_u32 = std.math.maxInt(u32); string_bytes: []u8, steps: []Step, @@ -1094,7 +1094,7 @@ pub const LazyPath = union(@This().Tag) { /// An index into `extra`, or `null`. pub const OptionalIndex = enum(u32) { - none = maxInt(u32), + none = max_u32, _, pub fn unwrap(this: @This()) ?Index { @@ -1149,7 +1149,7 @@ pub const GeneratedFileIndex = enum(u32) { }; pub const OptionalGeneratedFileIndex = enum(u32) { - none = maxInt(u32), + none = max_u32, _, pub fn init(i: ?GeneratedFileIndex) OptionalGeneratedFileIndex { @@ -1169,7 +1169,7 @@ pub const Package = struct { hash: String, pub const Index = enum(u32) { - root = maxInt(u32), + root = max_u32, _, /// Returns `null` for root package. @@ -1352,7 +1352,7 @@ pub const ImportTable = struct { /// Points into `extra`. pub const Index = enum(u32) { - invalid = maxInt(u32), + invalid = max_u32, _, pub fn get(this: @This(), c: *const Configuration) ImportTable { @@ -1385,7 +1385,7 @@ pub const Deps = struct { /// /// Stored identically to `Deps`. pub const OptionalStringList = enum(u32) { - none = maxInt(u32), + none = max_u32, _, pub fn slice(osl: OptionalStringList, c: *const Configuration) ?[]const String { @@ -1414,11 +1414,11 @@ pub const Path = extern struct { }; pub const InstallDestDir = enum(u32) { - none = maxInt(u32) - 4, - prefix = maxInt(u32) - 3, - lib = maxInt(u32) - 2, - bin = maxInt(u32) - 1, - header = maxInt(u32), + none = max_u32 - 4, + prefix = max_u32 - 3, + lib = max_u32 - 2, + bin = max_u32 - 1, + header = max_u32, /// A `String` path relative to the prefix. _, @@ -1433,7 +1433,7 @@ pub const OptionalString = enum(u32) { empty = 0, /// The string "root". root = 1, - none = maxInt(u32), + none = max_u32, _, pub fn init(s: String) OptionalString { @@ -1633,7 +1633,7 @@ pub const ResolvedTarget = struct { }; pub const OptionalIndex = enum(u32) { - none = maxInt(u32), + none = max_u32, _, pub fn unwrap(this: @This()) ?Index { @@ -1678,7 +1678,7 @@ pub const TargetQuery = struct { }; pub const OptionalIndex = enum(u32) { - none = maxInt(u32), + none = max_u32, _, pub fn init(i: Index) OptionalIndex { -- 2.54.0 From b998d71e939c304ba76860077b4483249f670b7d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 13 Mar 2026 16:20:54 -0700 Subject: [PATCH 046/179] maker: finish lowering compile step CLI args --- lib/compiler/Maker/Step/Compile.zig | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index c1d903668b48df83db0afd6fa8179ac0e9ff529c..35f13d3df3a77098e4a0cf7a4b065d9acebcb7b3 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -481,18 +481,21 @@ fn lowerZigArgs( const module_index = cli_named_modules.modules.keys()[module_cli_index]; try appendModuleFlags(module_index, zig_args, compile_index, maker); - if (true) @panic("TODO"); + const imports = mod.import_table.get(conf).imports.mal; // --dep arguments - try zig_args.ensureUnusedCapacity(gpa, mod.import_table.count() * 2); - for (mod.import_table.keys(), mod.import_table.values()) |name, import| { + try zig_args.ensureUnusedCapacity(gpa, imports.len * 2); + for (imports.items(.name), imports.items(.module)) |name, import| { const import_index = cli_named_modules.modules.getIndex(import).?; const import_cli_name = cli_named_modules.names.keys()[import_index]; zig_args.appendAssumeCapacity("--dep"); - if (std.mem.eql(u8, import_cli_name, name)) { + const name_slice = name.slice(conf); + if (std.mem.eql(u8, import_cli_name, name_slice)) { zig_args.appendAssumeCapacity(import_cli_name); } else { - zig_args.appendAssumeCapacity(try allocPrint(arena, "{s}={s}", .{ name, import_cli_name })); + zig_args.appendAssumeCapacity(try allocPrint(arena, "{s}={s}", .{ + name_slice, import_cli_name, + })); } } @@ -503,11 +506,12 @@ fn lowerZigArgs( // perhaps a set of linker objects, or C source files instead. // Linker objects are added to the CLI globally, while C source // files must have a module parent. - if (mod.root_source_file) |lp| { - const src = lp.getPath2(mod.owner, step); - try zig_args.append(gpa, try allocPrint(arena, "-M{s}={s}", .{ module_cli_name, src })); - } else if (moduleNeedsCliArg(mod)) { - try zig_args.append(gpa, try allocPrint(arena, "-M{s}", .{module_cli_name})); + try zig_args.ensureUnusedCapacity(gpa, 1); + if (mod.root_source_file.unwrap()) |lp| { + const src = try maker.resolveLazyPathIndexAbs(arena, lp, compile_index); + zig_args.appendAssumeCapacity(try allocPrint(arena, "-M{s}={s}", .{ module_cli_name, src })); + } else if (moduleNeedsCliArg(&mod, conf)) { + zig_args.appendAssumeCapacity(try allocPrint(arena, "-M{s}", .{module_cli_name})); } } } @@ -1262,8 +1266,8 @@ fn matchCompileError(actual: []const u8, expected: []const u8) bool { return false; } -fn moduleNeedsCliArg(mod: *const Module) bool { - return for (mod.link_objects.items) |o| switch (o) { +fn moduleNeedsCliArg(mod: *const Configuration.Module, conf: *const Configuration) bool { + return for (0..mod.link_objects.len) |i| switch (mod.link_objects.tag(conf.extra, i)) { .c_source_file, .c_source_files, .assembly_file, .win32_resource_file => break true, else => continue, } else false; -- 2.54.0 From dd51fc30f884aa1c3305793010a30d826689be89 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 13 Mar 2026 19:50:48 -0700 Subject: [PATCH 047/179] maker: finish migrating compile step make logic --- lib/compiler/Maker.zig | 42 ++++ lib/compiler/Maker/Graph.zig | 18 ++ lib/compiler/Maker/Step.zig | 288 +++++++++++++--------------- lib/compiler/Maker/Step/Compile.zig | 87 ++++++--- lib/compiler/Maker/WebServer.zig | 12 +- lib/std/Build/Configuration.zig | 23 ++- lib/std/Build/Step/Compile.zig | 5 +- lib/std/zig.zig | 36 ++-- 8 files changed, 308 insertions(+), 203 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 24818d4815e429c9f04baccdb44e5c2bba6c8218..36c6dd3dc3edf8fbe70d09a3486867d67b63c831 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1084,6 +1084,7 @@ fn makeStep( } else |err| switch (err) { error.MakeFailed => .failure, error.MakeSkipped => .skipped, + error.Canceled => |e| return e, }; @atomicStore(Step.State, &make_step.state, new_state, .monotonic); @@ -1764,6 +1765,10 @@ pub fn resolveLazyPathIndexAbs( return resolveLazyPathAbs(maker, arena, lazy_path_index.get(c), asking_step_index); } +pub fn generatedPath(maker: *Maker, index: Configuration.GeneratedFileIndex) *Path { + return &maker.generated_files[@intFromEnum(index)]; +} + fn packagePath( maker: *const Maker, arena: Allocator, @@ -1783,3 +1788,40 @@ fn packagePath( .sub_path = try Io.Dir.path.join(arena, &.{ pkg_root.sub_path, hash, sub_path }), }; } + +/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. +pub fn installFile( + maker: *Maker, + arena: Allocator, + src_lazy_path: Configuration.LazyPath, + dest_path: []const u8, + asking_step_index: Configuration.Step.Index, +) !Io.Dir.PrevStatus { + const graph = maker.graph; + const io = graph.io; + const src_path = try resolveLazyPath(maker, arena, src_lazy_path, asking_step_index); + { + const src_path_rendered = try src_path.toString(arena); + defer arena.free(src_path_rendered); + try graph.handleVerbose(.inherit, null, &.{ "install", "-C", src_path_rendered, dest_path }); + } + return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| { + const s = stepByIndex(maker, asking_step_index); + return s.fail(maker, "unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); + }; +} + +/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. +pub fn installDir( + maker: *Maker, + dest_path: []const u8, + asking_step_index: Configuration.Step.Index, +) !Io.Dir.CreatePathStatus { + const graph = maker.graph; + const io = graph.io; + try graph.handleVerbose(.inherit, null, &.{ "install", "-d", dest_path }); + return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| { + const s = stepByIndex(maker, asking_step_index); + return s.fail(maker, "unable to create dir '{s}': {t}", .{ dest_path, err }); + }; +} diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index c2ed939b54af0c89e3da1cebe955797a88be8d2d..e3b49b4b98733d8f4d67d51203578d17a877feff 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -47,3 +47,21 @@ sysroot: ?[]const u8 = null, search_prefixes: std.ArrayList([]const u8) = .empty, build_id: ?std.zig.BuildId = null, error_limit: ?u32 = null, + +/// Intention of verbose is to print all sub-process command lines to stderr +/// before spawning them. +pub fn handleVerbose( + graph: *const Graph, + cwd: std.process.Child.Cwd, + opt_env: ?*const std.process.Environ.Map, + argv: []const []const u8, +) error{OutOfMemory}!void { + if (!graph.verbose) return; + const arena = graph.arena; + const text = try std.zig.allocPrintCmd(arena, cwd, if (opt_env) |env| .{ + .child = env, + .parent = &graph.environ_map, + } else null, argv); + defer arena.free(text); + std.log.scoped(.verbose).info("{s}", .{text}); +} diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index c3c064aead143b5a4dd2a3667847929bacecfa3d..e238c2cc8e52327718579feabf9b0974f0b47db2 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -111,10 +111,9 @@ pub const Extended = union(enum) { progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { _ = todo; - _ = step_index; _ = maker; _ = progress_node; - @panic("TODO implement another step type"); + std.debug.panic("TODO implement another step type (index {d})", .{step_index}); } }; }; @@ -146,7 +145,7 @@ pub const Inputs = struct { .table = .{}, }; - pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, Files, Cache.Path.TableAdapter, false); + pub const Table = std.ArrayHashMapUnmanaged(Path, Files, Path.TableAdapter, false); /// The special file name "." means any changes inside the directory. pub const Files = std.ArrayList([]const u8); @@ -205,7 +204,7 @@ pub const MakeError = error{ /// Indicates the error is already reported. MakeFailed, MakeSkipped, -}; +} || Io.Cancelable; pub const ExtendedMakeError = MakeError || Allocator.Error; @@ -215,7 +214,7 @@ pub fn make( progress_node: std.Progress.Node, ) MakeError!void { const graph = maker.graph; - const process_arena = graph.arena; // TODO don't leak into the process arena + const arena = graph.arena; // TODO don't leak into the process arena const io = graph.io; const c = &maker.scanned_config.configuration; const conf_step = step_index.ptr(c); @@ -248,6 +247,7 @@ pub fn make( s.result_oom = true; return error.MakeFailed; }, + error.Canceled => |e| return e, }; if (!s.test_results.isSuccess()) { @@ -257,11 +257,11 @@ pub fn make( const max_rss = conf_step.max_rss.toBytes(); if (max_rss != 0 and s.result_peak_rss > max_rss) { if (std.fmt.allocPrint( - process_arena, + arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ s.result_peak_rss, max_rss }, )) |msg| { - s.oomWrap(s.result_error_msgs.append(process_arena, msg)); + s.oomWrap(s.result_error_msgs.append(arena, msg)); } else |_| s.result_oom = true; } } @@ -288,11 +288,12 @@ pub fn reset(step: *Step, gpa: Allocator) void { /// Populates `s.result_failed_command`. pub fn captureChildProcess( s: *Step, - gpa: Allocator, + maker: *Maker, progress_node: std.Progress.Node, argv: []const []const u8, ) !std.process.RunResult { - const graph = s.owner.graph; + const gpa = maker.gpa; + const graph = maker.graph; const arena = graph.arena; const io = graph.io; @@ -300,14 +301,14 @@ pub fn captureChildProcess( assert(s.result_failed_command == null); s.result_failed_command = try std.zig.allocPrintCmd(gpa, .inherit, null, argv); - try handleChildProcUnsupported(s); - try handleVerbose(s, .inherit, argv); + try handleChildProcUnsupported(s, maker); + try graph.handleVerbose(.inherit, null, argv); const result = std.process.run(arena, io, .{ .argv = argv, .environ_map = &graph.environ_map, .progress_node = progress_node, - }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); + }) catch |err| return s.fail(maker, "failed to run {s}: {t}", .{ argv[0], err }); if (result.stderr.len > 0) { try s.result_error_msgs.append(arena, result.stderr); @@ -316,7 +317,9 @@ pub fn captureChildProcess( return result; } -pub fn fail(step: *Step, maker: *const Maker, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { +pub const FailError = error{ OutOfMemory, MakeFailed }; + +pub fn fail(step: *Step, maker: *const Maker, comptime fmt: []const u8, args: anytype) FailError { try step.addError(maker, fmt, args); return error.MakeFailed; } @@ -351,15 +354,16 @@ pub const ZigProcess = struct { /// is the zig compiler - the same version that compiled the build runner. /// Populates `s.result_failed_command`. pub fn evalZigProcess( - s: *Step, + step_index: Configuration.Step.Index, + maker: *Maker, argv: []const []const u8, prog_node: std.Progress.Node, watch: bool, - maker: *Maker, -) !?Cache.Path { +) (Step.ExtendedMakeError || error{NeedCompileErrorCheck})!?Path { + const s = maker.stepByIndex(step_index); const gpa = maker.gpa; - const b = s.owner; - const io = b.graph.io; + const graph = maker.graph; + const io = graph.io; // If an error occurs, it's happened in this command: assert(s.result_failed_command == null); @@ -371,36 +375,33 @@ pub fn evalZigProcess( zp.progress_ipc_index = null; var exited = false; defer if (exited) { - s.cast(Compile).?.zig_process = null; + s.extended.compile.zig_process = null; zp.deinit(io); gpa.destroy(zp); } else zp.saveState(prog_node); - const result = zigProcessUpdate(s, zp, watch, maker) catch |err| switch (err) { + const result = zigProcessUpdate(step_index, maker, zp, watch) catch |err| switch (err) { error.BrokenPipe, error.EndOfStream => |reason| { - std.log.info("{s} restart required: {t}", .{ argv[0], reason }); // Process restart required. - const term = zp.child.wait(io) catch |e| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); - }; - _ = term; + std.log.info("{s} restart required: {t}", .{ argv[0], reason }); + _ = zp.child.wait(io) catch |e| return s.fail(maker, "unable to wait for {s}: {t}", .{ argv[0], e }); exited = true; break :update; }, - else => |e| return e, + error.OutOfMemory, error.Canceled, error.MakeFailed => |e| return e, + else => |e| return s.fail(maker, "zig child process monitoring failed: {t}", .{e}), }; - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); - } + if (s.result_error_bundle.errorMessageCount() > 0) + return s.fail(maker, "{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); if (s.result_error_msgs.items.len > 0 and result == null) { // Crash detected. const term = zp.child.wait(io) catch |e| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); + return s.fail(maker, "unable to wait for {s}: {t}", .{ argv[0], e }); }; s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; exited = true; - try handleChildProcessTerm(s, term); + try handleChildProcessTerm(s, maker, term); return error.MakeFailed; } @@ -408,31 +409,34 @@ pub fn evalZigProcess( } assert(argv.len != 0); - try handleChildProcUnsupported(s); - try handleVerbose(s, .inherit, argv); + try handleChildProcUnsupported(s, maker); + try graph.handleVerbose(.inherit, null, argv); const zp = try gpa.create(ZigProcess); defer if (!watch) gpa.destroy(zp); zp.child = std.process.spawn(io, .{ .argv = argv, - .environ_map = &b.graph.environ_map, + .environ_map = &graph.environ_map, .stdin = .pipe, .stdout = .pipe, .stderr = .pipe, .request_resource_usage_statistics = true, .progress_node = prog_node, - }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); + }) catch |err| return s.fail(maker, "failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ zp.child.stdout.?, zp.child.stderr.?, }); - if (watch) s.cast(Compile).?.zig_process = zp; + if (watch) s.extended.compile.zig_process = zp; defer if (!watch) zp.deinit(io); const result = result: { defer if (watch) zp.saveState(prog_node); - break :result try zigProcessUpdate(s, zp, watch, maker); + break :result zigProcessUpdate(step_index, maker, zp, watch) catch |err| switch (err) { + error.OutOfMemory, error.Canceled, error.MakeFailed => |e| return e, + else => |e| return s.fail(maker, "zig child process monitoring failed: {t}", .{e}), + }; }; if (!watch) { @@ -441,56 +445,36 @@ pub fn evalZigProcess( zp.child.stdin = null; const term = zp.child.wait(io) catch |err| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); + return s.fail(maker, "unable to wait for {s}: {t}", .{ argv[0], err }); }; s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; - // Special handling for Compile step that is expecting compile errors. - if (s.cast(Compile)) |compile| switch (term) { - .exited => { + // Special handling for compile step that is expecting compile errors. + const conf = &maker.scanned_config.configuration; + if (term == .exited) switch (step_index.ptr(conf).extended.get(conf.extra)) { + .compile => |compile| if (compile.flags4.expect_errors != .none) { // Note that the exit code may be 0 in this case due to the // compiler server protocol. - if (compile.expect_errors != null) { - return error.NeedCompileErrorCheck; - } + return error.NeedCompileErrorCheck; }, else => {}, }; - - try handleChildProcessTerm(s, term); + try handleChildProcessTerm(s, maker, term); } if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + return s.fail(maker, "{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); } return result; } -/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. -pub fn installFile(s: *Step, src_lazy_path: LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { - const b = s.owner; - const io = b.graph.io; - const src_path = src_lazy_path.getPath3(b, s); - try handleVerbose(s, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); - return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| - return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); -} - -/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. -pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { - const b = s.owner; - const io = b.graph.io; - try handleVerbose(s, .inherit, &.{ "install", "-d", dest_path }); - return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| - return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); -} - -fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, maker: *Maker) !?Path { +fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *ZigProcess, watch: bool) !?Path { + const s = maker.stepByIndex(step_index); const gpa = maker.gpa; - const b = s.owner; - const arena = b.allocator; - const io = b.graph.io; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + const io = graph.io; const start_ts = Io.Clock.awake.now(io); @@ -522,6 +506,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, maker: *Maker) !?Pat .zig_version => { if (!std.mem.eql(u8, builtin.zig_version_string, body)) { return s.fail( + maker, "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", .{ builtin.zig_version_string, body }, ); @@ -538,61 +523,65 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, maker: *Maker) !?Pat s.result_cached = emit_digest.flags.cache_hit; const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; result = .{ - .root_dir = b.cache_root, - .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), + .root_dir = graph.local_cache_root, + .sub_path = try arena.dupe(u8, "o" ++ Io.Dir.path.sep_str ++ Cache.binToHex(digest.*)), }; }, .file_system_inputs => { - s.clearWatchInputs(); + clearWatchInputs(s, maker); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); var it = std.mem.splitScalar(u8, body, 0); while (it.next()) |prefixed_path| { const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); const sub_path = try arena.dupe(u8, prefixed_path[1..]); - const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; + const sub_path_dirname = Io.Dir.path.dirname(sub_path) orelse ""; switch (prefix_index) { .cwd => { - const path: Cache.Path = .{ - .root_dir = Cache.Directory.cwd(), + const path: Path = .{ + .root_dir = .cwd(), .sub_path = sub_path_dirname, }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + try addWatchInputFromPath(s, maker, path, Io.Dir.path.basename(sub_path)); }, .zig_lib => zl: { - if (s.cast(Step.Compile)) |compile| { - if (compile.zig_lib_dir) |zig_lib_dir| { - const lp = try zig_lib_dir.join(arena, sub_path); - try addWatchInput(s, lp); + switch (conf_step.extended.get(conf.extra)) { + .compile => |compile| if (compile.zig_lib_dir.value) |zig_lib_dir| { + const resolved = try maker.resolveLazyPathIndex(arena, zig_lib_dir, step_index); + const appended = try resolved.join(arena, sub_path); + try addWatchInputPath(s, maker, appended); break :zl; - } + }, + else => {}, } - const path: Cache.Path = .{ - .root_dir = s.owner.graph.zig_lib_directory, + const path: Path = .{ + .root_dir = graph.zig_lib_directory, .sub_path = sub_path_dirname, }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + try addWatchInputFromPath(s, maker, path, Io.Dir.path.basename(sub_path)); }, .local_cache => { - const path: Cache.Path = .{ - .root_dir = b.cache_root, + const path: Path = .{ + .root_dir = graph.local_cache_root, .sub_path = sub_path_dirname, }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + try addWatchInputFromPath(s, maker, path, Io.Dir.path.basename(sub_path)); }, .global_cache => { - const path: Cache.Path = .{ - .root_dir = s.owner.graph.global_cache_root, + const path: Path = .{ + .root_dir = graph.global_cache_root, .sub_path = sub_path_dirname, }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + try addWatchInputFromPath(s, maker, path, Io.Dir.path.basename(sub_path)); }, } } }, - .time_report => if (maker.web_server) |ws| { + .time_report => if (maker.web_server) |*ws| { const TimeReport = std.zig.Server.Message.TimeReport; const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); ws.updateTimeReportCompile(.{ - .compile = s.cast(Step.Compile).?, + .compile_step = step_index, .use_llvm = tr.flags.use_llvm, .stats = tr.stats, .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), @@ -636,46 +625,29 @@ fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { }; } -pub fn handleVerbose( - s: *Step, - arena: Allocator, - cwd: std.process.Child.Cwd, - opt_env: ?*const std.process.Environ.Map, - argv: []const []const u8, -) error{OutOfMemory}!void { - const graph = s.graph; - if (!graph.verbose) return; - // Intention of verbose is to print all sub-process command lines to - // stderr before spawning them. - const text = try std.zig.allocPrintCmd(arena, cwd, if (opt_env) |env| .{ - .child = env, - .parent = &graph.environ_map, - } else null, argv); - std.log.scoped(.verbose).info("{s}", .{text}); -} - /// Asserts that the caller has already populated `s.result_failed_command`. -pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { +pub inline fn handleChildProcUnsupported(s: *Step, maker: *Maker) FailError!void { + assert(s.result_failed_command != null); if (!std.process.can_spawn) { - return s.fail("unable to spawn process: host cannot spawn child processes", .{}); + return s.fail(maker, "unable to spawn process: host cannot spawn child processes", .{}); } } /// Asserts that the caller has already populated `s.result_failed_command`. -pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { +pub fn handleChildProcessTerm(s: *Step, maker: *Maker, term: std.process.Child.Term) FailError!void { assert(s.result_failed_command != null); return switch (term) { - .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), - .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), - .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), - .unknown => s.fail("process terminated unexpectedly", .{}), + .exited => |code| if (code != 0) s.fail(maker, "process exited with error code {d}", .{code}), + .signal => |sig| s.fail(maker, "process terminated with signal {t}", .{sig}), + .stopped => |sig| s.fail(maker, "process stopped with signal {t}", .{sig}), + .unknown => s.fail(maker, "process terminated unexpectedly", .{}), }; } /// Prefer `cacheHitAndWatch` unless you already added watch inputs /// separately from using the cache system. -pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool { - s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); +pub fn cacheHit(s: *Step, maker: *Maker, man: *Cache.Manifest) !bool { + s.result_cached = man.hit() catch |err| return failWithCacheError(s, maker, man, err); return s.result_cached; } @@ -683,36 +655,37 @@ pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool { /// the full set of files picked up by the cache manifest. /// /// Must be accompanied with `writeManifestAndWatch`. -pub fn cacheHitAndWatch(s: *Step, man: *Cache.Manifest) !bool { - const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); +pub fn cacheHitAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !bool { + const is_hit = man.hit() catch |err| return failWithCacheError(s, maker, man, err); s.result_cached = is_hit; // The above call to hit() populates the manifest with files, so in case of // a hit, we need to populate watch inputs. - if (is_hit) try setWatchInputsFromManifest(s, man); + if (is_hit) try setWatchInputsFromManifest(s, maker, man); return is_hit; } fn failWithCacheError( s: *Step, + maker: *Maker, man: *const Cache.Manifest, err: Cache.Manifest.HitError, ) error{ OutOfMemory, Canceled, MakeFailed } { switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ + .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail(maker, "failed to check cache: {t} {t}", .{ man.diagnostic, e, }), .file_open, .file_stat, .file_read, .file_hash => |op| { const pp = man.files.keys()[op.file_index].prefixed_path; const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ - prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, + return s.fail(maker, "failed to check cache: '{s}{c}{s}' {t} {t}", .{ + prefix, Io.Dir.path.sep, pp.sub_path, man.diagnostic, op.err, }); }, }, error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), + error.InvalidFormat => return s.fail(maker, "failed to check cache: invalid manifest file format", .{}), } } @@ -730,48 +703,48 @@ pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { /// the full set of files picked up by the cache manifest. /// /// Must be accompanied with `cacheHitAndWatch`. -pub fn writeManifestAndWatch(s: *Step, man: *Cache.Manifest) !void { +pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { try writeManifest(s, man); - try setWatchInputsFromManifest(s, man); + try setWatchInputsFromManifest(s, maker, man); } -fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void { - const arena = s.owner.allocator; +fn setWatchInputsFromManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into process arena const prefixes = man.cache.prefixes(); - clearWatchInputs(s); + clearWatchInputs(s, maker); for (man.files.keys()) |file| { // The file path data is freed when the cache manifest is cleaned up at the end of `make`. const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); - try addWatchInputFromPath(s, .{ + try addWatchInputFromPath(s, maker, .{ .root_dir = prefixes[file.prefixed_path.prefix], - .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, std.fs.path.basename(sub_path)); + .sub_path = Io.Dir.path.dirname(sub_path) orelse "", + }, Io.Dir.path.basename(sub_path)); } } /// For steps that have a single input that never changes when re-running `make`. -pub fn singleUnchangingWatchInput(step: *Step, lazy_path: LazyPath) Allocator.Error!void { - if (!step.inputs.populated()) try step.addWatchInput(lazy_path); +pub fn singleUnchangingWatchInput(step: *Step, maker: *Maker, lazy_path: LazyPath) Allocator.Error!void { + if (!step.inputs.populated()) try step.addWatchInput(maker, lazy_path); } -pub fn clearWatchInputs(step: *Step) void { - const gpa = step.owner.allocator; - step.inputs.clear(gpa); +pub fn clearWatchInputs(step: *Step, maker: *Maker) void { + step.inputs.clear(maker.gpa); } /// Places a *file* dependency on the path. -pub fn addWatchInput(step: *Step, lazy_file: LazyPath) Allocator.Error!void { +pub fn addWatchInput(step: *Step, maker: *Maker, lazy_file: LazyPath) Allocator.Error!void { switch (lazy_file) { .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), .cwd_relative => |path_string| { - try addWatchInputFromPath(step, .{ + try addWatchInputFromPath(step, maker, .{ .root_dir = .{ .path = null, .handle = Io.Dir.cwd(), }, - .sub_path = std.fs.path.dirname(path_string) orelse "", - }, std.fs.path.basename(path_string)); + .sub_path = Io.Dir.path.dirname(path_string) orelse "", + }, Io.Dir.path.basename(path_string)); }, // Nothing to watch because this dependency edge is modeled instead via `dependants`. .generated => {}, @@ -780,7 +753,7 @@ pub fn addWatchInput(step: *Step, lazy_file: LazyPath) Allocator.Error!void { /// Any changes inside the directory will trigger invalidation. /// -/// See also `addDirectoryWatchInputFromPath` which takes a `Cache.Path` instead. +/// See also `addDirectoryWatchInputFromPath` which takes a `Path` instead. /// /// Paths derived from this directory should also be manually added via /// `addDirectoryWatchInputFromPath` if and only if this function returns @@ -812,15 +785,15 @@ pub fn addDirectoryWatchInput(step: *Step, lazy_directory: LazyPath) Allocator.E /// dependency on `path` is not already accounted for by a `Step` dependency. /// In other words, before calling this function, first check that the /// `LazyPath` which this `path` is derived from is not `generated`. -pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void { - return addWatchInputFromPath(step, path, "."); +pub fn addDirectoryWatchInputFromPath(step: *Step, maker: *Maker, path: Path) !void { + return addWatchInputFromPath(step, maker, path, "."); } -fn addWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { - return addWatchInputFromPath(step, .{ +fn addWatchInputFromBuilder(step: *Step, maker: *Maker, package: Package, sub_path: []const u8) !void { + return addWatchInputFromPath(step, maker, .{ .root_dir = package.build_root, - .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, std.fs.path.basename(sub_path)); + .sub_path = Io.Dir.path.dirname(sub_path) orelse "", + }, Io.Dir.path.basename(sub_path)); } fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { @@ -830,9 +803,16 @@ fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: [] }); } -fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !void { - const gpa = step.owner.allocator; - const gop = try step.inputs.table.getOrPut(gpa, path); +fn addWatchInputPath(step: *Step, maker: *Maker, path: Path) Allocator.Error!void { + return addWatchInputFromPath(step, maker, .{ + .root_dir = path.root_dir, + .sub_path = Io.Dir.path.dirname(path.sub_path) orelse "", + }, Io.Dir.path.basename(path.sub_path)); +} + +fn addWatchInputFromPath(step: *Step, maker: *Maker, directory: Path, basename: []const u8) Allocator.Error!void { + const gpa = maker.gpa; + const gop = try step.inputs.table.getOrPut(gpa, directory); if (!gop.found_existing) gop.value_ptr.* = .empty; try gop.value_ptr.append(gpa, basename); } diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 35f13d3df3a77098e4a0cf7a4b065d9acebcb7b3..d781791e806bf09eb835a3ee76812cacc0969202 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -29,61 +29,91 @@ pub fn make( ) Step.ExtendedMakeError!void { const graph = maker.graph; const step = maker.stepByIndex(compile_index); + const conf = &maker.scanned_config.configuration; + const conf_step = compile_index.ptr(conf); + const conf_comp = conf_step.extended.get(conf.extra).compile; // Reset / repopulate persistent state. compile.zig_args.clearRetainingCapacity(); try lowerZigArgs(compile, compile_index, maker, &compile.zig_args, false); - if (true) @panic("TODO implement compile.make()"); - const process_arena = graph.arena; // TODO don't leak into the process_arena - const maybe_output_dir = step.evalZigProcess( + const maybe_output_dir = Step.evalZigProcess( + compile_index, + maker, compile.zig_args.items, progress_node, (graph.incremental == true) and (maker.watch or maker.web_server != null), - maker, ) catch |err| switch (err) { error.NeedCompileErrorCheck => { - assert(compile.expect_errors != null); try checkCompileErrors(compile, maker); return; }, else => |e| return e, }; + const root_module = conf_comp.root_module.get(conf); + const target = root_module.resolved_target.get(conf).?.result.get(conf); + // Update generated files if (maybe_output_dir) |output_dir| { - if (compile.emit_directory) |lp| { - lp.path = try allocPrint(process_arena, "{f}", .{output_dir}); - } - - // zig fmt: off - if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); - if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); - // hack for stage2_x86_64 + coff - if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); - if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); - if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); - if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); - if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); - if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); - if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); - // zig fmt: on + if (conf_comp.emit_directory.value) |gf| maker.generatedPath(gf).* = output_dir; + try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_bin.value, .bin); + try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_pdb.value, .pdb); + try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_implib.value, .implib); + try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_h.value, .h); + try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_docs.value, .docs); + try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_asm.value, .@"asm"); + try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_llvm_ir.value, .llvm_ir); + try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_llvm_bc.value, .llvm_bc); } - if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and - compile.version != null and compile.generated_bin != null and - std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) + if (conf_comp.flags3.kind == .lib and conf_comp.flags2.linkage == .dynamic and + conf_comp.version.value != null and conf_comp.generated_bin.value != null and + target.flags.os_tag != .windows) { + if (true) @panic("TODO"); try doAtomicSymLinks( step, - compile.getEmittedBin().getPath2(step), - compile.major_only_filename.?, - compile.name_only_filename.?, + conf_comp.getEmittedBin().getPath2(step), + conf_comp.major_only_filename.?, + conf_comp.name_only_filename.?, ); } } +fn updateGeneratedFile( + conf_comp: *const Configuration.Step.Compile, + maker: *Maker, + out_path: std.Build.Cache.Path, + target: *const Configuration.TargetQuery, + opt_gf: ?Configuration.GeneratedFileIndex, + ea: std.zig.EmitArtifact, +) Allocator.Error!void { + const gf = opt_gf orelse return; + const graph = maker.graph; + const conf = &maker.scanned_config.configuration; + const arena = graph.arena; // TODO don't leak into process arena + const name = try ea.cacheName(arena, .{ + .root_name = conf_comp.root_name.slice(conf), + .cpu_arch = target.flags.cpu_arch.unwrap().?, + .os_tag = target.flags.os_tag.unwrap().?, + .ofmt = target.flags.object_format.unwrap().?, + .abi = target.flags.abi.unwrap().?, + .output_mode = switch (conf_comp.flags3.kind) { + .lib => .Lib, + .obj, .test_obj => .Obj, + .exe, .@"test" => .Exe, + }, + .link_mode = conf_comp.flags2.linkage.unwrap(), + .version = if (conf_comp.version.value) |v| + std.SemanticVersion.parse(v.slice(conf)) catch unreachable + else + null, + }); + maker.generatedPath(gf).* = try out_path.join(arena, name); +} + /// List of importable modules in a compilation's module graph, including /// the root module. The root module is guaranteed to be first. const ModuleList = std.AutoArrayHashMapUnmanaged(Configuration.Module.Index, Configuration.String); @@ -154,7 +184,7 @@ fn lowerZigArgs( const root_module = conf_comp.root_module.get(conf); if (root_module.resolved_target.get(conf).?.query.unwrap()) |query| { - if (query.get(conf).flags.object_format.get()) |ofmt| { + if (query.get(conf).flags.object_format.unwrap()) |ofmt| { try zig_args.append(gpa, try allocPrint(arena, "-ofmt={t}", .{ofmt})); } } @@ -1146,6 +1176,7 @@ fn runPkgConfig(compile: *const Compile, maker: *const Maker, lib_name: []const } fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { + if (true) @panic("TODO"); // Clear this field so that it does not get printed by the build runner. const actual_eb = compile.step.result_error_bundle; compile.step.result_error_bundle = .empty; diff --git a/lib/compiler/Maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig index 1ea44e0ba0a47a2f2c607ca7f8d4c42bdba08c99..9ad6580868017b134d71bfef58d034e287f495ab 100644 --- a/lib/compiler/Maker/WebServer.zig +++ b/lib/compiler/Maker/WebServer.zig @@ -757,12 +757,16 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim }); return error.WasmCompilationFailed; }; + const target = std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ + .arch_os_abi = arch_os_abi, + .cpu_features = cpu_features, + }) catch unreachable) catch unreachable; const bin_name = try std.zig.binNameAlloc(arena, .{ .root_name = root_name, - .target = &(std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ - .arch_os_abi = arch_os_abi, - .cpu_features = cpu_features, - }) catch unreachable) catch unreachable), + .cpu_arch = target.cpu.arch, + .os_tag = target.os.tag, + .ofmt = target.ofmt, + .abi = target.abi, .output_mode = .Exe, }); return base_path.join(arena, bin_name); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 6f61eb31ef188c08fa8a0460b837a11e8566c7af..96fdb39c7157ef640dd9014774143ee12bb27e28 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -768,6 +768,14 @@ pub const Step = extern struct { .dynamic => .dynamic, }; } + + pub fn unwrap(this: @This()) ?std.builtin.LinkMode { + return switch (this) { + .static => .static, + .dynamic => .dynamic, + .default => null, + }; + } }; pub const Kind = enum(u3) { exe, @@ -1838,6 +1846,12 @@ pub const TargetQuery = struct { // TODO comptime assert the enums match return @enumFromInt(@intFromEnum(x orelse return .default)); } + + pub fn unwrap(this: @This()) ?std.Target.Cpu.Arch { + // TODO comptime assert the enums match + if (this == .default) return null; + return @enumFromInt(@intFromEnum(this)); + } }; pub const OsTag = enum(u6) { freestanding, @@ -1913,7 +1927,7 @@ pub const TargetQuery = struct { return @enumFromInt(@intFromEnum(x orelse return .default)); } - pub fn get(this: @This()) ?std.Target.ObjectFormat { + pub fn unwrap(this: @This()) ?std.Target.ObjectFormat { return switch (this) { .c => .c, .coff => .coff, @@ -2018,11 +2032,16 @@ pub const Storage = enum { pub const storage: Storage = .extended; + pub fn tag(this: @This(), c: *const Configuration) @FieldType(BaseFlags, "tag") { + const base_flags: BaseFlags = @bitCast(c.extra[@intFromEnum(this)]); + return base_flags.tag; + } + pub fn get(this: @This(), buffer: []const u32) U { var i: usize = @intFromEnum(this); const base_flags: BaseFlags = @bitCast(buffer[i]); return switch (base_flags.tag) { - inline else => |tag| @unionInit(U, @tagName(tag), data(buffer, &i, @FieldType(U, @tagName(tag)))), + inline else => |t| @unionInit(U, @tagName(t), data(buffer, &i, @FieldType(U, @tagName(t)))), }; } }; diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 81e07f317d8a0f86c974c2023827cbeffa1a7e45..d5c99660bb1996134b9308ce36dedd10d042db30 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -384,7 +384,10 @@ pub fn create(owner: *std.Build, options: Options) *Compile { const out_filename = std.zig.binNameAlloc(arena, .{ .root_name = name, - .target = target, + .cpu_arch = target.cpu.arch, + .os_tag = target.os.tag, + .ofmt = target.ofmt, + .abi = target.abi, .output_mode = switch (options.kind) { .lib => .Lib, .obj, .test_obj => .Obj, diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 63d3e3d0d68e85d3b616aa2a24b8f2116fe4e8d9..3abf43b3258452a24cdebb12f92e0d5085e338dc 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -146,7 +146,10 @@ pub fn lineDelta(source: []const u8, start: usize, end: usize) isize { pub const BinNameOptions = struct { root_name: []const u8, - target: *const std.Target, + cpu_arch: std.Target.Cpu.Arch, + os_tag: std.Target.Os.Tag, + ofmt: std.Target.ObjectFormat, + abi: std.Target.Abi, output_mode: std.builtin.OutputMode, link_mode: ?std.builtin.LinkMode = null, version: ?std.SemanticVersion = null, @@ -155,10 +158,12 @@ pub const BinNameOptions = struct { /// Returns the standard file system basename of a binary generated by the Zig compiler. pub fn binNameAlloc(allocator: Allocator, options: BinNameOptions) error{OutOfMemory}![]u8 { const root_name = options.root_name; - const t = options.target; - switch (t.ofmt) { + switch (options.ofmt) { .coff => switch (options.output_mode) { - .Exe => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, t.exeFileExt() }), + .Exe => return std.fmt.allocPrint(allocator, "{s}{s}", .{ + root_name, + options.os_tag.exeFileExt(options.cpu_arch), + }), .Lib => { const suffix = switch (options.link_mode orelse .static) { .static => ".lib", @@ -173,16 +178,16 @@ pub fn binNameAlloc(allocator: Allocator, options: BinNameOptions) error{OutOfMe .Lib => { switch (options.link_mode orelse .static) { .static => return std.fmt.allocPrint(allocator, "{s}{s}.a", .{ - t.libPrefix(), root_name, + options.os_tag.libPrefix(options.abi), root_name, }), .dynamic => { if (options.version) |ver| { return std.fmt.allocPrint(allocator, "{s}{s}.so.{d}.{d}.{d}", .{ - t.libPrefix(), root_name, ver.major, ver.minor, ver.patch, + options.os_tag.libPrefix(options.abi), root_name, ver.major, ver.minor, ver.patch, }); } else { return std.fmt.allocPrint(allocator, "{s}{s}.so", .{ - t.libPrefix(), root_name, + options.os_tag.libPrefix(options.abi), root_name, }); } }, @@ -195,16 +200,16 @@ pub fn binNameAlloc(allocator: Allocator, options: BinNameOptions) error{OutOfMe .Lib => { switch (options.link_mode orelse .static) { .static => return std.fmt.allocPrint(allocator, "{s}{s}.a", .{ - t.libPrefix(), root_name, + options.os_tag.libPrefix(options.abi), root_name, }), .dynamic => { if (options.version) |ver| { return std.fmt.allocPrint(allocator, "{s}{s}.{d}.{d}.{d}.dylib", .{ - t.libPrefix(), root_name, ver.major, ver.minor, ver.patch, + options.os_tag.libPrefix(options.abi), root_name, ver.major, ver.minor, ver.patch, }); } else { return std.fmt.allocPrint(allocator, "{s}{s}.dylib", .{ - t.libPrefix(), root_name, + options.os_tag.libPrefix(options.abi), root_name, }); } }, @@ -213,11 +218,14 @@ pub fn binNameAlloc(allocator: Allocator, options: BinNameOptions) error{OutOfMe .Obj => return std.fmt.allocPrint(allocator, "{s}.o", .{root_name}), }, .wasm => switch (options.output_mode) { - .Exe => return std.fmt.allocPrint(allocator, "{s}{s}", .{ root_name, t.exeFileExt() }), + .Exe => return std.fmt.allocPrint(allocator, "{s}{s}", .{ + root_name, + options.os_tag.exeFileExt(options.cpu_arch), + }), .Lib => { switch (options.link_mode orelse .static) { .static => return std.fmt.allocPrint(allocator, "{s}{s}.a", .{ - t.libPrefix(), root_name, + options.os_tag.libPrefix(options.abi), root_name, }), .dynamic => return std.fmt.allocPrint(allocator, "{s}.wasm", .{root_name}), } @@ -231,10 +239,10 @@ pub fn binNameAlloc(allocator: Allocator, options: BinNameOptions) error{OutOfMe .plan9 => switch (options.output_mode) { .Exe => return allocator.dupe(u8, root_name), .Obj => return std.fmt.allocPrint(allocator, "{s}{s}", .{ - root_name, t.ofmt.fileExt(t.cpu.arch), + root_name, options.ofmt.fileExt(options.cpu_arch), }), .Lib => return std.fmt.allocPrint(allocator, "{s}{s}.a", .{ - t.libPrefix(), root_name, + options.os_tag.libPrefix(options.abi), root_name, }), }, } -- 2.54.0 From aa0652ff8dc9add09567d69a92ddaff5ad386919 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 17 Mar 2026 12:18:56 -0700 Subject: [PATCH 048/179] maker: implement InstallArtifact and InstallFile --- BRANCH_TODO | 6 +- lib/compiler/Maker.zig | 214 ++++++++++++++++---- lib/compiler/Maker/Graph.zig | 3 +- lib/compiler/Maker/Step.zig | 60 +++--- lib/compiler/Maker/Step/Compile.zig | 45 +--- lib/compiler/Maker/Step/InstallArtifact.zig | 137 ++++++++----- lib/compiler/Maker/Step/InstallFile.zig | 25 +++ lib/compiler/configurer.zig | 35 ++-- lib/std/Build/Cache/Path.zig | 7 + lib/std/Build/Configuration.zig | 67 ++++-- lib/std/Build/Step/Compile.zig | 35 ---- lib/std/Build/Step/InstallArtifact.zig | 58 ++---- lib/std/Build/Step/InstallFile.zig | 12 -- 13 files changed, 421 insertions(+), 283 deletions(-) create mode 100644 lib/compiler/Maker/Step/InstallFile.zig diff --git a/BRANCH_TODO b/BRANCH_TODO index 6dd57427caf43f9774f7495957b13f4a7919869f..b55cf45b5b16d29d35f005e860f8893058783227 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,9 +1,10 @@ -* replace b.dupe() with string internment +* implement the build options * don't forget to add -listen arg back * get zig init template working * finish migrating the rest of the build steps * make zig-pkg path root configurable in maker (make sure --system still works) * eliminate calls to getPath, getPath2, getPath3 +* replace b.dupe() with string internment * solve the TODOs added in this branch * get zig tests passing * test a bunch of third party projects / help people migrate @@ -13,3 +14,6 @@ ## Followup Issues * link_eh_frame_hdr should be DefaultingBool * make --foo, --no-foo CLI args uniform (make them -f args instead) +* install steps should provide generated files for installed things, then delete the run step hack + + diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 36c6dd3dc3edf8fbe70d09a3486867d67b63c831..4e71912ef93292f3f64394e7252c34f8aa5463ff 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -7,6 +7,7 @@ const Cache = std.Build.Cache; const Configuration = std.Build.Configuration; const File = std.Io.File; const Io = std.Io; +const Dir = std.Io.Dir; const Path = std.Build.Cache.Path; const Writer = std.Io.Writer; const assert = std.debug.assert; @@ -82,7 +83,7 @@ pub fn main(init: process.Init.Minimal) !void { const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); - const cwd: Io.Dir = .cwd(); + const cwd: Dir = .cwd(); const zig_lib_directory: Cache.Directory = .{ .path = zig_lib_dir, @@ -497,7 +498,7 @@ pub fn main(init: process.Init.Minimal) !void { const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ .root_dir = .cwd(), - .sub_path = try Io.Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), + .sub_path = try Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), } else if (override_install_prefix) |cwd_relative| .{ .root_dir = .cwd(), .sub_path = cwd_relative, @@ -1675,7 +1676,7 @@ fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; if (wf.mode != .tmp) continue; const path = wf.generated_directory.path orelse continue; - Io.Dir.cwd().deleteTree(io, path) catch |err| { + Dir.cwd().deleteTree(io, path) catch |err| { log.warn("failed to delete {s}: {t}", .{ path, err }); }; } @@ -1699,29 +1700,14 @@ pub fn resolveLazyPath( ) Allocator.Error!Path { _ = asking_step_index; // TODO use this to enhance debugability when this function fails const c = &maker.scanned_config.configuration; - const graph = maker.graph; return switch (lazy_path) { .source_path => |sp| try packagePath(maker, arena, sp.owner, sp.sub_path.slice(c)), - .relative => |relative| switch (relative.flags.base) { - .cwd => .{ - .root_dir = .cwd(), - .sub_path = relative.sub_path.slice(c), - }, - .local_cache => .{ - .root_dir = graph.local_cache_root, - }, - .global_cache => .{ - .root_dir = graph.global_cache_root, - }, - .build_root => .{ - .root_dir = graph.build_root_directory, - }, - }, + .relative => |relative| relativePath(maker, relative), .generated => |gen| { - const base = maker.generated_files[@intFromEnum(gen.index)]; + const base = generatedPath(maker, gen.index); var file_path = base; for (0..gen.flags.up) |_| { - file_path.sub_path = Io.Dir.path.dirname(file_path.sub_path) orelse + file_path.sub_path = Dir.path.dirname(file_path.sub_path) orelse fatal("invalid LazyPath traversal: up {d} times from {f}", .{ gen.flags.up, base }); } return file_path.join(arena, gen.sub_path.slice(c)); @@ -1750,7 +1736,7 @@ pub fn resolveLazyPathAbs( const p = try resolveLazyPath(maker, arena, lazy_path, asking_step_index); const root_dir_path = p.root_dir.path orelse return p.subPathOrDot(); if (p.sub_path.len == 0) return root_dir_path; - return Io.Dir.path.join(arena, &.{ root_dir_path, p.sub_path }); + return Dir.path.join(arena, &.{ root_dir_path, p.sub_path }); } /// `resolveLazyPath` is preferred, but this can be necessary when passing Path @@ -1765,11 +1751,11 @@ pub fn resolveLazyPathIndexAbs( return resolveLazyPathAbs(maker, arena, lazy_path_index.get(c), asking_step_index); } -pub fn generatedPath(maker: *Maker, index: Configuration.GeneratedFileIndex) *Path { +pub fn generatedPath(maker: *const Maker, index: Configuration.GeneratedFileIndex) *Path { return &maker.generated_files[@intFromEnum(index)]; } -fn packagePath( +pub fn packagePath( maker: *const Maker, arena: Allocator, package_index: Configuration.Package.Index, @@ -1785,43 +1771,183 @@ fn packagePath( const pkg_root = graph.pkg_root; return .{ .root_dir = pkg_root.root_dir, - .sub_path = try Io.Dir.path.join(arena, &.{ pkg_root.sub_path, hash, sub_path }), + .sub_path = try Dir.path.join(arena, &.{ pkg_root.sub_path, hash, sub_path }), }; } -/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. -pub fn installFile( +pub fn relativePath(maker: *const Maker, relative: Configuration.LazyPath.Relative) Path { + const graph = maker.graph; + const c = &maker.scanned_config.configuration; + const sub_path = relative.sub_path.slice(c); + return switch (relative.flags.base) { + .cwd => .{ + .root_dir = .cwd(), + .sub_path = sub_path, + }, + .local_cache => .{ + .root_dir = graph.local_cache_root, + .sub_path = sub_path, + }, + .global_cache => .{ + .root_dir = graph.global_cache_root, + .sub_path = sub_path, + }, + .build_root => .{ + .root_dir = graph.build_root_directory, + .sub_path = sub_path, + }, + }; +} + +pub fn resolveInstallDir( + maker: *Maker, + arena: Allocator, + dest_dir: Configuration.InstallDestDir, +) Allocator.Error!Path { + const c = &maker.scanned_config.configuration; + return switch (dest_dir.unpack().?) { + .prefix => maker.install_paths.prefix, + .lib => maker.install_paths.lib, + .bin => maker.install_paths.bin, + .header => maker.install_paths.include, + .sub_path => |s| try maker.install_paths.prefix.join(arena, s.slice(c)), + }; +} + +pub fn installLazyPathSub( + maker: *Maker, + arena: Allocator, + source: Configuration.LazyPath.Index, + dest_dir: Configuration.InstallDestDir, + sub_path: []const u8, + asking_step_index: Configuration.Step.Index, +) !Dir.PrevStatus { + const src_path = try resolveLazyPathIndex(maker, arena, source, asking_step_index); + const dest_dir_path = try resolveInstallDir(maker, arena, dest_dir); + const dest_path = try dest_dir_path.join(arena, sub_path); + return installPath(maker, arena, src_path, dest_path, asking_step_index); +} + +pub fn installLazyPath( + maker: *Maker, + arena: Allocator, + source: Configuration.LazyPath.Index, + dest_dir: Configuration.InstallDestDir, + asking_step_index: Configuration.Step.Index, +) !Dir.PrevStatus { + const src_path = try resolveLazyPathIndex(maker, arena, source, asking_step_index); + const dest_dir_path = try resolveInstallDir(maker, arena, dest_dir); + const dest_path = try dest_dir_path.join(arena, src_path.basename()); + return installPath(maker, arena, src_path, dest_path, asking_step_index); +} + +pub fn installGenerated( + maker: *Maker, + arena: Allocator, + source: Configuration.GeneratedFileIndex, + dest_dir: Configuration.InstallDestDir, + asking_step_index: Configuration.Step.Index, +) !Dir.PrevStatus { + const src_path = generatedPath(maker, source).*; + const dest_dir_path = try resolveInstallDir(maker, arena, dest_dir); + const dest_path = try dest_dir_path.join(arena, src_path.basename()); + return installPath(maker, arena, src_path, dest_path, asking_step_index); +} + +pub fn installPath( maker: *Maker, arena: Allocator, - src_lazy_path: Configuration.LazyPath, - dest_path: []const u8, + src_path: Path, + dest_path: Path, asking_step_index: Configuration.Step.Index, -) !Io.Dir.PrevStatus { +) !Dir.PrevStatus { const graph = maker.graph; const io = graph.io; - const src_path = try resolveLazyPath(maker, arena, src_lazy_path, asking_step_index); - { - const src_path_rendered = try src_path.toString(arena); - defer arena.free(src_path_rendered); - try graph.handleVerbose(.inherit, null, &.{ "install", "-C", src_path_rendered, dest_path }); - } - return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| { + if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ + "install", "-C", try src_path.toString(arena), try dest_path.toString(arena), + }); + return Dir.updateFile( + src_path.root_dir.handle, + io, + src_path.sub_path, + dest_path.root_dir.handle, + dest_path.sub_path, + .{}, + ) catch |err| { const s = stepByIndex(maker, asking_step_index); - return s.fail(maker, "unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); + return s.fail(maker, "unable to update file from {f} to {f}: {t}", .{ src_path, dest_path, err }); }; } -/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. +/// Wrapper around `Dir.createDirPathStatus` that handles verbose and error output. pub fn installDir( maker: *Maker, - dest_path: []const u8, + arena: Allocator, + dest_path: Path, asking_step_index: Configuration.Step.Index, -) !Io.Dir.CreatePathStatus { +) !Dir.CreatePathStatus { const graph = maker.graph; const io = graph.io; - try graph.handleVerbose(.inherit, null, &.{ "install", "-d", dest_path }); - return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| { + if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ + "install", "-d", try dest_path.toString(arena), + }); + return dest_path.root_dir.handle.createDirPathStatus(io, dest_path.sub_path, .default_dir) catch |err| { const s = stepByIndex(maker, asking_step_index); - return s.fail(maker, "unable to create dir '{s}': {t}", .{ dest_path, err }); + return s.fail(maker, "unable to create dir {f}: {t}", .{ dest_path, err }); + }; +} + +pub fn installSymLinks( + maker: *Maker, + arena: Allocator, + output_path: Path, + compile_step_index: Configuration.Step.Index, + asking_step_index: Configuration.Step.Index, +) !void { + const c = &maker.scanned_config.configuration; + const conf_step = compile_step_index.ptr(c); + const conf_comp = conf_step.extended.get(c.extra).compile; + const root_module = conf_comp.root_module.get(c); + const target = root_module.resolved_target.get(c).?.result.get(c); + const os_tag = target.flags.os_tag.unwrap().?; + + assert(conf_comp.flags3.kind == .lib); + assert(conf_comp.flags2.linkage == .dynamic); + assert(os_tag != .windows); + + const version = std.SemanticVersion.parse(conf_comp.version.value.?.slice(c)) catch unreachable; + const name = conf_comp.root_name.slice(c); + + const filename_major_only, const filename_name_only = if (os_tag.isDarwin()) .{ + try std.fmt.allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }), + try std.fmt.allocPrint(arena, "lib{s}.dylib", .{name}), + } else .{ + try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }), + try std.fmt.allocPrint(arena, "lib{s}.so", .{name}), + }; + + return installSymLinksInner(maker, arena, output_path, asking_step_index, filename_major_only, filename_name_only); +} + +fn installSymLinksInner( + maker: *Maker, + arena: Allocator, + output_path: Path, + asking_step_index: Configuration.Step.Index, + filename_major_only: []const u8, + filename_name_only: []const u8, +) !void { + const io = maker.graph.io; + const step = stepByIndex(maker, asking_step_index); + const out_dir = output_path.dirname().?; + // sym link for libfoo.so.1 to libfoo.so.1.2.3 + const major_only_path = try out_dir.join(arena, filename_major_only); + output_path.root_dir.handle.symLinkAtomic(io, output_path.sub_path, major_only_path.sub_path, .{}) catch |err| { + return step.fail(maker, "unable to symlink {f} -> {f}: {t}", .{ output_path, major_only_path, err }); + }; + // sym link for libfoo.so to libfoo.so.1 + const name_only_path = try out_dir.join(arena, filename_name_only); + major_only_path.root_dir.handle.symLinkAtomic(io, major_only_path.sub_path, name_only_path.sub_path, .{}) catch |err| { + return step.fail(maker, "unable to symlink {f} -> {s}: {t}", .{ name_only_path, filename_major_only, err }); }; } diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index e3b49b4b98733d8f4d67d51203578d17a877feff..3be18927fddc5403a6333b672726c471931dcc96 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -5,6 +5,7 @@ const std = @import("std"); const Io = std.Io; const Allocator = std.mem.Allocator; const Configuration = std.Build.Configuration; +const Path = std.Build.Cache.Path; io: Io, /// Process lifetime. @@ -16,7 +17,7 @@ global_cache_root: std.Build.Cache.Directory, local_cache_root: std.Build.Cache.Directory, zig_lib_directory: std.Build.Cache.Directory, build_root_directory: std.Build.Cache.Directory, -pkg_root: std.Build.Cache.Path, +pkg_root: Path, debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, incremental: ?bool = null, diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index e238c2cc8e52327718579feabf9b0974f0b47db2..f73b6b7e5b52ca5cd8284d6515ba1c9f614756f8 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -8,6 +8,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const Cache = std.Build.Cache; const Io = std.Io; +const Dir = std.Io.Dir; const LazyPath = std.Build.Configuration.LazyPath; const Package = std.Build.Configuration.Package; const Path = std.Build.Cache.Path; @@ -19,6 +20,8 @@ const Maker = @import("../Maker.zig"); const Compile = @import("Step/Compile.zig"); const Run = @import("Step/Run.zig"); +const InstallArtifact = @import("Step/InstallArtifact.zig"); +const InstallFile = @import("Step/InstallFile.zig"); /// Avoid false sharing. _: void align(std.atomic.cache_line) = {}, @@ -69,9 +72,9 @@ pub const Extended = union(enum) { config_header: Todo, fail: Todo, fmt: Todo, - install_artifact: Todo, + install_artifact: InstallArtifact, install_dir: Todo, - install_file: Todo, + install_file: InstallFile, objcopy: Todo, options: Todo, remove_dir: Todo, @@ -524,7 +527,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; result = .{ .root_dir = graph.local_cache_root, - .sub_path = try arena.dupe(u8, "o" ++ Io.Dir.path.sep_str ++ Cache.binToHex(digest.*)), + .sub_path = try arena.dupe(u8, "o" ++ Dir.path.sep_str ++ Cache.binToHex(digest.*)), }; }, .file_system_inputs => { @@ -535,14 +538,14 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi while (it.next()) |prefixed_path| { const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); const sub_path = try arena.dupe(u8, prefixed_path[1..]); - const sub_path_dirname = Io.Dir.path.dirname(sub_path) orelse ""; + const sub_path_dirname = Dir.path.dirname(sub_path) orelse ""; switch (prefix_index) { .cwd => { const path: Path = .{ .root_dir = .cwd(), .sub_path = sub_path_dirname, }; - try addWatchInputFromPath(s, maker, path, Io.Dir.path.basename(sub_path)); + try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); }, .zig_lib => zl: { switch (conf_step.extended.get(conf.extra)) { @@ -558,21 +561,21 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi .root_dir = graph.zig_lib_directory, .sub_path = sub_path_dirname, }; - try addWatchInputFromPath(s, maker, path, Io.Dir.path.basename(sub_path)); + try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); }, .local_cache => { const path: Path = .{ .root_dir = graph.local_cache_root, .sub_path = sub_path_dirname, }; - try addWatchInputFromPath(s, maker, path, Io.Dir.path.basename(sub_path)); + try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); }, .global_cache => { const path: Path = .{ .root_dir = graph.global_cache_root, .sub_path = sub_path_dirname, }; - try addWatchInputFromPath(s, maker, path, Io.Dir.path.basename(sub_path)); + try addWatchInputFromPath(s, maker, path, Dir.path.basename(sub_path)); }, } } @@ -680,7 +683,7 @@ fn failWithCacheError( const pp = man.files.keys()[op.file_index].prefixed_path; const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; return s.fail(maker, "failed to check cache: '{s}{c}{s}' {t} {t}", .{ - prefix, Io.Dir.path.sep, pp.sub_path, man.diagnostic, op.err, + prefix, Dir.path.sep, pp.sub_path, man.diagnostic, op.err, }); }, }, @@ -718,14 +721,14 @@ fn setWatchInputsFromManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !vo const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); try addWatchInputFromPath(s, maker, .{ .root_dir = prefixes[file.prefixed_path.prefix], - .sub_path = Io.Dir.path.dirname(sub_path) orelse "", - }, Io.Dir.path.basename(sub_path)); + .sub_path = Dir.path.dirname(sub_path) orelse "", + }, Dir.path.basename(sub_path)); } } /// For steps that have a single input that never changes when re-running `make`. -pub fn singleUnchangingWatchInput(step: *Step, maker: *Maker, lazy_path: LazyPath) Allocator.Error!void { - if (!step.inputs.populated()) try step.addWatchInput(maker, lazy_path); +pub fn singleUnchangingWatchInput(step: *Step, maker: *Maker, arena: Allocator, lazy_path: LazyPath) Allocator.Error!void { + if (!step.inputs.populated()) try step.addWatchInput(maker, arena, lazy_path); } pub fn clearWatchInputs(step: *Step, maker: *Maker) void { @@ -733,19 +736,15 @@ pub fn clearWatchInputs(step: *Step, maker: *Maker) void { } /// Places a *file* dependency on the path. -pub fn addWatchInput(step: *Step, maker: *Maker, lazy_file: LazyPath) Allocator.Error!void { +pub fn addWatchInput(step: *Step, maker: *Maker, arena: Allocator, lazy_file: LazyPath) Allocator.Error!void { + const conf = &maker.scanned_config.configuration; switch (lazy_file) { - .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), - .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), - .cwd_relative => |path_string| { - try addWatchInputFromPath(step, maker, .{ - .root_dir = .{ - .path = null, - .handle = Io.Dir.cwd(), - }, - .sub_path = Io.Dir.path.dirname(path_string) orelse "", - }, Io.Dir.path.basename(path_string)); + .source_path => |source_path| { + const sub_path = source_path.sub_path.slice(conf); + const pkg_path = try maker.packagePath(arena, source_path.owner, sub_path); + try addWatchInputPath(step, maker, pkg_path); }, + .relative => |relative| try addWatchInputPath(step, maker, maker.relativePath(relative)), // Nothing to watch because this dependency edge is modeled instead via `dependants`. .generated => {}, } @@ -766,7 +765,7 @@ pub fn addDirectoryWatchInput(step: *Step, lazy_directory: LazyPath) Allocator.E try addDirectoryWatchInputFromPath(step, .{ .root_dir = .{ .path = null, - .handle = Io.Dir.cwd(), + .handle = .cwd(), }, .sub_path = path_string, }); @@ -789,13 +788,6 @@ pub fn addDirectoryWatchInputFromPath(step: *Step, maker: *Maker, path: Path) !v return addWatchInputFromPath(step, maker, path, "."); } -fn addWatchInputFromBuilder(step: *Step, maker: *Maker, package: Package, sub_path: []const u8) !void { - return addWatchInputFromPath(step, maker, .{ - .root_dir = package.build_root, - .sub_path = Io.Dir.path.dirname(sub_path) orelse "", - }, Io.Dir.path.basename(sub_path)); -} - fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { return addDirectoryWatchInputFromPath(step, .{ .root_dir = package.build_root, @@ -806,8 +798,8 @@ fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: [] fn addWatchInputPath(step: *Step, maker: *Maker, path: Path) Allocator.Error!void { return addWatchInputFromPath(step, maker, .{ .root_dir = path.root_dir, - .sub_path = Io.Dir.path.dirname(path.sub_path) orelse "", - }, Io.Dir.path.basename(path.sub_path)); + .sub_path = Dir.path.dirname(path.sub_path) orelse "", + }, Dir.path.basename(path.sub_path)); } fn addWatchInputFromPath(step: *Step, maker: *Maker, directory: Path, basename: []const u8) Allocator.Error!void { diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index d781791e806bf09eb835a3ee76812cacc0969202..464acf5ba2509192bfa6fd2e3d86592537ae6bc3 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -28,7 +28,7 @@ pub fn make( progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { const graph = maker.graph; - const step = maker.stepByIndex(compile_index); + const arena = graph.arena; // TODO don't leak into process arena const conf = &maker.scanned_config.configuration; const conf_step = compile_index.ptr(conf); const conf_comp = conf_step.extended.get(conf.extra).compile; @@ -69,16 +69,12 @@ pub fn make( } if (conf_comp.flags3.kind == .lib and conf_comp.flags2.linkage == .dynamic and - conf_comp.version.value != null and conf_comp.generated_bin.value != null and - target.flags.os_tag != .windows) + conf_comp.version.value != null and target.flags.os_tag != .windows) { - if (true) @panic("TODO"); - try doAtomicSymLinks( - step, - conf_comp.getEmittedBin().getPath2(step), - conf_comp.major_only_filename.?, - conf_comp.name_only_filename.?, - ); + if (conf_comp.generated_bin.value) |generated_bin| { + const full_dest_path = maker.generatedPath(generated_bin).*; + try maker.installSymLinks(arena, full_dest_path, compile_index, compile_index); + } } } @@ -964,35 +960,6 @@ pub fn rebuildInFuzzMode(compile: *Compile, maker: *Maker, progress_node: std.Pr return maybe_output_bin_path.?; } -pub fn doAtomicSymLinks( - step: *Step, - maker: *Maker, - output_path: []const u8, - filename_major_only: []const u8, - filename_name_only: []const u8, -) !void { - const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into process arena - const io = graph.io; - const out_dir = Dir.path.dirname(output_path) orelse "."; - const out_basename = Dir.path.basename(output_path); - // sym link for libfoo.so.1 to libfoo.so.1.2.3 - const major_only_path = try Dir.path.join(arena, &.{ out_dir, filename_major_only }); - const cwd: Io.Dir = .cwd(); - cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { - return step.fail(maker, "unable to symlink {s} -> {s}: {t}", .{ - major_only_path, out_basename, err, - }); - }; - // sym link for libfoo.so to libfoo.so.1 - const name_only_path = try Dir.path.join(arena, &.{ out_dir, filename_name_only }); - cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { - return step.fail(maker, "unable to symlink {s} -> {s}: {t}", .{ - name_only_path, filename_major_only, err, - }); - }; -} - pub const PkgConfigError = error{ PkgConfigCrashed, PkgConfigFailed, diff --git a/lib/compiler/Maker/Step/InstallArtifact.zig b/lib/compiler/Maker/Step/InstallArtifact.zig index ba3846c1a574f9934f4196f03529dac3abcbe275..2d761566484769b91c8fd347a3631ac4ee7e129b 100644 --- a/lib/compiler/Maker/Step/InstallArtifact.zig +++ b/lib/compiler/Maker/Step/InstallArtifact.zig @@ -1,88 +1,127 @@ +const InstallArtifact = @This(); -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); - const b = step.owner; - const io = b.graph.io; +const std = @import("std"); +const Io = std.Io; +const Configuration = std.Build.Configuration; +const assert = std.debug.assert; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + install_artifact: *InstallArtifact, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + _ = install_artifact; + _ = progress_node; + const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into process arena + const io = graph.io; + const conf_step = step_index.ptr(conf); + const conf_ia = conf_step.extended.get(conf.extra).install_artifact; + const compile_step_index = conf_step.deps.get(conf).steps.slice[0]; + const conf_comp_step = compile_step_index.ptr(conf); + const conf_comp = conf_comp_step.extended.get(conf.extra).compile; + const root_module = conf_comp.root_module.get(conf); + const target = root_module.resolved_target.get(conf).?.result.get(conf); var all_cached = true; - if (install_artifact.dest_dir) |dest_dir| { - const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); - const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); - all_cached = all_cached and p == .fresh; + if (conf_ia.bin_dir.value) |bin_dir| { + if (conf_comp.generated_bin.value) |generated_bin| { + const bin_sub_path = if (conf_ia.bin_sub_path.value) |s| s.slice(conf) else try std.zig.binNameAlloc(arena, .{ + .root_name = conf_comp.root_name.slice(conf), + .cpu_arch = target.flags.cpu_arch.unwrap().?, + .os_tag = target.flags.os_tag.unwrap().?, + .ofmt = target.flags.object_format.unwrap().?, + .abi = target.flags.abi.unwrap().?, + .output_mode = conf_comp.flags3.kind.toOutputMode(), + .link_mode = conf_comp.flags2.linkage.unwrap(), + .version = v: { + const string = conf_comp.version.value orelse break :v null; + const slice = string.slice(conf); + break :v std.SemanticVersion.parse(slice) catch @panic("bad semver string"); + }, + }); + const dest_dir = try maker.resolveInstallDir(arena, bin_dir); + const dest_path = try dest_dir.join(arena, bin_sub_path); + const src_path = maker.generatedPath(generated_bin).*; + const p = try maker.installPath(arena, src_path, dest_path, step_index); + all_cached = all_cached and p == .fresh; - if (install_artifact.dylib_symlinks) |dls| { - try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); + if (conf_ia.flags.dylib_symlinks) + try maker.installSymLinks(arena, dest_path, compile_step_index, step_index); } - - install_artifact.artifact.installed_path = full_dest_path; } - if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { - const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); - all_cached = all_cached and p == .fresh; + if (conf_ia.implib_dir.value) |implib_dir| { + if (conf_comp.generated_implib.value) |generated_implib| { + const p = try maker.installGenerated(arena, generated_implib, implib_dir, step_index); + all_cached = all_cached and p == .fresh; + } } - if (install_artifact.implib_dir) |implib_dir| { - const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); - all_cached = all_cached and p == .fresh; + if (conf_ia.pdb_dir.value) |pdb_dir| { + if (conf_comp.generated_pdb.value) |generated_pdb| { + const p = try maker.installGenerated(arena, generated_pdb, pdb_dir, step_index); + all_cached = all_cached and p == .fresh; + } } - if (install_artifact.pdb_dir) |pdb_dir| { - const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); - all_cached = all_cached and p == .fresh; - } + if (conf_ia.h_dir.value) |h_dir| { + const h_prefix = try maker.resolveInstallDir(arena, h_dir); - if (install_artifact.h_dir) |h_dir| { - if (install_artifact.emitted_h) |emitted_h| { - const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); - const p = try step.installFile(emitted_h, full_h_path); + if (conf_comp.generated_h.value) |generated_h| { + const p = try maker.installGenerated(arena, generated_h, h_dir, step_index); all_cached = all_cached and p == .fresh; } - for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { + for (conf_comp.installed_headers.slice) |installation| switch (installation.get(conf.extra)) { .file => |file| { - const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); - const p = try step.installFile(file.source, full_h_path); + const src_path = try maker.resolveLazyPathIndex(arena, file.source, step_index); + const dest_path = try h_prefix.join(arena, file.dest_sub_path.slice(conf)); + const p = try maker.installPath(arena, src_path, dest_path, step_index); all_cached = all_cached and p == .fresh; }, .directory => |dir| { - const src_dir_path = dir.source.getPath3(b, step); - const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); + const src_dir_path = try maker.resolveLazyPathIndex(arena, dir.source, step_index); + const full_h_prefix = try h_prefix.join(arena, dir.dest_sub_path.slice(conf)); var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {s}", .{ - src_dir_path, @errorName(err), - }); + return step.fail(maker, "unable to open source directory {f}: {t}", .{ src_dir_path, err }); }; defer src_dir.close(io); - var it = try src_dir.walk(b.allocator); - next_entry: while (try it.next(io)) |entry| { - for (dir.options.exclude_extensions) |ext| { - if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; + var it = try src_dir.walk(arena); + next_entry: while (it.next(io) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return step.fail(maker, "failed to iterate directory {f}: {t}", .{ src_dir_path, e }), + }) |entry| { + for (dir.exclude_extensions.slice) |ext| { + if (std.mem.endsWith(u8, entry.path, ext.slice(conf))) continue :next_entry; } - if (dir.options.include_extensions) |incs| { - for (incs) |inc| { - if (std.mem.endsWith(u8, entry.path, inc)) break; + if (dir.flags.include_extensions) { + for (dir.include_extensions.slice) |inc| { + if (std.mem.endsWith(u8, entry.path, inc.slice(conf))) break; } else { continue :next_entry; } } - const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); + const full_dest_path = try full_h_prefix.join(arena, entry.path); switch (entry.kind) { .directory => { - try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); - const p = try step.installDir(full_dest_path); + const p = try maker.installDir(arena, full_dest_path, step_index); all_cached = all_cached and p == .existed; }, .file => { - const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); + const entry_dir_path = try maker.resolveLazyPathIndex(arena, dir.source, step_index); + const entry_path = try entry_dir_path.join(arena, entry.path); + const p = try maker.installPath(arena, entry_path, full_dest_path, step_index); all_cached = all_cached and p == .fresh; }, else => continue, diff --git a/lib/compiler/Maker/Step/InstallFile.zig b/lib/compiler/Maker/Step/InstallFile.zig new file mode 100644 index 0000000000000000000000000000000000000000..439c793c6f1c7ef5b8875b6913e2ad6f0785a6e8 --- /dev/null +++ b/lib/compiler/Maker/Step/InstallFile.zig @@ -0,0 +1,25 @@ +const InstallFile = @This(); + +const std = @import("std"); +const Configuration = std.Build.Configuration; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + install_file: *InstallFile, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + _ = install_file; + _ = progress_node; + const arena = maker.graph.arena; // TODO don't leak into process arena + const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_if = conf_step.extended.get(conf.extra).install_file; + try step.singleUnchangingWatchInput(maker, arena, conf_if.source.get(conf)); + const p = try maker.installLazyPathSub(arena, conf_if.source, conf_if.dest_dir, conf_if.dest_sub_path.slice(conf), step_index); + step.result_cached = p == .fresh; +} diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 18b7cdf1b7d12f9b6439d343790f43494c74f844..816f71158c909cb1a45a24ccc6aaf160a1529113 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -739,21 +739,28 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { const ia: *Step.InstallArtifact = @fieldParentPtr("step", step); break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.InstallArtifact, .{ .flags = .{ - .dylib_symlinks = ia.dylib_symlinks != null, + .dylib_symlinks = ia.dylib_symlinks, + .bin_dir = ia.dest_dir != null, + .implib_dir = ia.implib_dir != null, + .pdb_dir = ia.pdb_dir != null, + .h_dir = ia.h_dir != null, + .bin_sub_path = ia.dest_sub_path != null, }, - .dest_dir = try addInstallDir(wc, ia.dest_dir), - .dest_sub_path = try wc.addString(ia.dest_sub_path), - .emitted_bin = try s.addOptionalLazyPathEnum(ia.emitted_bin), - .implib_dir = try addInstallDir(wc, ia.implib_dir), - .emitted_implib = try s.addOptionalLazyPathEnum(ia.emitted_implib), - .pdb_dir = try addInstallDir(wc, ia.pdb_dir), - .emitted_pdb = try s.addOptionalLazyPathEnum(ia.emitted_pdb), - .h_dir = try addInstallDir(wc, ia.h_dir), - .emitted_h = try s.addOptionalLazyPathEnum(ia.emitted_h), - .artifact = s.stepIndex(&ia.artifact.step), + .bin_dir = .{ .value = try addInstallDirDefaultNull(wc, ia.dest_dir) }, + .implib_dir = .{ .value = try addInstallDirDefaultNull(wc, ia.implib_dir) }, + .pdb_dir = .{ .value = try addInstallDirDefaultNull(wc, ia.pdb_dir) }, + .h_dir = .{ .value = try addInstallDirDefaultNull(wc, ia.h_dir) }, + .bin_sub_path = .{ .value = try s.addOptionalString(ia.dest_sub_path) }, + }))); + }, + .install_file => e: { + const sif: *Step.InstallFile = @fieldParentPtr("step", step); + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.InstallFile, .{ + .source = try s.addLazyPath(sif.source), + .dest_dir = try addInstallDir(wc, sif.dir), + .dest_sub_path = try wc.addString(sif.dest_rel_path), }))); }, - .install_file => @panic("TODO"), .install_dir => @panic("TODO"), .remove_dir => @panic("TODO"), .fail => @panic("TODO"), @@ -851,6 +858,10 @@ fn addInstallDir(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !Co } } +fn addInstallDirDefaultNull(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !?Configuration.InstallDestDir { + return try addInstallDir(wc, install_dir orelse return null); +} + /// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which /// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index 30565043ea22bf3704ad86f6f8c59be2a405fc83..14f200579c3e0ea6b796fa9c663a6f39007bb131 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -213,6 +213,13 @@ pub fn stem(p: Path) []const u8 { return fs.path.stem(p.sub_path); } +pub fn dirname(p: Path) ?Path { + return .{ + .root_dir = p.root_dir, + .sub_path = fs.path.dirname(p.subPathOpt() orelse return null) orelse "", + }; +} + pub fn basename(p: Path) []const u8 { return fs.path.basename(p.sub_path); } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 96fdb39c7157ef640dd9014774143ee12bb27e28..7c4f06562d0f262d818b9bdaecb95f44802138c3 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -478,29 +478,25 @@ pub const Step = extern struct { }; }; + /// The first dependency step index will be the compile step whose + /// artifacts are being installed with this step. pub const InstallArtifact = struct { flags: @This().Flags, - - dest_dir: InstallDestDir, - dest_sub_path: String, - emitted_bin: LazyPath.OptionalIndex, - - implib_dir: InstallDestDir, - emitted_implib: LazyPath.OptionalIndex, - - pdb_dir: InstallDestDir, - emitted_pdb: LazyPath.OptionalIndex, - - h_dir: InstallDestDir, - emitted_h: LazyPath.OptionalIndex, - - /// Always a compile step. - artifact: Step.Index, + bin_dir: Storage.FlagOptional(.flags, .bin_dir, InstallDestDir), + implib_dir: Storage.FlagOptional(.flags, .implib_dir, InstallDestDir), + pdb_dir: Storage.FlagOptional(.flags, .pdb_dir, InstallDestDir), + h_dir: Storage.FlagOptional(.flags, .h_dir, InstallDestDir), + bin_sub_path: Storage.FlagOptional(.flags, .bin_sub_path, String), pub const Flags = packed struct(u32) { tag: Tag = .install_artifact, dylib_symlinks: bool, - _: u26 = 0, + bin_dir: bool, + implib_dir: bool, + pdb_dir: bool, + h_dir: bool, + bin_sub_path: bool, + _: u21 = 0, }; }; @@ -790,6 +786,14 @@ pub const Step = extern struct { .@"test", .test_obj => true, }; } + + pub fn toOutputMode(kind: Kind) std.builtin.OutputMode { + return switch (kind) { + .exe, .@"test" => .Exe, + .lib => .Lib, + .obj, .test_obj => .Obj, + }; + } }; pub const Subsystem = enum(u4) { console, @@ -991,7 +995,10 @@ pub const Step = extern struct { }; pub const InstallFile = struct { - flags: @This().Flags, + flags: @This().Flags = .{}, + source: LazyPath.Index, + dest_dir: InstallDestDir, + dest_sub_path: String, pub const Flags = packed struct(u32) { tag: Tag = .install_file, @@ -1434,6 +1441,25 @@ pub const InstallDestDir = enum(u32) { assert(@intFromEnum(sub_path) < @intFromEnum(InstallDestDir.none)); return @enumFromInt(@intFromEnum(sub_path)); } + + pub const Unpacked = union(enum) { + prefix, + lib, + bin, + header, + sub_path: String, + }; + + pub fn unpack(this: @This()) ?Unpacked { + return switch (this) { + .none => null, + .prefix => .prefix, + .lib => .lib, + .bin => .bin, + .header => .header, + _ => .{ .sub_path = @enumFromInt(@intFromEnum(this)) }, + }; + } }; /// Points into `string_bytes`, null-terminated. @@ -2366,7 +2392,8 @@ pub const Storage = enum { else => comptime unreachable, }, .auto => switch (Field.storage) { - .flag_optional, .enum_optional, .extended => 1, + .flag_optional, .enum_optional => (@sizeOf(Field.Value) + 3) / 4, + .extended => 1, .length_prefixed_list, .flag_length_prefixed_list, .flag_list, @@ -2520,7 +2547,7 @@ pub const LoadError = Io.Reader.Error || Allocator.Error; pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { const header = try reader.takeStruct(Header, .little); - var result: Configuration = .{ + const result: Configuration = .{ .string_bytes = try arena.alloc(u8, header.string_bytes_len), .steps = try arena.alloc(Step, header.steps_len), .path_deps_sub = try arena.alloc(String, header.path_deps_len), diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index d5c99660bb1996134b9308ce36dedd10d042db30..8773baf1b2bde5c050e42331f845bdbe585a554e 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -26,12 +26,9 @@ name: []const u8, linker_script: ?LazyPath = null, version_script: ?LazyPath = null, out_filename: []const u8, -out_lib_filename: []const u8, linkage: ?std.builtin.LinkMode = null, version: ?std.SemanticVersion, kind: Kind, -major_only_filename: ?[]const u8, -name_only_filename: ?[]const u8, formatted_panics: ?bool = null, compress_debug_sections: std.zig.CompressDebugSections = .none, verbose_link: bool, @@ -413,9 +410,6 @@ pub fn create(owner: *std.Build, options: Options) *Compile { }), .version = options.version, .out_filename = out_filename, - .out_lib_filename = undefined, - .major_only_filename = null, - .name_only_filename = null, .installed_headers = .empty, .zig_lib_dir = null, .exec_cmd_args = null, @@ -463,35 +457,6 @@ pub fn create(owner: *std.Build, options: Options) *Compile { lp.addStepDependencies(&compile.step); } - if (compile.kind == .lib) { - if (compile.linkage != null and compile.linkage.? == .static) { - compile.out_lib_filename = compile.out_filename; - } else if (compile.version) |version| { - if (target.os.tag.isDarwin()) { - compile.major_only_filename = owner.fmt("lib{s}.{d}.dylib", .{ - compile.name, - version.major, - }); - compile.name_only_filename = owner.fmt("lib{s}.dylib", .{compile.name}); - compile.out_lib_filename = compile.out_filename; - } else if (target.os.tag == .windows) { - compile.out_lib_filename = owner.fmt("{s}.lib", .{compile.name}); - } else { - compile.major_only_filename = owner.fmt("lib{s}.so.{d}", .{ compile.name, version.major }); - compile.name_only_filename = owner.fmt("lib{s}.so", .{compile.name}); - compile.out_lib_filename = compile.out_filename; - } - } else { - if (target.os.tag.isDarwin()) { - compile.out_lib_filename = compile.out_filename; - } else if (target.os.tag == .windows) { - compile.out_lib_filename = owner.fmt("{s}.lib", .{compile.name}); - } else { - compile.out_lib_filename = compile.out_filename; - } - } - } - return compile; } diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig index 820de6b142c2a0f2d909935fdf782d7647672b97..4f4aa4301b622c8042d6428855452083137bca8e 100644 --- a/lib/std/Build/Step/InstallArtifact.zig +++ b/lib/std/Build/Step/InstallArtifact.zig @@ -8,7 +8,7 @@ const LazyPath = std.Build.LazyPath; step: Step, dest_dir: ?InstallDir, -dest_sub_path: []const u8, +dest_sub_path: ?[]const u8, emitted_bin: ?LazyPath, implib_dir: ?InstallDir, @@ -24,7 +24,7 @@ emitted_compiler_rt_dyn_lib: ?LazyPath, h_dir: ?InstallDir, emitted_h: ?LazyPath, -dylib_symlinks: ?DylibSymlinkInfo, +dylib_symlinks: bool, artifact: *Step.Compile, @@ -67,6 +67,16 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins }, .override => |o| o, }; + const pdb_dir: ?InstallDir = switch (options.pdb_dir) { + .disabled => null, + .default => if (artifact.producesPdbFile()) dest_dir else null, + .override => |o| o, + }; + const implib_dir: ?InstallDir = switch (options.implib_dir) { + .disabled => null, + .default => if (artifact.producesImplib()) .lib else null, + .override => |o| o, + }; install_artifact.* = .{ .step = Step.init(.{ .tag = base_tag, @@ -74,54 +84,30 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins .owner = owner, }), .dest_dir = dest_dir, - .pdb_dir = switch (options.pdb_dir) { - .disabled => null, - .default => if (artifact.producesPdbFile()) dest_dir else null, - .override => |o| o, - }, - .compiler_rt_dyn_lib_dir = switch (options.compiler_rt_dyn_lib_dir) { - .disabled => null, - .default => if (artifact.producesCompilerRtDynLib()) dest_dir else null, - .override => |o| o, - }, + .pdb_dir = pdb_dir, .h_dir = switch (options.h_dir) { .disabled => null, .default => if (artifact.kind == .lib) .header else null, .override => |o| o, }, - .implib_dir = switch (options.implib_dir) { - .disabled => null, - .default => if (artifact.producesImplib()) .lib else null, - .override => |o| o, - }, + .implib_dir = implib_dir, - .dylib_symlinks = if (options.dylib_symlinks orelse (dest_dir != null and - artifact.isDynamicLibrary() and - artifact.version != null and - std.Build.wantSharedLibSymLinks(artifact.rootModuleTarget()))) .{ - .major_only_filename = artifact.major_only_filename.?, - .name_only_filename = artifact.name_only_filename.?, - } else null, + .dylib_symlinks = options.dylib_symlinks orelse (dest_dir != null and + artifact.isDynamicLibrary() and artifact.version != null and + std.Build.wantSharedLibSymLinks(artifact.rootModuleTarget())), - .dest_sub_path = options.dest_sub_path orelse artifact.out_filename, + .dest_sub_path = options.dest_sub_path, - .emitted_bin = null, - .emitted_pdb = null, - .emitted_compiler_rt_dyn_lib = null, + .emitted_bin = if (dest_dir != null) artifact.getEmittedBin() else null, + .emitted_pdb = if (pdb_dir != null) artifact.getEmittedPdb() else null, + // https://github.com/ziglang/zig/issues/9698 .emitted_h = null, - .emitted_implib = null, + .emitted_implib = if (implib_dir != null) artifact.getEmittedImplib() else null, .artifact = artifact, }; install_artifact.step.dependOn(&artifact.step); - if (install_artifact.dest_dir != null) install_artifact.emitted_bin = artifact.getEmittedBin(); - if (install_artifact.compiler_rt_dyn_lib_dir != null) install_artifact.emitted_compiler_rt_dyn_lib = artifact.getEmittedCompilerRtDynLib(); - if (install_artifact.pdb_dir != null) install_artifact.emitted_pdb = artifact.getEmittedPdb(); - // https://github.com/ziglang/zig/issues/9698 - //if (install_artifact.h_dir != null) install_artifact.emitted_h = artifact.getEmittedH(); - if (install_artifact.implib_dir != null) install_artifact.emitted_implib = artifact.getEmittedImplib(); - return install_artifact; } diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index a73f126d16087893b9787bff6c372ba7f15cdedc..5b80b3d1a1b1b7c2b7103e1e196fa8766e316de3 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -25,7 +25,6 @@ pub fn create( .tag = base_tag, .name = owner.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), .owner = owner, - .makeFn = make, }), .source = source.dupe(owner), .dir = dir.dupe(owner), @@ -34,14 +33,3 @@ pub fn create( source.addStepDependencies(&install_file.step); return install_file; } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const install_file: *InstallFile = @fieldParentPtr("step", step); - try step.singleUnchangingWatchInput(install_file.source); - - const full_dest_path = b.getInstallPath(install_file.dir, install_file.dest_rel_path); - const p = try step.installFile(install_file.source, full_dest_path); - step.result_cached = p == .fresh; -} -- 2.54.0 From 1a63d26836f5c87e45280772b8ba9c822ba75b78 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 17 Mar 2026 17:10:00 -0700 Subject: [PATCH 049/179] maker: implement TopLevel step --- BRANCH_TODO | 1 + lib/compiler/Maker/Step.zig | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index b55cf45b5b16d29d35f005e860f8893058783227..5ddc34a0996823b21a2c81e92ee6d834eca7577a 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -10,6 +10,7 @@ * test a bunch of third party projects / help people migrate * refactor with DefaultingEnum * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) +* https://codeberg.org/ziglang/zig/issues/31397 ## Followup Issues * link_eh_frame_hdr should be DefaultingBool diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index f73b6b7e5b52ca5cd8284d6515ba1c9f614756f8..c1c9c1f2cc3db29fc8685b806f90b1f56353c7dd 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -79,7 +79,7 @@ pub const Extended = union(enum) { options: Todo, remove_dir: Todo, run: Run, - top_level: Todo, + top_level: TopLevel, translate_c: Todo, update_source_files: Todo, write_file: Todo, @@ -119,6 +119,20 @@ pub const Extended = union(enum) { std.debug.panic("TODO implement another step type (index {d})", .{step_index}); } }; + + pub const TopLevel = struct { + pub fn make( + top_level: *TopLevel, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, + ) Step.ExtendedMakeError!void { + _ = top_level; + _ = step_index; + _ = maker; + _ = progress_node; + } + }; }; pub const State = enum { -- 2.54.0 From c6d37f389591e722f59ca7f2b719ec8cfc0a9984 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 17 Mar 2026 19:05:13 -0700 Subject: [PATCH 050/179] configurer: make string duplication also intern I had this idea to make b.dupe() also intern the strings since they will be ultimately serialized to Configuration. Unfortunately the idea does not work, because although a process-lived arena is used for the string_bytes ArrayList of the Configuration.Wip, when the ArrayList is resized, Allocator.free() memsets the freed memory to undefined, even though it still technically lives due to being in a process-scoped arena. So this commit will need to be partially reverted. However, I kept it for posterity, and there are some more changes which I will now note below. - dupePaths: don't rewrite backslashes to forward slashes. backslashes are valid in filenames on non-windows systems. - always compile configurer in single-threaded mode - use arena allocator for everything, no gpa for anything - construct the Configuration.Wip instance earlier, so some stuff can be prepopulated as desired. - don't forget to flush --- BRANCH_TODO | 2 +- lib/compiler/configurer.zig | 46 +++++++------- lib/std/Build.zig | 75 +++++++++++------------ lib/std/Build/Configuration.zig | 7 --- lib/std/Build/Module.zig | 7 ++- lib/std/Build/Step/Compile.zig | 66 ++++++++++---------- lib/std/Build/Step/Run.zig | 100 ++++++++++++++++++------------- lib/std/Build/Step/WriteFile.zig | 30 +++++----- src/main.zig | 1 + 9 files changed, 173 insertions(+), 161 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 5ddc34a0996823b21a2c81e92ee6d834eca7577a..29cafe75c4fd452484ee7391716f58d363061ce3 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,10 +1,10 @@ +* remove Cache from configurer * implement the build options * don't forget to add -listen arg back * get zig init template working * finish migrating the rest of the build steps * make zig-pkg path root configurable in maker (make sure --system still works) * eliminate calls to getPath, getPath2, getPath3 -* replace b.dupe() with string internment * solve the TODOs added in this branch * get zig tests passing * test a bunch of third party projects / help people migrate diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 816f71158c909cb1a45a24ccc6aaf160a1529113..ad59f57f910bfaa2e8d6971d708c4101e2618af2 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -24,31 +24,22 @@ pub const std_options: std.Options = .{ }; pub fn main(init: process.Init.Minimal) !void { - // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not - // always the case. So, we do need a true gpa for some things. - var debug_gpa_state: std.heap.DebugAllocator(.{ - // We'd rather have `zig build` run faster than catch harmless leaks in - // the user's build.zig script. - .stack_trace_frames = 0, - }) = .init; - defer _ = debug_gpa_state.deinit(); - const gpa = debug_gpa_state.allocator(); + var arena_allocator: std.heap.ArenaAllocator = .init(std.heap.page_allocator); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); - var threaded: std.Io.Threaded = .init(gpa, .{ + // The configurer is always short-lived because all it does is serialize + // the configuration, which is picked up by a separate maker process. + var threaded: std.Io.Threaded = .init(arena, .{ .environ = init.environ, .argv0 = .init(init.args), }); defer threaded.deinit(); const io = threaded.io(); - // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. - var arena_allocator: std.heap.ArenaAllocator = .init(std.heap.page_allocator); - defer arena_allocator.deinit(); - const arena = arena_allocator.allocator(); - const args = try init.args.toSlice(arena); - // skip my own exe name + // Skip own executable name. var arg_idx: usize = 1; const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); @@ -84,7 +75,7 @@ pub fn main(init: process.Init.Minimal) !void { .arena = arena, .cache = .{ .io = io, - .gpa = gpa, + .gpa = arena, .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), .cwd = try process.currentPathAlloc(io, arena), }, @@ -97,7 +88,18 @@ pub fn main(init: process.Init.Minimal) !void { .result = try std.zig.system.resolveTargetQuery(io, .{}), }, .generated_files = .empty, + + // Created before running the user's configure script so that some things + // can be added during script execution such as strings. + // + // Use of arena here is load-bearing because `std.Build.dupe` is + // implemented by string internment, and then returning the interned + // slice. When the string bytes array is reallocated, that reference + // must stay alive. + .wip_configuration = .init(arena), }; + assert(try graph.wip_configuration.addString("") == .empty); + assert(try graph.wip_configuration.addString("root") == .root); graph.cache.addPrefix(.{ .path = null, .handle = cwd }); graph.cache.addPrefix(build_root_directory); @@ -200,19 +202,15 @@ pub fn main(init: process.Init.Minimal) !void { fatal(" access the help menu with 'zig build -h'", .{}); } - var wc: Configuration.Wip = .init(gpa); - defer wc.deinit(); - assert(try wc.addString("") == .empty); - assert(try wc.addString("root") == .root); - - try serializeSystemIntegrationOptions(&graph, &wc); + try serializeSystemIntegrationOptions(&graph, &graph.wip_configuration); var stdout_buffer: [1024]u8 = undefined; var file_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); - serialize(builder, &wc, &file_writer.interface) catch |err| switch (err) { + serialize(builder, &graph.wip_configuration, &file_writer.interface) catch |err| switch (err) { error.WriteFailed => fatal("failed to write configuration output: {t}", .{file_writer.err.?}), error.OutOfMemory => |e| return e, }; + file_writer.flush() catch |err| fatal("failed to write configuration output: {t}", .{err}); // This executable is short-lived and run in Debug mode, so we'd rather // have `zig build` run faster than catch resource leaks in the user's diff --git a/lib/std/Build.zig b/lib/std/Build.zig index cfe9dfa946c2cedfa8981460673181ee807eb966..bc9e1e98a0bc6fbb42a54847f7b4a0ab1558d9cb 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -115,11 +115,37 @@ pub const Graph = struct { /// Indexes correspond to `Configuration.GeneratedFileIndex`. generated_files: std.ArrayList(*Step), + wip_configuration: Configuration.Wip, pub fn addGeneratedFile(graph: *Graph, owner: *Step) Configuration.GeneratedFileIndex { graph.generated_files.append(graph.arena, owner) catch @panic("OOM"); return @enumFromInt(graph.generated_files.items.len - 1); } + + pub fn dupeString(graph: *Graph, bytes: []const u8) [:0]const u8 { + // This code assumes the `Configuration.Wip` uses arena allocation such + // that references to string_bytes never die even when the ArrayList is + // reallocated. + const wc = &graph.wip_configuration; + const i = wc.addString(bytes) catch @panic("OOM"); + return wc.string_bytes.items[@intFromEnum(i)..][0..bytes.len :0]; + } + + pub fn dupePath(graph: *Graph, bytes: []const u8) [:0]const u8 { + if (builtin.os.tag != .windows) return dupeString(graph, bytes); + const arena = graph.arena; + const the_copy = arena.dupe(u8, bytes) catch @panic("OOM"); + defer arena.free(the_copy); + mem.replaceScalar(u8, the_copy, '/', '\\'); + return dupeString(graph, the_copy); + } + + pub fn dupeStrings(graph: *Graph, strings: []const []const u8) []const []const u8 { + const arena = graph.arena; + const array = arena.alloc([]const u8, strings.len) catch @panic("OOM"); + for (array, strings) |*dest, source| dest.* = dupeString(graph, source); + return array; + } }; const AvailableDeps = []const struct { []const u8, []const u8 }; @@ -869,36 +895,18 @@ pub fn addConfigHeader( return config_header_step; } -/// Allocator.dupe without the need to handle out of memory. -pub fn dupe(b: *Build, bytes: []const u8) []u8 { - return dupeInner(b.allocator, bytes); -} - -pub fn dupeInner(allocator: Allocator, bytes: []const u8) []u8 { - return allocator.dupe(u8, bytes) catch @panic("OOM"); +pub fn dupe(b: *Build, bytes: []const u8) [:0]const u8 { + return b.graph.dupeString(bytes); } /// Duplicates an array of strings without the need to handle out of memory. -pub fn dupeStrings(b: *Build, strings: []const []const u8) [][]u8 { - const array = b.allocator.alloc([]u8, strings.len) catch @panic("OOM"); - for (array, strings) |*dest, source| dest.* = b.dupe(source); - return array; +pub fn dupeStrings(b: *Build, strings: []const []const u8) []const []const u8 { + return b.graph.dupeStrings(strings); } -/// Duplicates a path and converts all slashes to the OS's canonical path separator. -pub fn dupePath(b: *Build, bytes: []const u8) []u8 { - return dupePathInner(b.allocator, bytes); -} - -fn dupePathInner(allocator: Allocator, bytes: []const u8) []u8 { - const the_copy = dupeInner(allocator, bytes); - for (the_copy) |*byte| { - switch (byte.*) { - '/', '\\' => byte.* = fs.path.sep, - else => {}, - } - } - return the_copy; +/// Duplicates a path, canonicalizing path separators. +pub fn dupePath(b: *Build, bytes: []const u8) [:0]const u8 { + return b.graph.dupePath(bytes); } pub fn addWriteFile(b: *Build, file_path: []const u8, data: []const u8) *Step.WriteFile { @@ -2268,25 +2276,18 @@ pub const LazyPath = union(enum) { /// /// The `b` parameter is only used for its allocator. All *Build instances /// share the same allocator. - pub fn dupe(lazy_path: LazyPath, b: *Build) LazyPath { - return lazy_path.dupeInner(b.allocator); - } - - fn dupeInner(lazy_path: LazyPath, allocator: Allocator) LazyPath { + pub fn dupe(lazy_path: LazyPath, graph: *Graph) LazyPath { return switch (lazy_path) { - .src_path => |sp| .{ .src_path = .{ - .owner = sp.owner, - .sub_path = sp.owner.dupePath(sp.sub_path), - } }, - .cwd_relative => |p| .{ .cwd_relative = dupePathInner(allocator, p) }, + .src_path => |sp| .{ .src_path = .{ .owner = sp.owner, .sub_path = sp.owner.dupePath(sp.sub_path) } }, + .cwd_relative => |p| .{ .cwd_relative = graph.dupePath(p) }, .generated => |gen| .{ .generated = .{ .index = gen.index, .up = gen.up, - .sub_path = dupePathInner(allocator, gen.sub_path), + .sub_path = graph.dupePath(gen.sub_path), } }, .dependency => |dep| .{ .dependency = .{ .dependency = dep.dependency, - .sub_path = dupePathInner(allocator, dep.sub_path), + .sub_path = graph.dupePath(dep.sub_path), } }, }; } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 7c4f06562d0f262d818b9bdaecb95f44802138c3..47fe0009a63177e2a71de0d603fe204ce5043f8a 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1419,13 +1419,6 @@ pub const Path = extern struct { global_cache, build_root, }; - - pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path { - _ = c; - _ = arena; - _ = path; - @panic("TODO"); - } }; pub const InstallDestDir = enum(u32) { diff --git a/lib/std/Build/Module.zig b/lib/std/Build/Module.zig index 22100e1b2e4ea08c9972eeaee78c555e24b0f43f..1c271023efad2c4458f5185d88620a692b7581ae 100644 --- a/lib/std/Build/Module.zig +++ b/lib/std/Build/Module.zig @@ -240,13 +240,14 @@ pub fn init( owner: *std.Build, value: union(enum) { options: CreateOptions, existing: *const Module }, ) void { - const allocator = owner.allocator; + const graph = owner.graph; + const arena = graph.arena; switch (value) { .options => |options| { m.* = .{ .owner = owner, - .root_source_file = if (options.root_source_file) |lp| lp.dupe(owner) else null, + .root_source_file = if (options.root_source_file) |lp| lp.dupe(graph) else null, .import_table = .empty, .resolved_target = options.target, .optimize = options.optimize, @@ -277,7 +278,7 @@ pub fn init( .no_builtin = options.no_builtin, }; - m.import_table.ensureUnusedCapacity(allocator, options.imports.len) catch @panic("OOM"); + m.import_table.ensureUnusedCapacity(arena, options.imports.len) catch @panic("OOM"); for (options.imports) |dep| { m.import_table.putAssumeCapacity(dep.name, dep.module); } diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 8773baf1b2bde5c050e42331f845bdbe585a554e..f16c4ee83b2e38b89577cd435b99294846d8621b 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -296,10 +296,10 @@ pub const HeaderInstallation = union(enum) { source: LazyPath, dest_rel_path: []const u8, - pub fn dupe(file: File, b: *std.Build) File { + pub fn dupe(file: File, graph: *std.Build.Graph) File { return .{ - .source = file.source.dupe(b), - .dest_rel_path = b.dupePath(file.dest_rel_path), + .source = file.source.dupe(graph), + .dest_rel_path = graph.dupePath(file.dest_rel_path), }; } }; @@ -424,13 +424,13 @@ pub fn create(owner: *std.Build, options: Options) *Compile { }; if (options.zig_lib_dir) |lp| { - compile.zig_lib_dir = lp.dupe(compile.step.owner); + compile.zig_lib_dir = lp.dupe(graph); lp.addStepDependencies(&compile.step); } if (options.test_runner) |runner| { compile.test_runner = .{ - .path = runner.path.dupe(compile.step.owner), + .path = runner.path.dupe(graph), .mode = runner.mode, }; runner.path.addStepDependencies(&compile.step); @@ -440,20 +440,20 @@ pub fn create(owner: *std.Build, options: Options) *Compile { // gets embedded, so for any other target the manifest file is just ignored. if (target.ofmt == .coff) { if (options.win32_manifest) |lp| { - compile.win32_manifest = lp.dupe(compile.step.owner); + compile.win32_manifest = lp.dupe(graph); lp.addStepDependencies(&compile.step); } if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { // Building a Win32 DLL, check for win32 .def file. if (options.win32_module_definition) |lp| { - compile.win32_module_definition = lp.dupe(compile.step.owner); + compile.win32_module_definition = lp.dupe(graph); lp.addStepDependencies(&compile.step); } } } if (options.entitlements) |lp| { - compile.entitlements = lp.dupe(compile.step.owner); + compile.entitlements = lp.dupe(graph); lp.addStepDependencies(&compile.step); } @@ -464,12 +464,13 @@ pub fn create(owner: *std.Build, options: Options) *Compile { /// When a module links with this artifact, all headers marked for installation are added to that /// module's include search path. pub fn installHeader(cs: *Compile, source: LazyPath, dest_rel_path: []const u8) void { - const b = cs.step.owner; + const graph = cs.step.owner.graph; + const arena = graph.arena; const installation: HeaderInstallation = .{ .file = .{ - .source = source.dupe(b), - .dest_rel_path = b.dupePath(dest_rel_path), + .source = source.dupe(graph), + .dest_rel_path = graph.dupePath(dest_rel_path), } }; - cs.installed_headers.append(b.allocator, installation) catch @panic("OOM"); + cs.installed_headers.append(arena, installation) catch @panic("OOM"); cs.addHeaderInstallationToIncludeTree(installation); installation.getSource().addStepDependencies(&cs.step); } @@ -483,13 +484,14 @@ pub fn installHeadersDirectory( dest_rel_path: []const u8, options: HeaderInstallation.Directory.Options, ) void { - const b = cs.step.owner; + const graph = cs.step.owner.graph; + const arena = graph.arena; const installation: HeaderInstallation = .{ .directory = .{ - .source = source.dupe(b), - .dest_rel_path = b.dupePath(dest_rel_path), - .options = options.dupe(b), + .source = source.dupe(graph), + .dest_rel_path = graph.dupePath(dest_rel_path), + .options = options.dupe(graph), } }; - cs.installed_headers.append(b.allocator, installation) catch @panic("OOM"); + cs.installed_headers.append(arena, installation) catch @panic("OOM"); cs.addHeaderInstallationToIncludeTree(installation); installation.getSource().addStepDependencies(&cs.step); } @@ -506,9 +508,10 @@ pub fn installConfigHeader(cs: *Compile, config_header: *Step.ConfigHeader) void /// module's include search path. pub fn installLibraryHeaders(cs: *Compile, lib: *Compile) void { assert(lib.kind == .lib); - const arena = cs.owner.allocator; + const graph = cs.step.owner.graph; + const arena = graph.arena; for (lib.installed_headers.items) |installation| { - const installation_copy = installation.dupe(lib.step.owner); + const installation_copy = installation.dupe(graph); cs.installed_headers.append(arena, installation_copy) catch @panic("OOM"); cs.addHeaderInstallationToIncludeTree(installation_copy); installation_copy.getSource().addStepDependencies(&cs.step); @@ -556,21 +559,21 @@ pub fn addObjCopy(cs: *Compile, options: Step.ObjCopy.Options) *Step.ObjCopy { } pub fn setLinkerScript(compile: *Compile, source: LazyPath) void { - const b = compile.step.owner; - compile.linker_script = source.dupe(b); + const graph = compile.step.owner.graph; + compile.linker_script = source.dupe(graph); source.addStepDependencies(&compile.step); } pub fn setVersionScript(compile: *Compile, source: LazyPath) void { - const b = compile.step.owner; - compile.version_script = source.dupe(b); + const graph = compile.step.owner.graph; + compile.version_script = source.dupe(graph); source.addStepDependencies(&compile.step); } pub fn forceUndefinedSymbol(compile: *Compile, symbol_name: []const u8) void { - const b = compile.step.owner; - const arena = b.allocator; - compile.force_undefined_symbols.put(arena, b.dupe(symbol_name), {}) catch @panic("OOM"); + const graph = compile.step.owner.graph; + const arena = graph.allocator; + compile.force_undefined_symbols.put(arena, graph.dupeString(symbol_name), {}) catch @panic("OOM"); } /// Returns whether the library, executable, or object depends on a particular system library. @@ -655,9 +658,9 @@ pub fn setVerboseCC(compile: *Compile, value: bool) void { } pub fn setLibCFile(compile: *Compile, libc_file: ?LazyPath) void { - const b = compile.step.owner; + const graph = compile.step.owner.graph; if (libc_file) |f| { - compile.libc_file = f.dupe(b); + compile.libc_file = f.dupe(graph); f.addStepDependencies(&compile.step); } else { compile.libc_file = null; @@ -733,11 +736,12 @@ pub fn getEmittedLlvmBc(compile: *Compile) LazyPath { } pub fn setExecCmd(compile: *Compile, args: []const ?[]const u8) void { - const b = compile.step.owner; + const graph = compile.step.owner.graph; + const arena = graph.arena; assert(compile.kind == .@"test"); - const duped_args = b.allocator.alloc(?[]u8, args.len) catch @panic("OOM"); + const duped_args = arena.alloc(?[]u8, args.len) catch @panic("OOM"); for (args, 0..) |arg, i| { - duped_args[i] = if (arg) |a| b.dupe(a) else null; + duped_args[i] = if (arg) |a| graph.dupeString(a) else null; } compile.exec_cmd_args = duped_args; } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index e51be09242ca074ecfaa5bec5369620c32a03ae2..f55074109c946da6f7cbae12417524ce040d9e25 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -139,7 +139,7 @@ pub const Arg = union(enum) { lazy_path: PrefixedLazyPath, decorated_directory: DecoratedLazyPath, file_content: PrefixedLazyPath, - bytes: []u8, + bytes: [:0]const u8, output_file: *Output, output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. @@ -228,13 +228,14 @@ pub fn addArtifactArg(run: *Run, artifact: *Step.Compile) void { } pub fn addPrefixedArtifactArg(run: *Run, prefix: []const u8, artifact: *Step.Compile) void { - const b = run.step.owner; + const graph = run.step.owner.graph; + const arena = graph.arena; const prefixed_artifact: PrefixedArtifact = .{ - .prefix = b.dupe(prefix), + .prefix = graph.dupeString(prefix), .artifact = artifact, }; - run.argv.append(b.allocator, .{ .artifact = prefixed_artifact }) catch @panic("OOM"); + run.argv.append(arena, .{ .artifact = prefixed_artifact }) catch @panic("OOM"); const bin_file = artifact.getEmittedBin(); bin_file.addStepDependencies(&run.step); @@ -279,8 +280,8 @@ pub fn addPrefixedOutputFileArg( const output = arena.create(Output) catch @panic("OOM"); output.* = .{ - .prefix = b.dupe(prefix), - .basename = b.dupe(basename), + .prefix = graph.dupeString(prefix), + .basename = graph.dupeString(basename), .generated_file = graph.addGeneratedFile(&run.step), }; run.argv.append(arena, .{ .output_file = output }) catch @panic("OOM"); @@ -318,13 +319,14 @@ pub fn addFileArg(run: *Run, lp: std.Build.LazyPath) void { /// * `addFileArg` - same thing but without the prefix /// * `addOutputFileArg` - for files generated by the child process pub fn addPrefixedFileArg(run: *Run, prefix: []const u8, lp: std.Build.LazyPath) void { - const b = run.step.owner; + const graph = run.step.owner.graph; + const arena = graph.arena; const prefixed_file_source: PrefixedLazyPath = .{ - .prefix = b.dupe(prefix), - .lazy_path = lp.dupe(b), + .prefix = graph.dupeString(prefix), + .lazy_path = lp.dupe(graph), }; - run.argv.append(b.allocator, .{ .lazy_path = prefixed_file_source }) catch @panic("OOM"); + run.argv.append(arena, .{ .lazy_path = prefixed_file_source }) catch @panic("OOM"); lp.addStepDependencies(&run.step); } @@ -365,7 +367,8 @@ pub fn addFileContentArg(run: *Run, lp: std.Build.LazyPath) void { /// Related: /// * `addFileContentArg` - same thing but without the prefix pub fn addPrefixedFileContentArg(run: *Run, prefix: []const u8, lp: std.Build.LazyPath) void { - const b = run.step.owner; + const graph = run.step.owner.graph; + const arena = graph.arena; // Some parts of this step's configure phase API rely on the first argument being somewhat // transparent/readable, but the content of the file specified by `lp` remains completely @@ -375,10 +378,10 @@ pub fn addPrefixedFileContentArg(run: *Run, prefix: []const u8, lp: std.Build.La } const prefixed_file_source: PrefixedLazyPath = .{ - .prefix = b.dupe(prefix), - .lazy_path = lp.dupe(b), + .prefix = graph.dupeString(prefix), + .lazy_path = lp.dupe(graph), }; - run.argv.append(b.allocator, .{ .file_content = prefixed_file_source }) catch @panic("OOM"); + run.argv.append(arena, .{ .file_content = prefixed_file_source }) catch @panic("OOM"); lp.addStepDependencies(&run.step); } @@ -415,18 +418,19 @@ pub fn addPrefixedOutputDirectoryArg( basename: []const u8, ) std.Build.LazyPath { if (basename.len == 0) @panic("basename must not be empty"); - const b = run.step.owner; + const graph = run.step.owner.graph; + const arena = graph.arena; - const output = b.allocator.create(Output) catch @panic("OOM"); + const output = arena.create(Output) catch @panic("OOM"); output.* = .{ - .prefix = b.dupe(prefix), - .basename = b.dupe(basename), + .prefix = graph.dupeString(prefix), + .basename = graph.dupeString(basename), .generated_file = .{ .step = &run.step }, }; - run.argv.append(b.allocator, .{ .output_directory = output }) catch @panic("OOM"); + run.argv.append(arena, .{ .output_directory = output }) catch @panic("OOM"); if (run.rename_step_with_output_arg) { - run.setName(b.fmt("{s} ({s})", .{ run.step.name, basename })); + run.setName(std.fmt.allocPrint(arena, "{s} ({s})", .{ run.step.name, basename }) catch @panic("OOM")); } return .{ .generated = .{ .file = &output.generated_file } }; @@ -437,10 +441,11 @@ pub fn addDirectoryArg(run: *Run, lazy_directory: std.Build.LazyPath) void { } pub fn addPrefixedDirectoryArg(run: *Run, prefix: []const u8, lazy_directory: std.Build.LazyPath) void { - const b = run.step.owner; - run.argv.append(b.allocator, .{ .decorated_directory = .{ - .prefix = b.dupe(prefix), - .lazy_path = lazy_directory.dupe(b), + const graph = run.step.owner.graph; + const arena = graph.arena; + run.argv.append(arena, .{ .decorated_directory = .{ + .prefix = graph.dupeString(prefix), + .lazy_path = lazy_directory.dupe(graph), .suffix = "", } }) catch @panic("OOM"); lazy_directory.addStepDependencies(&run.step); @@ -452,11 +457,12 @@ pub fn addDecoratedDirectoryArg( lazy_directory: std.Build.LazyPath, suffix: []const u8, ) void { - const b = run.step.owner; - run.argv.append(b.allocator, .{ .decorated_directory = .{ - .prefix = b.dupe(prefix), - .lazy_path = lazy_directory.dupe(b), - .suffix = b.dupe(suffix), + const graph = run.step.owner.graph; + const arena = graph.arena; + run.argv.append(arena, .{ .decorated_directory = .{ + .prefix = graph.dupeString(prefix), + .lazy_path = lazy_directory.dupe(graph), + .suffix = graph.dupeString(suffix), } }) catch @panic("OOM"); lazy_directory.addStepDependencies(&run.step); } @@ -479,8 +485,8 @@ pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []co const dep_file = arena.create(Output) catch @panic("OOM"); dep_file.* = .{ - .prefix = b.dupe(prefix), - .basename = b.dupe(basename), + .prefix = graph.dupeString(prefix), + .basename = graph.dupeString(basename), .generated_file = graph.addGeneratedFile(&run.step), }; @@ -492,8 +498,9 @@ pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []co } pub fn addArg(run: *Run, arg: []const u8) void { - const b = run.step.owner; - run.argv.append(b.allocator, .{ .bytes = b.dupe(arg) }) catch @panic("OOM"); + const graph = run.step.owner.graph; + const arena = graph.arena; + run.argv.append(arena, .{ .bytes = graph.dupeString(arg) }) catch @panic("OOM"); } pub fn addArgs(run: *Run, args: []const []const u8) void { @@ -509,8 +516,9 @@ pub fn setStdIn(run: *Run, stdin: StdIn) void { } pub fn setCwd(run: *Run, cwd: Build.LazyPath) void { + const graph = run.step.owner.graph; cwd.addStepDependencies(&run.step); - run.cwd = cwd.dupe(run.step.owner); + run.cwd = cwd.dupe(graph); } pub fn clearEnvironment(run: *Run) void { @@ -580,24 +588,28 @@ pub fn removeEnvironmentVariable(run: *Run, key: []const u8) void { /// Adds a check for exact stderr match. Does not add any other checks. pub fn expectStdErrEqual(run: *Run, bytes: []const u8) void { - run.addCheck(.{ .expect_stderr_exact = run.step.owner.dupe(bytes) }); + const graph = run.step.owner.graph; + run.addCheck(.{ .expect_stderr_exact = graph.dupeString(bytes) }); } pub fn expectStdErrMatch(run: *Run, bytes: []const u8) void { - run.addCheck(.{ .expect_stderr_match = run.step.owner.dupe(bytes) }); + const graph = run.step.owner.graph; + run.addCheck(.{ .expect_stderr_match = graph.dupeString(bytes) }); } /// Adds a check for exact stdout match as well as a check for exit code 0, if /// there is not already an expected termination check. pub fn expectStdOutEqual(run: *Run, bytes: []const u8) void { - run.addCheck(.{ .expect_stdout_exact = run.step.owner.dupe(bytes) }); + const graph = run.step.owner.graph; + run.addCheck(.{ .expect_stdout_exact = graph.dupeString(bytes) }); if (!run.hasTermCheck()) run.expectExitCode(0); } /// Adds a check for stdout match as well as a check for exit code 0, if there /// is not already an expected termination check. pub fn expectStdOutMatch(run: *Run, bytes: []const u8) void { - run.addCheck(.{ .expect_stdout_match = run.step.owner.dupe(bytes) }); + const graph = run.step.owner.graph; + run.addCheck(.{ .expect_stdout_match = graph.dupeString(bytes) }); if (!run.hasTermCheck()) run.expectExitCode(0); } @@ -641,7 +653,7 @@ pub fn captureStdErr(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPa captured.* = .{ .output = .{ .prefix = "", - .basename = if (options.basename) |basename| b.dupe(basename) else "stderr", + .basename = if (options.basename) |basename| graph.dupeString(basename) else "stderr", .generated_file = graph.addGeneratedFile(&run.step), }, .trim_whitespace = options.trim_whitespace, @@ -664,7 +676,7 @@ pub fn captureStdOut(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPa captured.* = .{ .output = .{ .prefix = "", - .basename = if (options.basename) |basename| b.dupe(basename) else "stdout", + .basename = if (options.basename) |basename| graph.dupeString(basename) else "stdout", .generated_file = graph.addGeneratedFile(&run.step), }, .trim_whitespace = options.trim_whitespace, @@ -678,7 +690,9 @@ pub fn captureStdOut(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPa /// If the Run step is determined to have side-effects, the Run step is always /// executed when it appears in the build graph, regardless of whether this /// file has been modified. -pub fn addFileInput(self: *Run, file_input: std.Build.LazyPath) void { - file_input.addStepDependencies(&self.step); - self.file_inputs.append(self.step.owner.allocator, file_input.dupe(self.step.owner)) catch @panic("OOM"); +pub fn addFileInput(run: *Run, file_input: std.Build.LazyPath) void { + const graph = run.step.owner.graph; + const arena = graph.arena; + file_input.addStepDependencies(&run.step); + run.file_inputs.append(arena, file_input.dupe(graph)) catch @panic("OOM"); } diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 14097a76edff48beab3a94510f888f6e5ad0bc0e..3e269ded6d5ffaf40da4606cce7f867bcfbc84c2 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -55,10 +55,10 @@ pub const Directory = struct { /// `exclude_extensions` takes precedence over `include_extensions`. include_extensions: ?[]const []const u8 = null, - pub fn dupe(opts: Options, b: *std.Build) Options { + pub fn dupe(opts: Options, graph: *std.Build.Graph) Options { return .{ - .exclude_extensions = b.dupeStrings(opts.exclude_extensions), - .include_extensions = if (opts.include_extensions) |incs| b.dupeStrings(incs) else null, + .exclude_extensions = graph.dupeStrings(opts.exclude_extensions), + .include_extensions = if (opts.include_extensions) |incs| graph.dupeStrings(incs) else null, }; } @@ -103,13 +103,13 @@ pub fn create(owner: *std.Build) *WriteFile { } pub fn add(write_file: *WriteFile, sub_path: []const u8, bytes: []const u8) std.Build.LazyPath { - const b = write_file.step.owner; - const gpa = b.allocator; - const file = File{ - .sub_path = b.dupePath(sub_path), - .contents = .{ .bytes = b.dupe(bytes) }, + const graph = write_file.step.owner.graph; + const arena = graph.arena; + const file: File = .{ + .sub_path = graph.dupePath(sub_path), + .contents = .{ .bytes = graph.dupeString(bytes) }, }; - write_file.files.append(gpa, file) catch @panic("OOM"); + write_file.files.append(arena, file) catch @panic("OOM"); write_file.maybeUpdateName(); return .{ .generated = .{ @@ -154,14 +154,14 @@ pub fn addCopyDirectory( sub_path: []const u8, options: Directory.Options, ) std.Build.LazyPath { - const b = write_file.step.owner; - const gpa = b.allocator; + const graph = write_file.step.owner.graph; + const arena = graph.arena; const dir = Directory{ - .source = source.dupe(b), - .sub_path = b.dupePath(sub_path), - .options = options.dupe(b), + .source = source.dupe(graph), + .sub_path = graph.dupePath(sub_path), + .options = options.dupe(graph), }; - write_file.directories.append(gpa, dir) catch @panic("OOM"); + write_file.directories.append(arena, dir) catch @panic("OOM"); write_file.maybeUpdateName(); source.addStepDependencies(&write_file.step); diff --git a/src/main.zig b/src/main.zig index f0590f27de92b34b8e733dd0948af6457ebe9c09..8978bb9d58b99034715e8207d0eb6a8a80a61e9a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5356,6 +5356,7 @@ fn cmdBuild( .cc_argv = &.{}, .inherited = .{ .resolved_target = resolved_target, + .single_threaded = true, }, .global = config, .parent = null, -- 2.54.0 From e436d9c4ad623d8facefb8e1637b26e63e900277 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 17 Mar 2026 19:12:22 -0700 Subject: [PATCH 051/179] configurer: back out the string interning from prev commit partial revert of 2d3fbb687fba1ed52b42998ac4dcbf2a042644ea - see its commit message for reasoning --- lib/std/Build.zig | 20 +++++++------------- lib/std/Build/Step/Run.zig | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index bc9e1e98a0bc6fbb42a54847f7b4a0ab1558d9cb..8772361c71a7f2c6190ea1c5cd10effd236ccc03 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -122,22 +122,16 @@ pub const Graph = struct { return @enumFromInt(graph.generated_files.items.len - 1); } - pub fn dupeString(graph: *Graph, bytes: []const u8) [:0]const u8 { - // This code assumes the `Configuration.Wip` uses arena allocation such - // that references to string_bytes never die even when the ArrayList is - // reallocated. - const wc = &graph.wip_configuration; - const i = wc.addString(bytes) catch @panic("OOM"); - return wc.string_bytes.items[@intFromEnum(i)..][0..bytes.len :0]; + pub fn dupeString(graph: *Graph, bytes: []const u8) []const u8 { + return graph.arena.dupe(u8, bytes) catch @panic("OOM"); } - pub fn dupePath(graph: *Graph, bytes: []const u8) [:0]const u8 { - if (builtin.os.tag != .windows) return dupeString(graph, bytes); + pub fn dupePath(graph: *Graph, bytes: []const u8) []const u8 { const arena = graph.arena; + if (builtin.os.tag != .windows) return graph.arena.dupe(u8, bytes) catch @panic("OOM"); const the_copy = arena.dupe(u8, bytes) catch @panic("OOM"); - defer arena.free(the_copy); mem.replaceScalar(u8, the_copy, '/', '\\'); - return dupeString(graph, the_copy); + return the_copy; } pub fn dupeStrings(graph: *Graph, strings: []const []const u8) []const []const u8 { @@ -895,7 +889,7 @@ pub fn addConfigHeader( return config_header_step; } -pub fn dupe(b: *Build, bytes: []const u8) [:0]const u8 { +pub fn dupe(b: *Build, bytes: []const u8) []const u8 { return b.graph.dupeString(bytes); } @@ -905,7 +899,7 @@ pub fn dupeStrings(b: *Build, strings: []const []const u8) []const []const u8 { } /// Duplicates a path, canonicalizing path separators. -pub fn dupePath(b: *Build, bytes: []const u8) [:0]const u8 { +pub fn dupePath(b: *Build, bytes: []const u8) []const u8 { return b.graph.dupePath(bytes); } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index f55074109c946da6f7cbae12417524ce040d9e25..bfb540a0eec2f32cceb9e44c47ac49801e3f81d3 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -139,7 +139,7 @@ pub const Arg = union(enum) { lazy_path: PrefixedLazyPath, decorated_directory: DecoratedLazyPath, file_content: PrefixedLazyPath, - bytes: [:0]const u8, + bytes: []const u8, output_file: *Output, output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. -- 2.54.0 From 8bc09132121d751760b332a30a82c3dfa343a48a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 17 Mar 2026 23:02:59 -0700 Subject: [PATCH 052/179] compiler: fix compilation errors --- lib/std/Build/Configuration.zig | 7 +++++++ src/Compilation.zig | 6 +++++- src/main.zig | 5 ++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 47fe0009a63177e2a71de0d603fe204ce5043f8a..7c4f06562d0f262d818b9bdaecb95f44802138c3 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1419,6 +1419,13 @@ pub const Path = extern struct { global_cache, build_root, }; + + pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path { + _ = c; + _ = arena; + _ = path; + @panic("TODO"); + } }; pub const InstallDestDir = enum(u32) { diff --git a/src/Compilation.zig b/src/Compilation.zig index 870ac3f79893b9301c8daa075276c10237bce75f..5445710ea4a7f01d2175306417b4e3eb179e1362 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1747,9 +1747,13 @@ pub const CreateOptions = struct { .no => return null, .yes_cache => { assert(opts.cache_mode != .none); + const target = &opts.root_mod.resolved_target.result; return try ea.cacheName(arena, .{ .root_name = opts.root_name, - .target = &opts.root_mod.resolved_target.result, + .cpu_arch = target.cpu.arch, + .os_tag = target.os.tag, + .ofmt = target.ofmt, + .abi = target.abi, .output_mode = opts.config.output_mode, .link_mode = opts.config.link_mode, .version = opts.version, diff --git a/src/main.zig b/src/main.zig index 8978bb9d58b99034715e8207d0eb6a8a80a61e9a..f774eb522fecf9c9dd093be05d3a4d80dbb4d5c2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3380,7 +3380,10 @@ fn buildOutputType( .pch => try std.fmt.allocPrint(arena, "{s}.pch", .{root_name}), else => try std.zig.binNameAlloc(arena, .{ .root_name = root_name, - .target = target, + .cpu_arch = target.cpu.arch, + .os_tag = target.os.tag, + .ofmt = target.ofmt, + .abi = target.abi, .output_mode = create_module.resolved_options.output_mode, .link_mode = create_module.resolved_options.link_mode, .version = optional_version, -- 2.54.0 From 81ee4ab32c8617af3ce9690d562131a6a0924f1c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 19 Mar 2026 02:24:33 -0700 Subject: [PATCH 053/179] configurer: serialize all data from run steps --- BRANCH_TODO | 2 + lib/compiler/configurer.zig | 244 +++++++++++++++++++++++++++++--- lib/std/Build/Configuration.zig | 161 ++++++++++++++++----- 3 files changed, 354 insertions(+), 53 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 29cafe75c4fd452484ee7391716f58d363061ce3..fa99fb66653e47384ed96a4081735819cb5cf247 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,3 +1,5 @@ +* make more stuff use IndexType +* make addExtra return Index using reflection * remove Cache from configurer * implement the build options * don't forget to add -listen arg back diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index ad59f57f910bfaa2e8d6971d708c4101e2618af2..e37d392bd1e137e872d8e4ce5fdd3f421e20abe9 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -351,6 +351,169 @@ const Serialize = struct { }))); } + fn addEnvironMap(s: *Serialize, opt_map: ?*std.process.Environ.Map) !?Configuration.EnvironMap.Index { + const wc = s.wc; + const map = opt_map orelse return null; + return @enumFromInt(try wc.addDeduped(@as(Configuration.EnvironMap, .{ + .keys = try wc.addStringList(map.array_hash_map.keys()), + .values = try wc.addStringList(map.array_hash_map.values()), + }))); + } + + fn initArgsList(s: *Serialize, args: []const Step.Run.Arg) ![]const Configuration.Step.Run.Arg.Index { + const wc = s.wc; + const result = try s.arena.alloc(Configuration.Step.Run.Arg.Index, args.len); + for (result, args) |*dest, src| { + dest.* = @enumFromInt(try wc.addExtra(@as(Configuration.Step.Run.Arg, switch (src) { + .artifact => |a| .{ + .flags = .{ + .tag = .artifact, + .prefix = a.prefix.len != 0, + .suffix = false, + .basename = false, + .path = false, + .producer = true, + .generated = false, + .dep_file = false, + }, + .prefix = .{ .value = try s.addOptionalString(a.prefix) }, + .suffix = .{ .value = null }, + .basename = .{ .value = null }, + .path = .{ .value = null }, + .producer = .{ .value = stepIndex(s, &a.artifact.step) }, + .generated = .{ .value = null }, + }, + .lazy_path => |a| .{ + .flags = .{ + .tag = .path_file, + .prefix = a.prefix.len != 0, + .suffix = false, + .basename = false, + .path = true, + .producer = false, + .generated = false, + .dep_file = false, + }, + .prefix = .{ .value = try s.addOptionalString(a.prefix) }, + .suffix = .{ .value = null }, + .basename = .{ .value = null }, + .path = .{ .value = try addLazyPath(s, a.lazy_path) }, + .producer = .{ .value = null }, + .generated = .{ .value = null }, + }, + .decorated_directory => |a| .{ + .flags = .{ + .tag = .path_directory, + .prefix = a.prefix.len != 0, + .suffix = a.suffix.len != 0, + .basename = false, + .path = true, + .producer = false, + .generated = false, + .dep_file = false, + }, + .prefix = .{ .value = try addOptionalString(s, a.prefix) }, + .suffix = .{ .value = try addOptionalString(s, a.suffix) }, + .basename = .{ .value = null }, + .path = .{ .value = try addLazyPath(s, a.lazy_path) }, + .producer = .{ .value = null }, + .generated = .{ .value = null }, + }, + .file_content => |a| .{ + .flags = .{ + .tag = .file_content, + .prefix = a.prefix.len != 0, + .suffix = false, + .basename = false, + .path = true, + .producer = false, + .generated = false, + .dep_file = false, + }, + .prefix = .{ .value = try addOptionalString(s, a.prefix) }, + .suffix = .{ .value = null }, + .basename = .{ .value = null }, + .path = .{ .value = try addLazyPath(s, a.lazy_path) }, + .producer = .{ .value = null }, + .generated = .{ .value = null }, + }, + .bytes => |a| .{ + .flags = .{ + .tag = .string, + .prefix = true, + .suffix = false, + .basename = false, + .path = false, + .producer = false, + .generated = false, + .dep_file = false, + }, + .prefix = .{ .value = try addOptionalString(s, a) }, + .suffix = .{ .value = null }, + .basename = .{ .value = null }, + .path = .{ .value = null }, + .producer = .{ .value = null }, + .generated = .{ .value = null }, + }, + .output_file => |a| .{ + .flags = .{ + .tag = .output_file, + .prefix = a.prefix.len != 0, + .suffix = false, + .basename = a.basename.len != 0, + .path = false, + .producer = false, + .generated = true, + .dep_file = false, + }, + .prefix = .{ .value = try addOptionalString(s, a.prefix) }, + .suffix = .{ .value = null }, + .basename = .{ .value = try addOptionalString(s, a.basename) }, + .path = .{ .value = null }, + .producer = .{ .value = null }, + .generated = .{ .value = a.generated_file }, + }, + .output_directory => |a| .{ + .flags = .{ + .tag = .output_directory, + .prefix = a.prefix.len != 0, + .suffix = false, + .basename = a.basename.len != 0, + .path = false, + .producer = false, + .generated = true, + .dep_file = false, + }, + .prefix = .{ .value = try addOptionalString(s, a.prefix) }, + .suffix = .{ .value = null }, + .basename = .{ .value = try addOptionalString(s, a.basename) }, + .path = .{ .value = null }, + .producer = .{ .value = null }, + .generated = .{ .value = a.generated_file }, + }, + .cli_rest_positionals => .{ + .flags = .{ + .tag = .cli_rest_positionals, + .prefix = false, + .suffix = false, + .basename = false, + .path = false, + .producer = false, + .generated = false, + .dep_file = false, + }, + .prefix = .{ .value = null }, + .suffix = .{ .value = null }, + .basename = .{ .value = null }, + .path = .{ .value = null }, + .producer = .{ .value = null }, + .generated = .{ .value = null }, + }, + }))); + } + return result; + } + fn initLazyPathList(s: *Serialize, list: []const std.Build.LazyPath) ![]const Configuration.LazyPath.Index { const result = try s.arena.alloc(Configuration.LazyPath.Index, list.len); for (result, list) |*dest, src| dest.* = try addLazyPath(s, src); @@ -768,16 +931,33 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .update_source_files => @panic("TODO"), .run => e: { const run: *Step.Run = @fieldParentPtr("step", step); - - const captured_stdout: Configuration.OptionalString = if (run.captured_stdout) |cs| - .init(try wc.addString(cs.output.basename)) - else - .none; - - const captured_stderr: Configuration.OptionalString = if (run.captured_stderr) |cs| - .init(try wc.addString(cs.output.basename)) - else - .none; + var expect_stderr_exact: ?Configuration.Bytes = null; + var expect_stdout_exact: ?Configuration.Bytes = null; + var expect_stderr_match: std.ArrayList(Configuration.Bytes) = .empty; + var expect_stdout_match: std.ArrayList(Configuration.Bytes) = .empty; + var expect_term: ?struct { + status: Configuration.Step.Run.ExpectTermStatus, + value: u32, + } = null; + switch (run.stdio) { + .check => |checks| for (checks.items) |check| switch (check) { + .expect_stderr_exact => |bytes| expect_stderr_exact = try wc.addBytes(bytes), + .expect_stdout_exact => |bytes| expect_stdout_exact = try wc.addBytes(bytes), + .expect_stderr_match => |bytes| { + try expect_stderr_match.append(arena, try wc.addBytes(bytes)); + }, + .expect_stdout_match => |bytes| { + try expect_stdout_match.append(arena, try wc.addBytes(bytes)); + }, + .expect_term => |t| expect_term = switch (t) { + .exited => |x| .{ .status = .exited, .value = x }, + .signal => |x| .{ .status = .signal, .value = @intFromEnum(x) }, + .stopped => |x| .{ .status = .stopped, .value = x }, + .unknown => |x| .{ .status = .unknown, .value = x }, + }, + }, + else => {}, + } const extra_index = try wc.addExtra(@as(Configuration.Step.Run, .{ .flags = .{ @@ -802,16 +982,44 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .stderr_trim_whitespace = if (run.captured_stderr) |cs| cs.trim_whitespace else .none, .stdio_limit = run.stdio_limit != .unlimited, .producer = run.producer != null, + .cwd = run.cwd != null, + .captured_stdout = run.captured_stdout != null, + .captured_stderr = run.captured_stderr != null, + .environ_map = run.environ_map != null, }, - .file_inputs_len = @intCast(run.file_inputs.items.len), - .args_len = @intCast(run.argv.items.len), - .cwd = try s.addOptionalLazyPathEnum(run.cwd), - .captured_stdout = captured_stdout, - .captured_stderr = captured_stderr, + .flags2 = .{ + .expect_stderr_exact = expect_stderr_exact != null, + .expect_stdout_exact = expect_stdout_exact != null, + .expect_stderr_match = expect_stderr_match.items.len != 0, + .expect_stdout_match = expect_stdout_match.items.len != 0, + .expect_term = expect_term != null, + .expect_term_status = if (expect_term) |t| t.status else .exited, + }, + .file_inputs = .{ .slice = try s.initLazyPathList(run.file_inputs.items) }, + .args = .{ .slice = try s.initArgsList(run.argv.items) }, + .cwd = .{ .value = try s.addOptionalLazyPath(run.cwd) }, + .captured_stdout = .{ .value = if (run.captured_stdout) |cs| .{ + .basename = try wc.addString(cs.output.basename), + .generated_file = cs.output.generated_file, + } else null }, + .captured_stderr = .{ .value = if (run.captured_stderr) |cs| .{ + .basename = try wc.addString(cs.output.basename), + .generated_file = cs.output.generated_file, + } else null }, + .environ_map = .{ .value = try s.addEnvironMap(run.environ_map) }, + .expect_term_value = .{ .value = if (expect_term) |t| t.value else null }, + .stdio_limit = .{ .value = run.stdio_limit.toInt() }, + .producer = .{ .value = if (run.producer) |cs| s.stepIndex(&cs.step) else null }, + .expect_stderr_exact = .{ .value = if (expect_stderr_exact) |bytes| bytes else null }, + .expect_stdout_exact = .{ .value = if (expect_stdout_exact) |bytes| bytes else null }, + .expect_stderr_match = .{ .slice = expect_stderr_match.items }, + .expect_stdout_match = .{ .slice = expect_stdout_match.items }, + .stdin = .{ .u = switch (run.stdin) { + .none => .none, + .bytes => |bytes| .{ .bytes = try wc.addBytes(bytes) }, + .lazy_path => |lp| .{ .lazy_path = try s.addLazyPath(lp) }, + } }, })); - - log.err("TODO serialize the trailing Run step data", .{}); - break :e @enumFromInt(extra_index); }, .check_file => @panic("TODO"), diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 7c4f06562d0f262d818b9bdaecb95f44802138c3..df72da06cf39d693a3024ba83582353f2ba9784f 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -188,6 +188,18 @@ pub const Wip = struct { return .init(try addString(wip, bytes orelse return .none)); } + pub fn addStringList(wip: *Wip, list: []const []const u8) Allocator.Error!StringList { + _ = wip; + _ = list; + @panic("TODO"); + } + + pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes { + _ = wip; + _ = bytes; + @panic("TODO"); + } + pub fn addSemVer(wip: *Wip, sv: std.SemanticVersion) Allocator.Error!String { var buffer: [256]u8 = undefined; var writer: std.Io.Writer = .fixed(&buffer); @@ -500,52 +512,65 @@ pub const Step = extern struct { }; }; - /// Trailing: - /// * LazyPath.Index for each file_inputs_len - /// * Arg for each args_len - /// * environ_map if corresponding flag is set - /// * stdin: Bytes, // if StdIn.bytes is chosen - /// * stdin: LazyPath.Index, // if StdIn.lazy_path is chosen - /// * checks: Checks, // if StdIo.check is chosen - /// * stdio_limit: u64, // if stdio_limit is set - /// * producer: Step.Index, // if producer is set. always compile step pub const Run = struct { flags: @This().Flags, - file_inputs_len: u32, - args_len: u32, - cwd: LazyPath.OptionalIndex, - captured_stdout: OptionalString, // basename - captured_stderr: OptionalString, // basename + flags2: Flags2, + args: Storage.LengthPrefixedList(Arg.Index), + cwd: Storage.FlagOptional(.flags, .cwd, LazyPath.Index), + captured_stdout: Storage.FlagOptional(.flags, .captured_stdout, CapturedStream), + captured_stderr: Storage.FlagOptional(.flags, .captured_stderr, CapturedStream), + file_inputs: Storage.LengthPrefixedList(LazyPath.Index), + stdio_limit: Storage.FlagOptional(.flags, .stdio_limit, u64), + /// Always a compile step. + producer: Storage.FlagOptional(.flags, .producer, Step.Index), + /// First half is keys, second half is values. + environ_map: Storage.FlagOptional(.flags, .environ_map, EnvironMap.Index), + stdin: Storage.FlagUnion(.flags, .stdin, StdIn), + expect_stderr_exact: Storage.FlagOptional(.flags2, .expect_stderr_exact, Bytes), + expect_stdout_exact: Storage.FlagOptional(.flags2, .expect_stdout_exact, Bytes), + expect_stderr_match: Storage.FlagLengthPrefixedList(.flags2, .expect_stderr_match, Bytes), + expect_stdout_match: Storage.FlagLengthPrefixedList(.flags2, .expect_stdout_match, Bytes), + expect_term_value: Storage.FlagOptional(.flags2, .expect_term, u32), + + pub const CapturedStream = extern struct { + generated_file: GeneratedFileIndex, + basename: String, + }; - /// Trailing: - /// * String if prefix set - /// * String if suffix set - /// * String if basename set - /// * Step.Index which is always a compile step if tag is artifact - /// * LazyPath.Index if tag is path_file, path_directory, or file_content pub const Arg = struct { - flags: Arg.Flags, + flags: @This().Flags, + prefix: Storage.FlagOptional(.flags, .prefix, String), + suffix: Storage.FlagOptional(.flags, .suffix, String), + basename: Storage.FlagOptional(.flags, .basename, String), + path: Storage.FlagOptional(.flags, .path, LazyPath.Index), + /// Always a compile step. + producer: Storage.FlagOptional(.flags, .producer, Step.Index), + generated: Storage.FlagOptional(.flags, .generated, GeneratedFileIndex), pub const Flags = packed struct(u32) { tag: Arg.Tag, prefix: bool, suffix: bool, basename: bool, - /// Implies Tag is output_file + path: bool, + producer: bool, + generated: bool, dep_file: bool, - _: u20 = 0, + _: u22 = 0, }; - pub const Tag = enum(u8) { + pub const Tag = enum(u3) { artifact, path_file, path_directory, + string, file_content, - bytes, output_file, output_directory, cli_rest_positionals, }; + + pub const Index = IndexType(@This()); }; pub const Color = enum(u4) { @@ -562,26 +587,47 @@ pub const Step = extern struct { manual, }; - pub const StdIn = enum(u2) { none, bytes, lazy_path }; + pub const StdIn = union(@This().Tag) { + none: void, + bytes: Bytes, + lazy_path: LazyPath.Index, + + pub const Tag = enum(u2) { none, bytes, lazy_path }; + }; pub const TrimWhitespace = enum(u2) { none, all, leading, trailing }; pub const StdIo = enum(u2) { infer_from_args, inherit, check, zig_test }; + pub const ExpectTermStatus = enum(u2) { exited, signal, stopped, unknown }; + pub const Flags = packed struct(u32) { tag: Tag = .run, - disable_zig_progress: bool, skip_foreign_checks: bool, failing_to_execute_foreign_is_an_error: bool, has_side_effects: bool, test_runner_mode: bool, color: Color, - stdin: StdIn, + stdin: StdIn.Tag, stdio: StdIo, stdout_trim_whitespace: TrimWhitespace, stderr_trim_whitespace: TrimWhitespace, stdio_limit: bool, producer: bool, - _: u8 = 0, + cwd: bool, + captured_stdout: bool, + captured_stderr: bool, + environ_map: bool, + _: u4 = 0, + }; + + pub const Flags2 = packed struct(u32) { + expect_stderr_exact: bool, + expect_stdout_exact: bool, + expect_stderr_match: bool, + expect_stdout_match: bool, + expect_term: bool, + expect_term_status: ExpectTermStatus, + _: u25 = 0, }; }; @@ -1395,17 +1441,37 @@ pub const Deps = struct { }; }; +pub const EnvironMap = struct { + keys: StringList, + values: StringList, + + pub const Index = IndexType(@This()); +}; + /// Points into `extra`, where the first element is count of strings, following /// elements is `String` per count. /// /// Stored identically to `Deps`. +pub const StringList = enum(u32) { + _, + + pub fn slice(this: @This(), c: *const Configuration) []const String { + const len = c.extra[@intFromEnum(this)]; + return @ptrCast(c.extra[@intFromEnum(this) + 1 ..][0..len]); + } +}; + pub const OptionalStringList = enum(u32) { none = max_u32, _, - pub fn slice(osl: OptionalStringList, c: *const Configuration) ?[]const String { - const len = c.extra[@intFromEnum(osl)]; - return @ptrCast(c.extra[@intFromEnum(osl) + 1 ..][0..len]); + pub fn unwrap(this: @This()) ?StringList { + if (this == .none) return null; + return @enumFromInt(@intFromEnum(this)); + } + + pub fn slice(this: @This(), c: *const Configuration) ?[]const String { + return (unwrap(this) orelse return null).slice(c); } }; @@ -1499,6 +1565,13 @@ pub const String = enum(u32) { } }; +/// Arbitrary sequence of bytes that may contain null bytes. +pub const Bytes = extern struct { + /// Points into `string_bytes`. + index: u32, + len: u32, +}; + pub const DefaultingBool = enum(u2) { false, true, @@ -2359,7 +2432,11 @@ pub const Storage = enum { }, }, }, - .@"extern" => comptime unreachable, + .@"extern" => { + const n = @divExact(@sizeOf(Field), @sizeOf(u32)); + defer i.* += n; + return @bitCast(buffer[i.*..][0..n].*); + }, }, else => comptime unreachable, } @@ -2404,7 +2481,7 @@ pub const Storage = enum { inline else => |v| extraFieldLen(v), }, }, - .@"extern" => comptime unreachable, + .@"extern" => @divExact(@sizeOf(Field), @sizeOf(u32)), }, else => @compileError("bad type: " ++ @typeName(Field)), }; @@ -2520,13 +2597,27 @@ pub const Storage = enum { }, }, }, - .@"extern" => comptime unreachable, + .@"extern" => { + const n = @divExact(@sizeOf(Field), @sizeOf(u32)); + buffer[i..][0..n].* = @bitCast(value); + return n; + }, }, else => @compileError("bad field type: " ++ @typeName(Field)), } } }; +fn IndexType(comptime T: type) type { + return enum(u32) { + _, + + pub fn get(this: @This(), c: *const Configuration) T { + return extraData(c, T, @intFromEnum(this)); + } + }; +} + pub fn extraData(c: *const Configuration, comptime T: type, index: usize) T { var i: usize = index; return Storage.data(c.extra, &i, T); -- 2.54.0 From a399d37886bfb8358e2f93744f0dd3f59542dcee Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 22 Mar 2026 21:42:33 -0700 Subject: [PATCH 054/179] maker: upgrade some of the run step logic --- BRANCH_TODO | 1 + lib/compiler/Maker.zig | 2 + lib/compiler/Maker/Fuzz.zig | 2 +- lib/compiler/Maker/Step/Run.zig | 277 +++++++++++++++++++------------- lib/std/Build/Cache.zig | 5 +- lib/std/Build/Configuration.zig | 2 + 6 files changed, 173 insertions(+), 116 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index fa99fb66653e47384ed96a4081735819cb5cf247..8eacc3b61a8f8fff25ae3a792dcbf0161a640198 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -18,5 +18,6 @@ * link_eh_frame_hdr should be DefaultingBool * make --foo, --no-foo CLI args uniform (make them -f args instead) * install steps should provide generated files for installed things, then delete the run step hack + - but artifact install steps also add paths for dyn libs on windows diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 4e71912ef93292f3f64394e7252c34f8aa5463ff..2b8b163447787f52ce150f4f586c8e9a9cdc0534 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -35,6 +35,7 @@ install_paths: InstallPaths, scanned_config: *const ScannedConfig, steps: []Step, generated_files: []Path, +run_args: ?[]const []const u8, available_rss: usize, max_rss_is_default: bool, @@ -534,6 +535,7 @@ pub fn main(init: process.Init.Minimal) !void { }, .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), .generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len), + .run_args = run_args, .available_rss = max_rss, .max_rss_is_default = false, diff --git a/lib/compiler/Maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig index 77fab2b2df6dfe80bb4c4ae4dea33871992782fd..73d1447966f349b8bb1f09a41f2ec5518c33d17e 100644 --- a/lib/compiler/Maker/Fuzz.zig +++ b/lib/compiler/Maker/Fuzz.zig @@ -203,7 +203,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { const graph = owner.graph; const io = graph.io; - run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) { + run.rerunInFuzzMode(run, fuzz, fuzz.prog_node) catch |err| switch (err) { error.MakeFailed => { var buf: [256]u8 = undefined; const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) { diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index b621a3cf50608ce15ccaca0cfdfe19b58f9d227c..4b7644f06bd440f5d6430caf51525e8b84b7ec1a 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -12,6 +12,7 @@ const Path = std.Build.Cache.Path; const assert = std.debug.assert; const mem = std.mem; const process = std.process; +const allocPrint = std.fmt.allocPrint; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); @@ -26,115 +27,158 @@ cached_test_metadata: ?CachedTestMetadata = null, /// executable that contains fuzz tests. rebuilt_executable: ?Path = null, +/// Persisted to reuse memory on subsequent calls to `make`. +argv: std.ArrayList([]const u8) = .empty, +/// Persisted to reuse memory on subsequent calls to `make`. +output_placeholders: std.ArrayList(IndexedOutput) = .empty, + pub fn make( run: *Run, - step_index: Configuration.Step.Index, + run_index: Configuration.Step.Index, maker: *Maker, progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { - if (true) @panic("TODO implement run.make()"); const graph = maker.graph; - const step = maker.stepByIndex(step_index); + const gpa = maker.gpa; + const step = maker.stepByIndex(run_index); const io = graph.io; const arena = graph.arena; // TODO don't leak into the process arena - const has_side_effects = run.hasSideEffects(); + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; + const argv_list = &run.argv; + const output_placeholders = &run.output_placeholders; - var argv_list = std.array_list.Managed([]const u8).init(arena); - var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); + argv_list.clearRetainingCapacity(); + output_placeholders.clearRetainingCapacity(); var man = graph.cache.obtain(); defer man.deinit(); - if (run.environ_map) |environ_map| { - for (environ_map.keys(), environ_map.values()) |key, value| { - man.hash.addBytes(key); - man.hash.addBytes(value); + if (conf_run.environ_map.value) |environ_map_index| { + const environ_map = environ_map_index.get(conf); + for (environ_map.keys.slice(conf), environ_map.values.slice(conf)) |key, value| { + man.hash.addBytesZ(key.slice(conf)); + man.hash.addBytesZ(value.slice(conf)); } } - man.hash.add(run.color); - man.hash.add(run.disable_zig_progress); + man.hash.add(conf_run.flags.color); + man.hash.add(conf_run.flags.disable_zig_progress); - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(bytes); - man.hash.addBytes(bytes); + for (conf_run.args.slice) |arg_index| { + const arg = arg_index.get(conf); + try argv_list.ensureUnusedCapacity(gpa, 1); + switch (arg.flags.tag) { + .string => { + const prefix = arg.prefix.value.?.slice(conf); + argv_list.appendAssumeCapacity(prefix); + man.hash.addBytesZ(prefix); }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(graph, step); - try argv_list.append(graph.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(maker, file_path) })); - man.hash.addBytes(file.prefix); + .path_file => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); + argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ + prefix, try convertPathArg(run_index, maker, file_path), suffix, + })); + man.hash.addBytesZ(prefix); + man.hash.addBytesZ(suffix); _ = try man.addFilePath(file_path, null); }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(graph, step); - const resolved_arg = graph.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(maker, file_path), dd.suffix }); - try argv_list.append(resolved_arg); + .path_directory => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); + const resolved_arg = try mem.concat(arena, u8, &.{ + prefix, try convertPathArg(run_index, maker, file_path), suffix, + }); + argv_list.appendAssumeCapacity(resolved_arg); man.hash.addBytes(resolved_arg); }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(graph, step); + .file_content => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + result.writer.writeAll(prefix) catch return error.OutOfMemory; - const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { - return step.fail( - "unable to open input file '{f}': {t}", - .{ file_path, err }, - ); - }; + const file = file_path.root_dir.handle.openFile(io, file_path.sub_path, .{}) catch |err| + return step.fail(maker, "unable to open input file {f}: {t}", .{ file_path, err }); defer file.close(io); - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); + var file_reader = file.reader(io, &.{}); _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return step.fail( - "failed to read from '{f}': {t}", - .{ file_path, file_reader.err.? }, - ), + error.ReadFailed => switch (file_reader.err.?) { + error.Canceled => |e| return e, + else => |e| return step.fail(maker, "failed to read from {f}: {t}", .{ file_path, e }), + }, error.WriteFailed => return error.OutOfMemory, }; + result.writer.writeAll(suffix) catch return error.OutOfMemory; - try argv_list.append(result.written()); - man.hash.addBytes(file_plp.prefix); + argv_list.appendAssumeCapacity(result.written()); + man.hash.addBytesZ(prefix); + man.hash.addBytesZ(suffix); _ = try man.addFilePath(file_path, null); }, - .artifact => |pa| { - const artifact = pa.artifact; + .artifact => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const producer_index = arg.producer.value.?; + const producer_step = producer_index.ptr(conf); + const producer = producer_step.extended.get(conf.extra).compile; + const root_module = producer.root_module.get(conf); + const root_module_target = root_module.resolved_target.get(conf).?.result.get(conf); + const os_tag = root_module_target.flags.os_tag.unwrap().?; - if (artifact.rootModuleTarget().os.tag == .windows) { + if (true) @panic("TODO"); + + if (os_tag == .windows) { // On Windows we don't have rpaths so we have to add .dll search paths to PATH - addPathForDynLibs(artifact); + addPathForDynLibs(producer_index); } - const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; + const file_path = producer_index.installed_path orelse producer_index.generated_bin.?.path.?; - try argv_list.append(graph.fmt("{s}{s}", .{ - pa.prefix, - run.convertPathArg(maker, .{ .root_dir = .cwd(), .sub_path = file_path }), + argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ + prefix, + try convertPathArg(run_index, maker, .{ .root_dir = .cwd(), .sub_path = file_path }), + suffix, })); _ = try man.addFile(file_path, null); }, - .output_file, .output_directory => |output| { - man.hash.addBytes(output.prefix); - man.hash.addBytes(output.basename); + .output_file, .output_directory => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const basename = arg.basename.value.?.slice(conf); + + man.hash.addBytesZ(prefix); + man.hash.addBytesZ(basename); + man.hash.addBytesZ(suffix); + // Add a placeholder into the argument list because we need the // manifest hash to be updated with all arguments before the // object directory is computed. - try output_placeholders.append(.{ - .index = argv_list.items.len, - .tag = arg, - .output = output, + try output_placeholders.append(gpa, .{ + .index = @intCast(argv_list.items.len), + .arg_index = arg_index, }); - _ = try argv_list.addOne(); + argv_list.items.len += 1; + }, + .cli_rest_positionals => { + if (maker.run_args) |run_args| { + try argv_list.appendSlice(gpa, run_args); + for (run_args) |s| man.hash.addBytes(s); + } }, } } - switch (run.stdin) { + if (true) @panic("TODO"); + + switch (conf_run.stdin.u) { .bytes => |bytes| { man.hash.addBytes(bytes); }, @@ -145,28 +189,30 @@ pub fn make( .none => {}, } - if (run.captured_stdout) |captured| { + if (conf_run.captured_stdout) |captured| { man.hash.addBytes(captured.output.basename); man.hash.add(captured.trim_whitespace); } - if (run.captured_stderr) |captured| { + if (conf_run.captured_stderr) |captured| { man.hash.addBytes(captured.output.basename); man.hash.add(captured.trim_whitespace); } std.log.err("TODO hashStdIo", .{}); - //hashStdIo(&man.hash, run.stdio); + //hashStdIo(&man.hash, conf_run.stdio); - for (run.file_inputs.items) |lazy_path| { + for (conf_run.file_inputs.items) |lazy_path| { _ = try man.addFile(lazy_path.getPath2(graph, step), null); } - if (run.cwd) |cwd| { + if (conf_run.cwd) |cwd| { const cwd_path = cwd.getPath3(graph, step); _ = man.hash.addBytes(try cwd_path.toString(arena)); } + const has_side_effects = conf_run.flags.has_side_effects; + if (!has_side_effects and try step.cacheHitAndWatch(&man)) { // cache hit, skip running command const digest = man.final(); @@ -182,7 +228,7 @@ pub fn make( return; } - const dep_output_file = run.dep_output_file orelse { + const dep_output_file = conf_run.dep_output_file orelse { // We already know the final output paths, use them directly. const digest = if (has_side_effects) man.hash.final() @@ -205,18 +251,18 @@ pub fn make( else => unreachable, }; graph.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {t}", .{ + return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ graph.cache_root, output_sub_dir_path, err, }); }; - const arg_output_path = run.convertPathArg(maker, .{ + const arg_output_path = try convertPathArg(run_index, maker, .{ .root_dir = .cwd(), .sub_path = placeholder.output.generated_file.getPath(), }); argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) arg_output_path else - graph.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); + try allocPrint(arena, "{s}{s}", .{ placeholder.output.prefix, arg_output_path }); } try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); @@ -238,7 +284,7 @@ pub fn make( else => unreachable, }; graph.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {t}", .{ + return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ graph.cache_root, output_sub_dir_path, err, }); }; @@ -247,9 +293,9 @@ pub fn make( .sub_path = graph.pathJoin(&output_components), }; placeholder.output.generated_file.path = raw_output_path.toString(arena) catch @panic("OOM"); - argv_list.items[placeholder.index] = graph.fmt("{s}{s}", .{ + argv_list.items[placeholder.index] = try mem.concat(arena, u8, .{ placeholder.output.prefix, - run.convertPathArg(maker, raw_output_path), + try convertPathArg(run_index, maker, raw_output_path), }); } @@ -268,7 +314,7 @@ pub fn make( man.final(); const any_output = output_placeholders.items.len > 0 or - run.captured_stdout != null or run.captured_stderr != null; + conf_run.captured_stdout != null or conf_run.captured_stderr != null; // Rename into place if (any_output) { @@ -277,17 +323,17 @@ pub fn make( graph.cache_root.handle.rename(tmp_dir_path, graph.cache_root.handle, o_sub_path, io) catch |err| switch (err) { Dir.RenameError.DirNotEmpty => { graph.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { - return step.fail("unable to remove dir '{f}'{s}: {t}", .{ + return step.fail(maker, "unable to remove dir '{f}'{s}: {t}", .{ graph.cache_root, tmp_dir_path, del_err, }); }; graph.cache_root.handle.rename(tmp_dir_path, graph.cache_root.handle, o_sub_path, io) catch |retry_err| { - return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ + return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ graph.cache_root, tmp_dir_path, graph.cache_root, o_sub_path, retry_err, }); }; }, - else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ + else => return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ graph.cache_root, tmp_dir_path, graph.cache_root, o_sub_path, err, }), }; @@ -309,6 +355,7 @@ pub fn make( /// * The wait fails, indicating the child closed stdout and stderr fn waitZigTest( run: *Run, + maker: *Maker, child: *process.Child, options: Step.MakeOptions, multi_reader: *Io.File.MultiReader, @@ -412,6 +459,7 @@ fn waitZigTest( switch (header.tag) { .zig_version => { if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( + maker, "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", .{ builtin.zig_version_string, body }, ); @@ -1028,6 +1076,7 @@ const StdioPollEnum = enum { stdout, stderr }; fn evalZigTest( run: *Run, + maker: *Maker, spawn_options: process.SpawnOptions, options: Step.MakeOptions, fuzz_context: ?FuzzContext, @@ -1102,7 +1151,7 @@ fn evalZigTest( // The individual unit test results are irrelevant: the test runner itself broke! // Fail immediately without populating `s.test_results`. - return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); + return run.step.fail(maker, "unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); }, .no_poll => |no_poll| { // This might be a success (we requested exit and the child dutifully closed stdout) or @@ -1141,7 +1190,7 @@ fn evalZigTest( if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { // The individual unit test results are irrelevant: the test runner itself broke! // Fail immediately without populating `s.test_results`. - return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); + return run.step.fail(maker, "test process unexpectedly {f}", .{fmtTerm(term)}); } // We're done with all of the tests! Commit the test results and return. @@ -1181,7 +1230,7 @@ fn evalZigTest( run.step.result_stderr = try arena.dupe(u8, stderr); // The individual unit test results in `results` are irrelevant: the test runner // is broken! Fail immediately without populating `s.test_results`. - return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); + return run.step.fail(maker, "test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); }, } comptime unreachable; @@ -1313,7 +1362,7 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E switch (run.stdin) { .bytes => |bytes| { child.stdin.?.writeStreamingAll(io, bytes) catch |err| { - return run.step.fail("unable to write stdin: {t}", .{err}); + return run.step.fail(maker, "unable to write stdin: {t}", .{err}); }; child.stdin.?.close(io); child.stdin = null; @@ -1321,7 +1370,7 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E .lazy_path => |lazy_path| { const path = lazy_path.getPath3(graph, &run.step); const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { - return run.step.fail("unable to open stdin file: {t}", .{err}); + return run.step.fail(maker, "unable to open stdin file: {t}", .{err}); }; defer file.close(io); // TODO https://github.com/ziglang/zig/issues/23955 @@ -1330,15 +1379,15 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E var write_buffer: [1024]u8 = undefined; var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { - error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ + error.ReadFailed => return run.step.fail(maker, "failed to read from {f}: {t}", .{ path, file_reader.err.?, }), - error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ + error.WriteFailed => return run.step.fail(maker, "failed to write to stdin: {t}", .{ stdin_writer.err.?, }), }; stdin_writer.interface.flush() catch |err| switch (err) { - error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ + error.WriteFailed => return run.step.fail(maker, "failed to write to stdin: {t}", .{ stdin_writer.err.?, }), }; @@ -1418,15 +1467,13 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E } const IndexedOutput = struct { - index: usize, - tag: Configuration.Step.Run.Arg.Tag, - output: *Output, + index: u32, + arg_index: Configuration.Step.Run.Arg.Index, }; -const Output = void; // TODO - pub fn rerunInFuzzMode( run: *Run, + run_index: Configuration.Step.Index, fuzz: *std.Build.Fuzz, prog_node: std.Progress.Node, ) !void { @@ -1444,11 +1491,11 @@ pub fn rerunInFuzzMode( }, .lazy_path => |file| { const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(maker, file_path) })); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, convertPathArg(run_index, maker, file_path) })); }, .decorated_directory => |dd| { const file_path = dd.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(maker, file_path), dd.suffix })); + try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, convertPathArg(run_index, maker, file_path), dd.suffix })); }, .file_content => |file_plp| { const file_path = file_plp.lazy_path.getPath3(b, step); @@ -1477,7 +1524,7 @@ pub fn rerunInFuzzMode( }; try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, - run.convertPathArg(maker, .{ .root_dir = .cwd(), .sub_path = file_path }), + convertPathArg(run_index, maker, .{ .root_dir = .cwd(), .sub_path = file_path }), })); }, .output_file, .output_directory => unreachable, @@ -1675,7 +1722,7 @@ fn runCommand( const host_dl = graph.host.result.dynamic_linker.get() orelse "(none)"; - return step.fail( + return step.fail(maker, \\the host system is unable to execute binaries from the target \\ because the host dynamic linker is '{s}', \\ while the target dynamic linker is '{s}'. @@ -1688,7 +1735,7 @@ fn runCommand( const host_name = try graph.host.result.zigTriple(b.allocator); const foreign_name = try root_target.zigTriple(b.allocator); - return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ + return step.fail(maker, "the host system ({s}) is unable to execute binaries from the target ({s})", .{ host_name, foreign_name, }); }, @@ -1706,12 +1753,12 @@ fn runCommand( break :term spawnChildAndCollect(run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; if (e == error.MakeFailed) return error.MakeFailed; // error already reported - return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); + return step.fail(maker, "unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); }; } if (err == error.MakeFailed) return error.MakeFailed; // error already reported - return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); + return step.fail(maker, "failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); }; const generic_result = opt_generic_result orelse { @@ -1748,7 +1795,7 @@ fn runCommand( const sub_path = b.pathJoin(&output_components); const sub_path_dirname = Dir.path.dirname(sub_path).?; b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ + return step.fail(maker, "unable to make path '{f}{s}': {s}", .{ b.cache_root, sub_path_dirname, @errorName(err), }); }; @@ -1759,7 +1806,7 @@ fn runCommand( .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), }; b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { - return step.fail("unable to write file '{f}{s}': {s}", .{ + return step.fail(maker, "unable to write file '{f}{s}': {s}", .{ b.cache_root, sub_path, @errorName(err), }); }; @@ -1771,7 +1818,7 @@ fn runCommand( .check => |checks| for (checks.items) |check| switch (check) { .expect_stderr_exact => |expected_bytes| { if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { - return step.fail( + return step.fail(maker, \\========= expected this stderr: ========= \\{s} \\========= but found: ==================== @@ -1784,7 +1831,7 @@ fn runCommand( }, .expect_stderr_match => |match| { if (mem.find(u8, generic_result.stderr.?, match) == null) { - return step.fail( + return step.fail(maker, \\========= expected to find in stderr: ========= \\{s} \\========= but stderr does not contain it: ===== @@ -1797,7 +1844,7 @@ fn runCommand( }, .expect_stdout_exact => |expected_bytes| { if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { - return step.fail( + return step.fail(maker, \\========= expected this stdout: ========= \\{s} \\========= but found: ==================== @@ -1810,7 +1857,7 @@ fn runCommand( }, .expect_stdout_match => |match| { if (mem.find(u8, generic_result.stdout.?, match) == null) { - return step.fail( + return step.fail(maker, \\========= expected to find in stdout: ========= \\{s} \\========= but stdout does not contain it: ===== @@ -1823,7 +1870,7 @@ fn runCommand( }, .expect_term => |expected_term| { if (!termMatches(expected_term, generic_result.term)) { - return step.fail("process {f} (expected {f})", .{ + return step.fail(maker, "process {f} (expected {f})", .{ fmtTerm(generic_result.term), fmtTerm(expected_term), }); @@ -2069,21 +2116,23 @@ fn hasAnyOutputArgs(run: Run) bool { /// /// Whenever a path is included in the argv of a child, it should be put through this function first /// to make sure the child doesn't see paths relative to a cwd other than its own. -fn convertPathArg(run: *Run, maker: *Maker, path: Path) []const u8 { - const b = run.step.owner; +fn convertPathArg(run_index: Configuration.Step.Index, maker: *Maker, path: Path) ![]const u8 { + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; const graph = maker.graph; - const arena = graph.arena; + const arena = graph.arena; // TODO don't leak into process arena - const path_str = path.toString(arena) catch @panic("OOM"); + const path_str = try path.toString(arena); if (Dir.path.isAbsolute(path_str)) { // Absolute paths don't need changing. return path_str; } const child_cwd_rel: []const u8 = rel: { - const child_lazy_cwd = run.cwd orelse break :rel path_str; - const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); + const child_lazy_cwd = conf_run.cwd.value orelse break :rel path_str; + const child_cwd = try maker.resolveLazyPathIndexAbs(arena, child_lazy_cwd, run_index); // Convert it from relative to *our* cwd, to relative to the *child's* cwd. - break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); + break :rel try Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str); }; // Not every path can be made relative, e.g. if the path and the child cwd are on different // disk designators on Windows. In that case, `relative` will return an absolute path which we can @@ -2094,10 +2143,10 @@ fn convertPathArg(run: *Run, maker: *Maker, path: Path) []const u8 { // * On POSIX, the executable name cannot be a single component like 'foo' // * Some executables might treat a leading '-' like a flag, which we must avoid // There's no harm in it, so just *always* apply this prefix. - return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); + return Dir.path.join(arena, &.{ ".", child_cwd_rel }); } -fn addPathForDynLibs(artifact: *Step.Compile) void { +fn addPathForDynLibs(artifact: Configuration.Step.Index) void { if (true) @panic("TODO"); for (artifact.getCompileDependencies(true)) |compile| { if (compile.root_module.resolved_target.?.result.os.tag == .windows and @@ -2127,13 +2176,13 @@ fn failForeign( const host_name = try graph.host.result.zigTriple(process_arena); const foreign_name = try exe.rootModuleTarget().zigTriple(process_arena); - return step.fail( + return step.fail(maker, \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) \\ consider using {s} or enabling skip_foreign_checks in the Run step , .{ argv0, foreign_name, host_name, suggested_flag }); }, else => { - return step.fail("unable to spawn foreign binary '{s}'", .{argv0}); + return step.fail(maker, "unable to spawn foreign binary '{s}'", .{argv0}); }, } } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 3384e154cef1db389088441f7563ab17af44167f..c865cf6fc3566f0de87972ddee43dca000934c08 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -189,12 +189,15 @@ pub const File = struct { pub const HashHelper = struct { hasher: Hasher = hasher_init, - /// Record a slice of bytes as a dependency of the process being cached. pub fn addBytes(hh: *HashHelper, bytes: []const u8) void { hh.hasher.update(mem.asBytes(&bytes.len)); hh.hasher.update(bytes); } + pub fn addBytesZ(hh: *HashHelper, bytes: [:0]const u8) void { + hh.hasher.update(mem.absorbSentinel(u8, 0, bytes)); + } + pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void { hh.add(optional_bytes != null); hh.addBytes(optional_bytes orelse return); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index df72da06cf39d693a3024ba83582353f2ba9784f..9949969c39768da3d3d5a2d9ec9e397af3124665 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -561,8 +561,10 @@ pub const Step = extern struct { pub const Tag = enum(u3) { artifact, + /// `path` contains the file. path_file, path_directory, + /// `prefix` contains the string. string, file_content, output_file, -- 2.54.0 From 088f815031dd3570c9cc179e193cbd545e5bb3a3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 18 Apr 2026 22:19:48 -0700 Subject: [PATCH 055/179] there are some unresolved branch conflicts --- BRANCH_TODO | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/BRANCH_TODO b/BRANCH_TODO index 8eacc3b61a8f8fff25ae3a792dcbf0161a640198..bbd93aab5e54b3fd644009008b62fed693cf8120 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -21,3 +21,31 @@ - but artifact install steps also add paths for dyn libs on windows + +## Unresolved Branch Conflicts + +* move max_jobs to maker not configurer + ++// hack for stage2_x86_64 + coff ++generated_compiler_rt_dyn_lib: ?*GeneratedFile, + ++ // hack for stage2_x86_64 + coff ++ if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); + + if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { + const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); + all_cached = all_cached and p == .fresh; + } + + + .compiler_rt_dyn_lib_dir = switch (options.compiler_rt_dyn_lib_dir) { + .disabled => null, + .default => if (artifact.producesCompilerRtDynLib()) dest_dir else null, + .override => |o| o, + }, + + if (install_artifact.compiler_rt_dyn_lib_dir != null) install_artifact.emitted_compiler_rt_dyn_lib = artifact.getEmittedCompilerRtDynLib(); + + + -- 2.54.0 From 7c718fc72e23130b3093ccec7d644ee94017f3d6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 19 Apr 2026 10:56:24 -0700 Subject: [PATCH 056/179] fix compilation errors from rebase conflicts --- BRANCH_TODO | 30 ++------------------------ lib/compiler/Maker/Graph.zig | 3 +++ lib/compiler/configurer.zig | 2 +- lib/std/Build.zig | 10 --------- lib/std/Build/Cache.zig | 2 +- lib/std/Build/Step/InstallArtifact.zig | 4 ---- 6 files changed, 7 insertions(+), 44 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index bbd93aab5e54b3fd644009008b62fed693cf8120..eef38f57af77c0131f7bac2ec28ddb1636c6b132 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -13,6 +13,8 @@ * refactor with DefaultingEnum * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) * https://codeberg.org/ziglang/zig/issues/31397 +* restore the generated_compiler_rt_dyn_lib hack? +* run args ## Followup Issues * link_eh_frame_hdr should be DefaultingBool @@ -21,31 +23,3 @@ - but artifact install steps also add paths for dyn libs on windows - -## Unresolved Branch Conflicts - -* move max_jobs to maker not configurer - -+// hack for stage2_x86_64 + coff -+generated_compiler_rt_dyn_lib: ?*GeneratedFile, - -+ // hack for stage2_x86_64 + coff -+ if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); - - if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { - const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); - all_cached = all_cached and p == .fresh; - } - - - .compiler_rt_dyn_lib_dir = switch (options.compiler_rt_dyn_lib_dir) { - .disabled => null, - .default => if (artifact.producesCompilerRtDynLib()) dest_dir else null, - .override => |o| o, - }, - - if (install_artifact.compiler_rt_dyn_lib_dir != null) install_artifact.emitted_compiler_rt_dyn_lib = artifact.getEmittedCompilerRtDynLib(); - - - diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index 3be18927fddc5403a6333b672726c471931dcc96..50dba7f931137772a43423b29a4e430ccff06f38 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -48,6 +48,9 @@ sysroot: ?[]const u8 = null, search_prefixes: std.ArrayList([]const u8) = .empty, build_id: ?std.zig.BuildId = null, error_limit: ?u32 = null, +/// Steps should use `io` to limit the number of jobs, however in the case of +/// a single step spawning a fixed number of processes this can be used. +max_jobs: ?u32 = null, /// Intention of verbose is to print all sub-process command lines to stderr /// before spawning them. diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index e37d392bd1e137e872d8e4ce5fdd3f421e20abe9..538ff6481b2efbe13e232b0e7e9ddb0f11a6fe24 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -952,7 +952,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .expect_term => |t| expect_term = switch (t) { .exited => |x| .{ .status = .exited, .value = x }, .signal => |x| .{ .status = .signal, .value = @intFromEnum(x) }, - .stopped => |x| .{ .status = .stopped, .value = x }, + .stopped => |x| .{ .status = .stopped, .value = @intFromEnum(x) }, .unknown => |x| .{ .status = .unknown, .value = x }, }, }, diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 8772361c71a7f2c6190ea1c5cd10effd236ccc03..4e958a47faa437fdd342fdbf9d298d4d39a2e56f 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -32,13 +32,6 @@ allocator: Allocator, user_input_options: UserInputOptionsMap, available_options_map: AvailableOptionsMap, available_options_list: std.array_list.Managed(AvailableOption), -verbose: bool, -verbose_link: bool, -verbose_cc: bool, -verbose_air: bool, -verbose_llvm_ir: ?[]const u8, -verbose_llvm_bc: ?[]const u8, -verbose_llvm_cpu_features: bool, invalid_user_input: bool, default_step: *Step, top_level_steps: std.StringArrayHashMapUnmanaged(*Step.TopLevel), @@ -100,9 +93,6 @@ pub const Graph = struct { host: ResolvedTarget, dependency_cache: InitializedDepMap = .empty, allow_so_scripts: ?bool = null, - /// Steps should use `io` to limit the number of jobs, however in the case of - /// a single step spawning a fixed number of processes this can be used. - max_jobs: ?u32 = null, time_report: bool = false, /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also /// respects the '--color' flag. diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index c865cf6fc3566f0de87972ddee43dca000934c08..fc08080a976063388f90be8503d1df66f7d7cea1 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -195,7 +195,7 @@ pub const HashHelper = struct { } pub fn addBytesZ(hh: *HashHelper, bytes: [:0]const u8) void { - hh.hasher.update(mem.absorbSentinel(u8, 0, bytes)); + hh.hasher.update(mem.absorbSentinel(bytes)); } pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void { diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig index 4f4aa4301b622c8042d6428855452083137bca8e..a21dad2232c78f84d9f50a98ab2ba653fdfd4307 100644 --- a/lib/std/Build/Step/InstallArtifact.zig +++ b/lib/std/Build/Step/InstallArtifact.zig @@ -17,10 +17,6 @@ emitted_implib: ?LazyPath, pdb_dir: ?InstallDir, emitted_pdb: ?LazyPath, -// hack for stage2_x86_64 + coff -compiler_rt_dyn_lib_dir: ?InstallDir, -emitted_compiler_rt_dyn_lib: ?LazyPath, - h_dir: ?InstallDir, emitted_h: ?LazyPath, -- 2.54.0 From 24d260363f7130bd1a82a7aa07f123707c5de447 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 19 Apr 2026 12:45:35 -0700 Subject: [PATCH 057/179] configurer: fix bad serialization of strings in run args --- lib/compiler/Maker/ScannedConfig.zig | 5 +++++ lib/compiler/configurer.zig | 18 +++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 6115e05d629a17ec81348ae0e15770738e30295a..183146af278abe426d194ed932ed83300525b5aa 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -74,6 +74,11 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi Configuration.MaxRss => { try s.value(field_value.toBytes(), .{}); }, + Configuration.Step.Run.Arg.Index => { + var sub_struct = try s.beginStruct(.{}); + try printStruct(sc, &sub_struct, Configuration.Step.Run.Arg, field_value.get(c)); + try sub_struct.end(); + }, else => switch (@typeInfo(Field)) { .int => try s.int(field_value), .pointer => |info| switch (info.size) { diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 538ff6481b2efbe13e232b0e7e9ddb0f11a6fe24..2feac82180f0f9b3df5d65677d2dde5a0133af6c 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -376,7 +376,7 @@ const Serialize = struct { .generated = false, .dep_file = false, }, - .prefix = .{ .value = try s.addOptionalString(a.prefix) }, + .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, .basename = .{ .value = null }, .path = .{ .value = null }, @@ -394,7 +394,7 @@ const Serialize = struct { .generated = false, .dep_file = false, }, - .prefix = .{ .value = try s.addOptionalString(a.prefix) }, + .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, .basename = .{ .value = null }, .path = .{ .value = try addLazyPath(s, a.lazy_path) }, @@ -412,7 +412,7 @@ const Serialize = struct { .generated = false, .dep_file = false, }, - .prefix = .{ .value = try addOptionalString(s, a.prefix) }, + .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = try addOptionalString(s, a.suffix) }, .basename = .{ .value = null }, .path = .{ .value = try addLazyPath(s, a.lazy_path) }, @@ -430,7 +430,7 @@ const Serialize = struct { .generated = false, .dep_file = false, }, - .prefix = .{ .value = try addOptionalString(s, a.prefix) }, + .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, .basename = .{ .value = null }, .path = .{ .value = try addLazyPath(s, a.lazy_path) }, @@ -448,7 +448,7 @@ const Serialize = struct { .generated = false, .dep_file = false, }, - .prefix = .{ .value = try addOptionalString(s, a) }, + .prefix = .{ .value = try wc.addString(a) }, .suffix = .{ .value = null }, .basename = .{ .value = null }, .path = .{ .value = null }, @@ -466,9 +466,9 @@ const Serialize = struct { .generated = true, .dep_file = false, }, - .prefix = .{ .value = try addOptionalString(s, a.prefix) }, + .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, - .basename = .{ .value = try addOptionalString(s, a.basename) }, + .basename = .{ .value = if (a.basename.len != 0) try wc.addString(a.basename) else null }, .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, @@ -484,9 +484,9 @@ const Serialize = struct { .generated = true, .dep_file = false, }, - .prefix = .{ .value = try addOptionalString(s, a.prefix) }, + .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, - .basename = .{ .value = try addOptionalString(s, a.basename) }, + .basename = .{ .value = if (a.basename.len != 0) try wc.addString(a.basename) else null }, .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, -- 2.54.0 From d707e37ec2e095114a14ce3e4cfc6a04bd0dfd53 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 23 Apr 2026 12:53:16 -0700 Subject: [PATCH 058/179] update todo text file --- BRANCH_TODO | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/BRANCH_TODO b/BRANCH_TODO index eef38f57af77c0131f7bac2ec28ddb1636c6b132..51e8ad28c9593d696cf6d63f193c46cfbb8e9436 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -15,11 +15,16 @@ * https://codeberg.org/ziglang/zig/issues/31397 * restore the generated_compiler_rt_dyn_lib hack? * run args +* https://codeberg.org/ziglang/zig/pulls/30762 ## Followup Issues * link_eh_frame_hdr should be DefaultingBool * make --foo, --no-foo CLI args uniform (make them -f args instead) * install steps should provide generated files for installed things, then delete the run step hack - but artifact install steps also add paths for dyn libs on windows +* no more "artifact arg" to run step. if you want to run the post-install binary, get the lazy path + from the install step. +## Release Notes +* run args are all together now, not observable in configure phase whether run args are provided -- 2.54.0 From 5a4b5b549b606bdad026b977fd3fa669b6f7070d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 23 Apr 2026 15:21:16 -0700 Subject: [PATCH 059/179] maker: restore Step.Run logic for adding artifact arg When an artifact arg is added to a Run step, if the artifact is installed, then the installation path is added rather than the cache artifact path. This is probably something that should change in the future, but the goal of this branch is to generally avoid breakage other than that caused by phase separation. --- lib/compiler/Maker/Step/Compile.zig | 2 ++ lib/compiler/Maker/Step/InstallArtifact.zig | 4 ++++ lib/compiler/Maker/Step/Run.zig | 12 +++++------- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 464acf5ba2509192bfa6fd2e3d86592537ae6bc3..d4d370bb80421b6d277ed1b120a059eac118f616 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -20,6 +20,8 @@ const Maker = @import("../../Maker.zig"); zig_process: ?*Step.ZigProcess = null, /// Persisted to reuse memory on subsequent calls to `make`. zig_args: std.ArrayList([]const u8) = .empty, +/// Populated by InstallArtifact. +installed_path: ?Path = null, pub fn make( compile: *Compile, diff --git a/lib/compiler/Maker/Step/InstallArtifact.zig b/lib/compiler/Maker/Step/InstallArtifact.zig index 2d761566484769b91c8fd347a3631ac4ee7e129b..00d787e3eb554724ca42226397d1f260e91dfeff 100644 --- a/lib/compiler/Maker/Step/InstallArtifact.zig +++ b/lib/compiler/Maker/Step/InstallArtifact.zig @@ -55,6 +55,10 @@ pub fn make( if (conf_ia.flags.dylib_symlinks) try maker.installSymLinks(arena, dest_path, compile_step_index, step_index); + + const make_comp_step = maker.stepByIndex(compile_step_index); + const make_comp = &make_comp_step.extended.compile; + make_comp.installed_path = dest_path; } } diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 4b7644f06bd440f5d6430caf51525e8b84b7ec1a..7f821d94d4b27b40a33f3df070af0dda3bc1a114 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -132,22 +132,20 @@ pub fn make( const root_module = producer.root_module.get(conf); const root_module_target = root_module.resolved_target.get(conf).?.result.get(conf); const os_tag = root_module_target.flags.os_tag.unwrap().?; - - if (true) @panic("TODO"); + const producer_make_comp_step = maker.stepByIndex(producer_index); + const producer_make_comp = &producer_make_comp_step.extended.compile; if (os_tag == .windows) { // On Windows we don't have rpaths so we have to add .dll search paths to PATH addPathForDynLibs(producer_index); } - const file_path = producer_index.installed_path orelse producer_index.generated_bin.?.path.?; + const file_path = producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?).*; argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ - prefix, - try convertPathArg(run_index, maker, .{ .root_dir = .cwd(), .sub_path = file_path }), - suffix, + prefix, try convertPathArg(run_index, maker, file_path), suffix, })); - _ = try man.addFile(file_path, null); + _ = try man.addFilePath(file_path, null); }, .output_file, .output_directory => { const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; -- 2.54.0 From 378b790ee20fa64c549d114204dafc19d1e33362 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Apr 2026 10:54:17 -0700 Subject: [PATCH 060/179] maker: port more of Run step over --- lib/compiler/Maker/Step/Run.zig | 108 +++++++++++++++----------------- lib/std/Build/Configuration.zig | 4 ++ 2 files changed, 53 insertions(+), 59 deletions(-) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 7f821d94d4b27b40a33f3df070af0dda3bc1a114..9dc0b0fecf3f2575bf5cb7dde7a60242740812cd 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -48,6 +48,7 @@ pub fn make( const conf_run = conf_step.extended.get(conf.extra).run; const argv_list = &run.argv; const output_placeholders = &run.output_placeholders; + const cache_root = graph.local_cache_root; argv_list.clearRetainingCapacity(); output_placeholders.clearRetainingCapacity(); @@ -174,51 +175,62 @@ pub fn make( } } - if (true) @panic("TODO"); - switch (conf_run.stdin.u) { .bytes => |bytes| { - man.hash.addBytes(bytes); + man.hash.addBytes(bytes.slice(conf)); }, .lazy_path => |lazy_path| { - const file_path = lazy_path.getPath2(graph, step); - _ = try man.addFile(file_path, null); + const file_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index); + _ = try man.addFilePath(file_path, null); }, .none => {}, } - if (conf_run.captured_stdout) |captured| { - man.hash.addBytes(captured.output.basename); - man.hash.add(captured.trim_whitespace); + if (conf_run.captured_stdout.value) |captured| { + man.hash.addBytes(captured.basename.slice(conf)); + man.hash.add(conf_run.flags.stdout_trim_whitespace); } - if (conf_run.captured_stderr) |captured| { - man.hash.addBytes(captured.output.basename); - man.hash.add(captured.trim_whitespace); + if (conf_run.captured_stderr.value) |captured| { + man.hash.addBytes(captured.basename.slice(conf)); + man.hash.add(conf_run.flags.stderr_trim_whitespace); } - std.log.err("TODO hashStdIo", .{}); - //hashStdIo(&man.hash, conf_run.stdio); + switch (conf_run.flags.stdio) { + .infer_from_args, .inherit, .zig_test => {}, + .check => { + man.hash.addBytes(if (conf_run.expect_stderr_exact.value) |bytes| bytes.slice(conf) else ""); + man.hash.addBytes(if (conf_run.expect_stdout_exact.value) |bytes| bytes.slice(conf) else ""); + for (conf_run.expect_stderr_match.slice) |bytes| man.hash.addBytes(bytes.slice(conf)); + for (conf_run.expect_stdout_match.slice) |bytes| man.hash.addBytes(bytes.slice(conf)); + man.hash.add(conf_run.flags2.expect_term_status); + man.hash.addOptional(conf_run.expect_term_value.value); + }, + } - for (conf_run.file_inputs.items) |lazy_path| { - _ = try man.addFile(lazy_path.getPath2(graph, step), null); + for (conf_run.file_inputs.slice) |lazy_path| { + const file_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index); + _ = try man.addFilePath(file_path, null); } - if (conf_run.cwd) |cwd| { - const cwd_path = cwd.getPath3(graph, step); + if (conf_run.cwd.value) |lazy_path| { + const cwd_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index); _ = man.hash.addBytes(try cwd_path.toString(arena)); } const has_side_effects = conf_run.flags.has_side_effects; - if (!has_side_effects and try step.cacheHitAndWatch(&man)) { + if (true) @panic("TODO"); + + if (!has_side_effects and try step.cacheHitAndWatch(maker, &man)) { // cache hit, skip running command const digest = man.final(); try populateGeneratedPaths( arena, output_placeholders.items, - graph.cache_root, + &conf_run, + cache_root, &digest, ); @@ -236,7 +248,8 @@ pub fn make( try populateGeneratedPaths( arena, output_placeholders.items, - graph.cache_root, + &conf_run, + cache_root, &digest, ); @@ -248,9 +261,9 @@ pub fn make( .output_directory => output_sub_path, else => unreachable, }; - graph.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ - graph.cache_root, output_sub_dir_path, err, + cache_root, output_sub_dir_path, err, }); }; const arg_output_path = try convertPathArg(run_index, maker, .{ @@ -281,13 +294,13 @@ pub fn make( .output_directory => output_sub_path, else => unreachable, }; - graph.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ - graph.cache_root, output_sub_dir_path, err, + cache_root, output_sub_dir_path, err, }); }; const raw_output_path: Path = .{ - .root_dir = graph.cache_root, + .root_dir = cache_root, .sub_path = graph.pathJoin(&output_components), }; placeholder.output.generated_file.path = raw_output_path.toString(arena) catch @panic("OOM"); @@ -318,21 +331,21 @@ pub fn make( if (any_output) { const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; - graph.cache_root.handle.rename(tmp_dir_path, graph.cache_root.handle, o_sub_path, io) catch |err| switch (err) { + cache_root.handle.rename(tmp_dir_path, cache_root.handle, o_sub_path, io) catch |err| switch (err) { Dir.RenameError.DirNotEmpty => { - graph.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { + cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { return step.fail(maker, "unable to remove dir '{f}'{s}: {t}", .{ - graph.cache_root, tmp_dir_path, del_err, + cache_root, tmp_dir_path, del_err, }); }; - graph.cache_root.handle.rename(tmp_dir_path, graph.cache_root.handle, o_sub_path, io) catch |retry_err| { + cache_root.handle.rename(tmp_dir_path, cache_root.handle, o_sub_path, io) catch |retry_err| { return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - graph.cache_root, tmp_dir_path, graph.cache_root, o_sub_path, retry_err, + cache_root, tmp_dir_path, cache_root, o_sub_path, retry_err, }); }; }, else => return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - graph.cache_root, tmp_dir_path, graph.cache_root, o_sub_path, err, + cache_root, tmp_dir_path, cache_root, o_sub_path, err, }), }; } @@ -342,7 +355,8 @@ pub fn make( try populateGeneratedPaths( arena, output_placeholders.items, - graph.cache_root, + &conf_run, + cache_root, &digest, ); } @@ -1548,8 +1562,7 @@ const CapturedStdIo = void; // TODO get it from Configuration fn populateGeneratedPaths( arena: std.mem.Allocator, output_placeholders: []const IndexedOutput, - captured_stdout: ?*CapturedStdIo, - captured_stderr: ?*CapturedStdIo, + conf_run: *const Configuration.Step.Run, cache_root: Cache.Directory, digest: *const Cache.HexDigest, ) !void { @@ -1559,13 +1572,13 @@ fn populateGeneratedPaths( }); } - if (captured_stdout) |captured| { + if (conf_run.captured_stdout.value) |captured| { captured.output.generated_file.path = try cache_root.join(arena, &.{ "o", digest, captured.output.basename, }); } - if (captured_stderr) |captured| { + if (conf_run.captured_stderr.value) |captured| { captured.output.generated_file.path = try cache_root.join(arena, &.{ "o", digest, captured.output.basename, }); @@ -1985,29 +1998,6 @@ fn spawnChildAndCollect( } } -fn hashStdIo(hh: *Cache.HashHelper, stdio: void) void { - switch (stdio) { - .infer_from_args, .inherit, .zig_test => {}, - .check => |checks| for (checks.items) |check| { - hh.add(@as(std.meta.Tag(@This().StdIo.Check), check)); - switch (check) { - .expect_stderr_exact, - .expect_stderr_match, - .expect_stdout_exact, - .expect_stdout_match, - => |s| hh.addBytes(s), - - .expect_term => |term| { - hh.add(@as(std.meta.Tag(process.Child.Term), term)); - switch (term) { - inline .exited, .signal, .stopped => |x| hh.add(x), - .unknown => |x| hh.add(x), - } - }, - } - }, - } -} fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { return if (expected) |e| switch (e) { .exited => |expected_code| switch (actual) { diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 9949969c39768da3d3d5a2d9ec9e397af3124665..cd45617de667a88b631dd2f80c25dd3895a7fe22 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1572,6 +1572,10 @@ pub const Bytes = extern struct { /// Points into `string_bytes`. index: u32, len: u32, + + pub fn slice(bytes: Bytes, c: *const Configuration) []const u8 { + return c.string_bytes[bytes.index..][0..bytes.len]; + } }; pub const DefaultingBool = enum(u2) { -- 2.54.0 From 0d48cbb822551c07ac987c9c2e20d3251ee3a09c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Apr 2026 20:57:58 -0700 Subject: [PATCH 061/179] std.process.Environ.Map: add putAll and clearRetainingCapacity --- lib/std/process/Environ.zig | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/std/process/Environ.zig b/lib/std/process/Environ.zig index 3f467a8db936c6ee0720e1921f46cebc08f0111f..bf0408c31ddc4eebf48a08dea54b6391a524e012 100644 --- a/lib/std/process/Environ.zig +++ b/lib/std/process/Environ.zig @@ -96,6 +96,7 @@ pub const WindowsBlock = struct { } }; +/// Each key and each value are allocated independently and owned by this data structure. pub const Map = struct { array_hash_map: ArrayHashMap, allocator: Allocator, @@ -340,9 +341,6 @@ pub const Map = struct { /// Returns a full copy of `em` allocated with `gpa`, which is not necessarily /// the same allocator used to allocate `em`. pub fn clone(m: *const Map, gpa: Allocator) Allocator.Error!Map { - // Since we need to dupe the keys and values, the only way for error handling to not be a - // nightmare is to add keys to an empty map one-by-one. This could be avoided if this - // abstraction were a bit less... OOP-esque. var new: Map = .init(gpa); errdefer new.deinit(); try new.array_hash_map.ensureUnusedCapacity(gpa, m.array_hash_map.count()); @@ -352,6 +350,32 @@ pub const Map = struct { return new; } + /// Adds all the key-value pairs from `other` into this `m`. + pub fn putAll(m: *Map, other: *const Map) Allocator.Error!void { + const gpa = m.allocator; + try m.array_hash_map.ensureUnusedCapacity(gpa, other.array_hash_map.count()); + const start = m.count(); + errdefer while (m.array_hash_map.count() > start) { + const kv = m.array_hash_map.pop().?; + gpa.free(kv.key); + gpa.free(kv.value); + }; + for (other.array_hash_map.keys(), other.array_hash_map.values()) |key, value| { + try m.put(key, value); + } + } + + /// Set the length to zero, freeing all key and value memory, not freeing + /// the allocation for the entries. + pub fn clearRetainingCapacity(m: *Map) void { + const gpa = m.allocator; + for (m.array_hash_map.keys(), m.array_hash_map.values()) |k, v| { + gpa.free(k); + gpa.free(v); + } + m.array_hash_map.clearRetainingCapacity(); + } + /// Creates a null-delimited environment variable block in the format /// expected by POSIX, from a hash map plus options. pub fn createPosixBlock( -- 2.54.0 From c8b583885d75524fc92cc02a9d00a49a76f2ea70 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Apr 2026 20:58:30 -0700 Subject: [PATCH 062/179] maker: port Run step logic up to spawnChildAndCollect --- BRANCH_TODO | 1 + lib/compiler/Maker.zig | 34 +-- lib/compiler/Maker/Graph.zig | 12 + lib/compiler/Maker/Step.zig | 13 +- lib/compiler/Maker/Step/Run.zig | 500 +++++++++++++++++++------------- lib/compiler/configurer.zig | 5 +- lib/std/Build/Configuration.zig | 11 - lib/std/Build/Step/Run.zig | 12 +- 8 files changed, 327 insertions(+), 261 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 51e8ad28c9593d696cf6d63f193c46cfbb8e9436..7b28f34c27eaf74e21e7457de00cb0ac9b4606af 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -18,6 +18,7 @@ * https://codeberg.org/ziglang/zig/pulls/30762 ## Followup Issues +* reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make * link_eh_frame_hdr should be DefaultingBool * make --foo, --no-foo CLI args uniform (make them -f args instead) * install steps should provide generated files for installed things, then delete the run step hack diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 2b8b163447787f52ce150f4f586c8e9a9cdc0534..0ba2cfbbfc3b6fa0ea103ba93e9ed35f6a0b3a8b 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -153,17 +153,6 @@ pub fn main(init: process.Init.Minimal) !void { var debounce_interval_ms: u16 = 50; var webui_listen: ?Io.net.IpAddress = null; var debug_pkg_config: bool = false; - // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, - // this will be the directory $glibc-build-dir/install/glibcs - // Given the example of the aarch64 target, this is the directory - // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. - // Also works for dynamic musl. - var libc_runtimes_dir: ?[]const u8 = null; - var enable_wine = false; - var enable_qemu = false; - var enable_wasmtime = false; - var enable_darling = false; - var enable_rosetta = false; var run_args: ?[]const []const u8 = null; if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { @@ -314,7 +303,7 @@ pub fn main(init: process.Init.Minimal) !void { fatal("unrecognized optimization mode: {s}", .{rest}); } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--verbose")) { graph.verbose = true; } else if (mem.eql(u8, arg, "--verbose-air")) { @@ -370,25 +359,25 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "-fno-incremental")) { graph.incremental = false; } else if (mem.eql(u8, arg, "-fwine")) { - enable_wine = true; + graph.enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { - enable_wine = false; + graph.enable_wine = false; } else if (mem.eql(u8, arg, "-fqemu")) { - enable_qemu = true; + graph.enable_qemu = true; } else if (mem.eql(u8, arg, "-fno-qemu")) { - enable_qemu = false; + graph.enable_qemu = false; } else if (mem.eql(u8, arg, "-fwasmtime")) { - enable_wasmtime = true; + graph.enable_wasmtime = true; } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - enable_wasmtime = false; + graph.enable_wasmtime = false; } else if (mem.eql(u8, arg, "-frosetta")) { - enable_rosetta = true; + graph.enable_rosetta = true; } else if (mem.eql(u8, arg, "-fno-rosetta")) { - enable_rosetta = false; + graph.enable_rosetta = false; } else if (mem.eql(u8, arg, "-fdarling")) { - enable_darling = true; + graph.enable_darling = true; } else if (mem.eql(u8, arg, "-fno-darling")) { - enable_darling = false; + graph.enable_darling = false; } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { graph.allow_so_scripts = true; } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { @@ -533,6 +522,7 @@ pub fn main(init: process.Init.Minimal) !void { .bin = install_bin_path, .include = install_include_path, }, + .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), .generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len), .run_args = run_args, diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index 50dba7f931137772a43423b29a4e430ccff06f38..cb963003491d0a29969402029cd2c868e93d2f4c 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -52,6 +52,18 @@ error_limit: ?u32 = null, /// a single step spawning a fixed number of processes this can be used. max_jobs: ?u32 = null, +/// After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, +/// this will be the directory $glibc-build-dir/install/glibcs +/// Given the example of the aarch64 target, this is the directory +/// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. +/// Also works for dynamic musl. +libc_runtimes_dir: ?[]const u8 = null, +enable_wine: bool = false, +enable_qemu: bool = false, +enable_wasmtime: bool = false, +enable_darling: bool = false, +enable_rosetta: bool = false, + /// Intention of verbose is to print all sub-process command lines to stderr /// before spawning them. pub fn handleVerbose( diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index c1c9c1f2cc3db29fc8685b806f90b1f56353c7dd..8651ec86673677922789ee8eac0672db3bb6826c 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -62,12 +62,11 @@ comptime { // Common cache line size is 128. This check prevents accidentally crossing // an additional cache line. In the future it might be nice to try to fit // this struct in 128 bytes or less. - assert(@sizeOf(@This()) <= 128 * 3); + assert(@sizeOf(@This()) <= 128 * 4); } pub const Extended = union(enum) { check_file: Todo, - check_object: Todo, compile: Compile, config_header: Todo, fail: Todo, @@ -87,7 +86,6 @@ pub const Extended = union(enum) { pub fn init(tag: Configuration.Step.Tag) Extended { return switch (tag) { .check_file => .{ .check_file = .{} }, - .check_object => .{ .check_object = .{} }, .compile => .{ .compile = .{} }, .config_header => .{ .config_header = .{} }, .fail => .{ .fail = .{} }, @@ -645,9 +643,8 @@ fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { /// Asserts that the caller has already populated `s.result_failed_command`. pub inline fn handleChildProcUnsupported(s: *Step, maker: *Maker) FailError!void { assert(s.result_failed_command != null); - if (!std.process.can_spawn) { + if (!std.process.can_spawn) return s.fail(maker, "unable to spawn process: host cannot spawn child processes", .{}); - } } /// Asserts that the caller has already populated `s.result_failed_command`. @@ -708,10 +705,10 @@ fn failWithCacheError( /// Prefer `writeManifestAndWatch` unless you already added watch inputs /// separately from using the cache system. -pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { +pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { if (s.test_results.isSuccess()) { man.writeManifest() catch |err| { - try s.addError("unable to write cache manifest: {t}", .{err}); + try s.addError(maker, "unable to write cache manifest: {t}", .{err}); }; } } @@ -721,7 +718,7 @@ pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { /// /// Must be accompanied with `cacheHitAndWatch`. pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { - try writeManifest(s, man); + try writeManifest(s, maker, man); try setWatchInputsFromManifest(s, maker, man); } diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 9dc0b0fecf3f2575bf5cb7dde7a60242740812cd..4d6d491f168d9092ff2e8cda77a3a9f3b0c8a250 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -16,6 +16,7 @@ const allocPrint = std.fmt.allocPrint; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); +const Fuzz = @import("../../Maker/Fuzz.zig"); /// If this is a Zig unit test binary, this tracks the names of the unit /// tests that are also fuzz tests. Indexes cannot be used as they may @@ -31,6 +32,8 @@ rebuilt_executable: ?Path = null, argv: std.ArrayList([]const u8) = .empty, /// Persisted to reuse memory on subsequent calls to `make`. output_placeholders: std.ArrayList(IndexedOutput) = .empty, +/// Persisted to reuse memory on subsequent calls to `make`. +environ_map: std.process.Environ.Map = .{ .array_hash_map = .empty, .allocator = undefined }, pub fn make( run: *Run, @@ -67,6 +70,8 @@ pub fn make( man.hash.add(conf_run.flags.color); man.hash.add(conf_run.flags.disable_zig_progress); + var dep_file_count: usize = 0; + for (conf_run.args.slice) |arg_index| { const arg = arg_index.get(conf); try argv_list.ensureUnusedCapacity(gpa, 1); @@ -157,6 +162,9 @@ pub fn make( man.hash.addBytesZ(basename); man.hash.addBytesZ(suffix); + man.hash.add(arg.flags.dep_file); + dep_file_count += @intFromBool(arg.flags.dep_file); + // Add a placeholder into the argument list because we need the // manifest hash to be updated with all arguments before the // object directory is computed. @@ -220,145 +228,95 @@ pub fn make( const has_side_effects = conf_run.flags.has_side_effects; - if (true) @panic("TODO"); - if (!has_side_effects and try step.cacheHitAndWatch(maker, &man)) { - // cache hit, skip running command + // Cache hit; skip running command. const digest = man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - &conf_run, - cache_root, - &digest, - ); - + try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); + try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); step.result_cached = true; return; } - const dep_output_file = conf_run.dep_output_file orelse { - // We already know the final output paths, use them directly. - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - &conf_run, - cache_root, - &digest, - ); - + if (dep_file_count == 0) { + // We already know the final output paths; use them directly. + const digest = if (has_side_effects) man.hash.final() else man.final(); const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; - for (output_placeholders.items) |placeholder| { - const output_sub_path = graph.pathJoin(&.{ output_dir_path, placeholder.output.basename }); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ - cache_root, output_sub_dir_path, err, - }); - }; - const arg_output_path = try convertPathArg(run_index, maker, .{ - .root_dir = .cwd(), - .sub_path = placeholder.output.generated_file.getPath(), - }); - argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) - arg_output_path - else - try allocPrint(arena, "{s}{s}", .{ placeholder.output.prefix, arg_output_path }); - } - - try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); - if (!has_side_effects) try step.writeManifestAndWatch(&man); + try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); + try populateGeneratedPathsCreateDirs(run, run_index, maker, output_dir_path); + try runCommand(run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); + if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); return; - }; + } - // We do not know the final output paths yet, use temp paths to run the command. + // We do not know the final output paths yet; use temporary directory to run the command. var rand_int: u64 = undefined; io.random(@ptrCast(&rand_int)); const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + try populateGeneratedPathsCreateDirs(run, run_index, maker, tmp_dir_path); + try runCommand(run, run_index, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null); + for (output_placeholders.items) |placeholder| { - const output_components = .{ tmp_dir_path, placeholder.output.basename }; - const output_sub_path = graph.pathJoin(&output_components); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, + const arg = placeholder.arg_index.get(conf); + switch (arg.flags.tag) { + .output_file => if (arg.flags.dep_file) { + const generated_path = maker.generatedPath(arg.generated.value.?).*; + const result = if (has_side_effects) + man.addDepFile(generated_path.root_dir.handle, generated_path.sub_path) + else + man.addDepFilePost(generated_path.root_dir.handle, generated_path.sub_path); + result catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + else => |e| return step.fail(maker, "failed adding to cache the file {f}: {t}", .{ + generated_path, e, + }), + }; + }, + .output_directory => continue, else => unreachable, - }; - cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ - cache_root, output_sub_dir_path, err, - }); - }; - const raw_output_path: Path = .{ - .root_dir = cache_root, - .sub_path = graph.pathJoin(&output_components), - }; - placeholder.output.generated_file.path = raw_output_path.toString(arena) catch @panic("OOM"); - argv_list.items[placeholder.index] = try mem.concat(arena, u8, .{ - placeholder.output.prefix, - try convertPathArg(run_index, maker, raw_output_path), - }); + } } - try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null); - - const dep_file_dir = Dir.cwd(); - const dep_file_basename = dep_output_file.generated_file.getPath2(graph, step); - if (has_side_effects) - try man.addDepFile(dep_file_dir, dep_file_basename) - else - try man.addDepFilePost(dep_file_dir, dep_file_basename); - - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); + const digest = if (has_side_effects) man.hash.final() else man.final(); const any_output = output_placeholders.items.len > 0 or - conf_run.captured_stdout != null or conf_run.captured_stderr != null; + conf_run.captured_stdout.value != null or conf_run.captured_stderr.value != null; - // Rename into place if (any_output) { - const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; + // Rename into place. + const tmp_path: Path = .{ .root_dir = cache_root, .sub_path = tmp_dir_path }; + const dst_path: Path = .{ .root_dir = cache_root, .sub_path = "o" ++ Dir.path.sep_str ++ &digest }; + Dir.rename( + tmp_path.root_dir.handle, + tmp_path.sub_path, + dst_path.root_dir.handle, + dst_path.sub_path, + io, + ) catch |err| switch (err) { + error.DirNotEmpty => { + dst_path.root_dir.handle.deleteTree(io, dst_path.sub_path) catch |del_err| + return step.fail(maker, "failed to remove tree {f}: {t}", .{ dst_path, del_err }); - cache_root.handle.rename(tmp_dir_path, cache_root.handle, o_sub_path, io) catch |err| switch (err) { - Dir.RenameError.DirNotEmpty => { - cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { - return step.fail(maker, "unable to remove dir '{f}'{s}: {t}", .{ - cache_root, tmp_dir_path, del_err, - }); - }; - cache_root.handle.rename(tmp_dir_path, cache_root.handle, o_sub_path, io) catch |retry_err| { - return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - cache_root, tmp_dir_path, cache_root, o_sub_path, retry_err, - }); - }; + Dir.rename( + tmp_path.root_dir.handle, + tmp_path.sub_path, + dst_path.root_dir.handle, + dst_path.sub_path, + io, + ) catch |retry_err| return step.fail(maker, "failed to rename directory {f} to {f}: {t}", .{ + tmp_path, dst_path, retry_err, + }); }, - else => return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - cache_root, tmp_dir_path, cache_root, o_sub_path, err, + else => return step.fail(maker, "failed to rename directory {f} to {f}: {t}", .{ + tmp_path, dst_path, err, }), }; } - if (!has_side_effects) try step.writeManifestAndWatch(&man); + if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); - try populateGeneratedPaths( - arena, - output_placeholders.items, - &conf_run, - cache_root, - &digest, - ); + try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); + try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); } /// Reads stdout of a Zig test process until a termination condition is reached: @@ -918,9 +876,12 @@ const FuzzTestRunner = struct { } fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { + const fuzz = f.context.fuzz; + const maker = fuzz.maker; const step = &f.run.step; - const b = step.owner; - const io = b.graph.io; + const graph = maker.graph; + const io = graph.io; + const cache_root = graph.local_cache_root; if (f.coverage_id == null) return; @@ -938,7 +899,7 @@ const FuzzTestRunner = struct { }) { const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; - in_f = b.cache_root.handle.openFile(io, in_name, .{ + in_f = cache_root.handle.openFile(io, in_name, .{ .lock = .exclusive, .lock_nonblocking = true, }) catch |e| switch (e) { @@ -946,7 +907,7 @@ const FuzzTestRunner = struct { error.WouldBlock => continue, // Can not be from // the crashed instance since it is still locked. else => return step.fail("failed to open file '{f}{s}': {t}", .{ - b.cache_root, in_name, e, + cache_root, in_name, e, }), }; @@ -955,7 +916,7 @@ const FuzzTestRunner = struct { in_f.close(io); switch (e) { error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, + cache_root, in_name, in_r.err.?, }), error.EndOfStream => continue, } @@ -974,10 +935,10 @@ const FuzzTestRunner = struct { // Save it to a seperate file const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; - const out = b.cache_root.handle.createFile(io, crash_name, .{ + const out = cache_root.handle.createFile(io, crash_name, .{ .lock = .exclusive, // Multiple run steps could have found a crash at the same time }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ - b.cache_root, crash_name, e, + cache_root, crash_name, e, }); defer out.close(io); @@ -985,17 +946,17 @@ const FuzzTestRunner = struct { var out_w = out.writerStreaming(io, &out_w_buf); _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, + cache_root, in_name, in_r.err.?, }), error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ - b.cache_root, crash_name, out_w.err.?, + cache_root, crash_name, out_w.err.?, }), }; return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ f.run.fuzz_tests.items[header.test_i], fmtTerm(term), - b.cache_root, + cache_root, crash_name, }); } @@ -1492,54 +1453,85 @@ pub fn rerunInFuzzMode( const maker = fuzz.maker; const graph = maker.graph; const step = &run.step; - const b = step.owner; const io = graph.io; - const arena = b.allocator; - var argv_list: std.ArrayList([]const u8) = .empty; - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(arena, bytes); + const arena = graph.arena; // TODO don't leak into the process arena + const gpa = maker.gpa; + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; + const argv_list = &run.argv; + + argv_list.clearRetainingCapacity(); + + for (conf_run.args.slice) |arg_index| { + const arg = arg_index.get(conf); + try argv_list.ensureUnusedCapacity(gpa, 1); + switch (arg.flags.tag) { + .string => { + const prefix = arg.prefix.value.?.slice(conf); + argv_list.appendAssumeCapacity(prefix); }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, convertPathArg(run_index, maker, file_path) })); + .path_file => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); + argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ + prefix, try convertPathArg(run_index, maker, file_path), suffix, + })); }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, convertPathArg(run_index, maker, file_path), dd.suffix })); + .path_directory => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); + const resolved_arg = try mem.concat(arena, u8, &.{ + prefix, try convertPathArg(run_index, maker, file_path), suffix, + }); + argv_list.appendAssumeCapacity(resolved_arg); }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); + .file_content => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + result.writer.writeAll(prefix) catch return error.OutOfMemory; - const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); + const file = file_path.root_dir.handle.openFile(io, file_path.sub_path, .{}) catch |err| + return step.fail(maker, "unable to open input file {f}: {t}", .{ file_path, err }); defer file.close(io); - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); + var file_reader = file.reader(io, &.{}); _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, + error.ReadFailed => switch (file_reader.err.?) { + error.Canceled => |e| return e, + else => |e| return step.fail(maker, "failed to read from {f}: {t}", .{ file_path, e }), + }, error.WriteFailed => return error.OutOfMemory, }; + result.writer.writeAll(suffix) catch return error.OutOfMemory; - try argv_list.append(arena, result.written()); + argv_list.appendAssumeCapacity(result.written()); }, - .artifact => |pa| { - const artifact = pa.artifact; - const file_path: []const u8 = p: { - if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); - break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; - }; - try argv_list.append(arena, b.fmt("{s}{s}", .{ - pa.prefix, - convertPathArg(run_index, maker, .{ .root_dir = .cwd(), .sub_path = file_path }), + .artifact => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const producer_index = arg.producer.value.?; + const producer_step = producer_index.ptr(conf); + const producer = producer_step.extended.get(conf.extra).compile; + const producer_make_comp_step = maker.stepByIndex(producer_index); + const producer_make_comp = &producer_make_comp_step.extended.compile; + const file_path: Path = if (producer_index == conf_run.producer.value.?) + run.rebuilt_executable.? + else + producer_make_comp.installed_path orelse + maker.generatedPath(producer.generated_bin.value.?).*; + argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ + prefix, try convertPathArg(run_index, maker, file_path), suffix, })); }, - .output_file, .output_directory => unreachable, + .output_file => unreachable, + .output_directory => unreachable, + .cli_rest_positionals => unreachable, } } @@ -1552,7 +1544,7 @@ pub fn rerunInFuzzMode( var rand_int: u64 = undefined; io.random(@ptrCast(&rand_int)); const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - try runCommand(run, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{ + try runCommand(run, run_index, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{ .fuzz = fuzz, }); } @@ -1560,28 +1552,94 @@ pub fn rerunInFuzzMode( const CapturedStdIo = void; // TODO get it from Configuration fn populateGeneratedPaths( - arena: std.mem.Allocator, + maker: *Maker, output_placeholders: []const IndexedOutput, - conf_run: *const Configuration.Step.Run, cache_root: Cache.Directory, digest: *const Cache.HexDigest, ) !void { + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + for (output_placeholders) |placeholder| { - placeholder.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, placeholder.output.basename, - }); + const arg = placeholder.arg_index.get(conf); + maker.generatedPath(arg.generated.value.?).* = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ + "o", digest, arg.basename.value.?.slice(conf), + }), + }; } +} + +fn populateGeneratedPathsCreateDirs( + run: *Run, + run_index: Configuration.Step.Index, + maker: *Maker, + output_dir_path: []const u8, +) !void { + const step = maker.stepByIndex(run_index); + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const io = graph.io; + const arena = graph.arena; // TODO don't leak into the process arena + const cache_root = graph.local_cache_root; + const argv = run.argv.items; + + for (run.output_placeholders.items) |placeholder| { + const arg = placeholder.arg_index.get(conf); + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const basename = arg.basename.value.?.slice(conf); + + const generated_path: Path = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ output_dir_path, basename }), + }; + const create_path: Path = .{ + .root_dir = cache_root, + .sub_path = switch (arg.flags.tag) { + .output_file => Dir.path.dirname(generated_path.sub_path).?, + .output_directory => generated_path.sub_path, + else => unreachable, + }, + }; + create_path.root_dir.handle.createDirPath(io, create_path.sub_path) catch |err| + return step.fail(maker, "unable to make path {f}: {t}", .{ create_path, err }); + + maker.generatedPath(arg.generated.value.?).* = generated_path; + + const arg_output_path = try convertPathArg(run_index, maker, generated_path); + argv[placeholder.index] = try mem.concat(arena, u8, &.{ prefix, arg_output_path, suffix }); + } +} + +fn populateGeneratedStdIo( + maker: *Maker, + conf_run: *const Configuration.Step.Run, + cache_root: Cache.Directory, + digest: *const Cache.HexDigest, +) !void { + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena if (conf_run.captured_stdout.value) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); + maker.generatedPath(captured.generated_file).* = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ + "o", digest, captured.basename.slice(conf), + }), + }; } if (conf_run.captured_stderr.value) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); + maker.generatedPath(captured.generated_file).* = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ + "o", digest, captured.basename.slice(conf), + }), + }; } } @@ -1600,11 +1658,12 @@ fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTer } const FuzzContext = struct { - fuzz: *std.Build.Fuzz, + fuzz: *Fuzz, }; fn runCommand( run: *Run, + run_index: Configuration.Step.Index, maker: *Maker, progress_node: std.Progress.Node, argv: []const []const u8, @@ -1615,30 +1674,45 @@ fn runCommand( const graph = maker.graph; const arena = graph.arena; // TODO don't leak into process arena const gpa = maker.gpa; - const step = &run.step; - const b = step.owner; + const step = maker.stepByIndex(run_index); const io = graph.io; + const cache_root = graph.local_cache_root; + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; + const environ_map = &run.environ_map; - const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; + const cwd: process.Child.Cwd = if (conf_run.cwd.value) |lazy_cwd| + .{ .path = try maker.resolveLazyPathIndexAbs(arena, lazy_cwd, run_index) } + else + .inherit; - try step.handleChildProcUnsupported(); - try Step.handleVerbose(step.owner, cwd, run.environ_map, argv); - - const allow_skip = switch (run.stdio) { - .check, .zig_test => run.skip_foreign_checks, + const allow_skip = switch (conf_run.flags.stdio) { + .check, .zig_test => conf_run.flags.skip_foreign_checks, else => false, }; - var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); - defer interp_argv.deinit(); + var interp_argv: std.ArrayList([]const u8) = .empty; - var environ_map: EnvMap = env: { - const orig = run.environ_map orelse &graph.environ_map; - break :env try orig.clone(gpa); - }; - defer environ_map.deinit(); + // `environ_map` is initialized with an undefined `allocator` field; lazily + // initialize it here. + environ_map.allocator = gpa; + // In either case we add to this mutatable data structure so that we can + // tweak the environment below. + environ_map.clearRetainingCapacity(); + if (conf_run.environ_map.value) |env_map_index| { + const conf_env_map = env_map_index.get(conf); + for (conf_env_map.keys.slice(conf), conf_env_map.values.slice(conf)) |k, v| { + try environ_map.put(k.slice(conf), v.slice(conf)); + } + } else { + try environ_map.putAll(&graph.environ_map); + } + try graph.handleVerbose(cwd, environ_map, argv); - const opt_generic_result = spawnChildAndCollect(run, maker, progress_node, argv, &environ_map, has_side_effects, fuzz_context) catch |err| term: { + if (true) @panic("TODO"); + + const opt_generic_result = spawnChildAndCollect(run_index, run, maker, progress_node, argv, &environ_map, has_side_effects, fuzz_context) catch |err| term: { // InvalidExe: cpu arch mismatch // FileNotFound: can happen with a wrong dynamic linker path if (err == error.InvalidExe or err == error.FileNotFound) interpret: { @@ -1660,7 +1734,7 @@ fn runCommand( (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); const other_target = exe.root_module.resolved_target.?.result; switch (std.zig.system.getExternalExecutor(io, &graph.host.result, &other_target, .{ - .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, + .qemu_fixes_dl = need_cross_libc and graph.libc_runtimes_dir != null, .link_libc = exe.is_linking_libc, })) { .native, .rosetta => { @@ -1668,7 +1742,7 @@ fn runCommand( break :interpret; }, .wine => |bin_name| { - if (b.enable_wine) { + if (graph.enable_wine) { try interp_argv.append(bin_name); try interp_argv.appendSlice(argv); @@ -1682,21 +1756,21 @@ fn runCommand( } }, .qemu => |bin_name| { - if (b.enable_qemu) { + if (graph.enable_qemu) { try interp_argv.append(bin_name); if (need_cross_libc) { - if (b.libc_runtimes_dir) |dir| { + if (graph.libc_runtimes_dir) |dir| { try interp_argv.append("-L"); - try interp_argv.append(b.pathJoin(&.{ + try interp_argv.append(try Dir.path.join(arena, &.{ dir, try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( - b.allocator, + arena, root_target.cpu.arch, root_target.os.tag, root_target.abi, ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( - b.allocator, + arena, root_target.cpu.arch, root_target.abi, ) else unreachable, @@ -1708,7 +1782,7 @@ fn runCommand( } else return failForeign(run, "-fqemu", argv[0], exe); }, .darling => |bin_name| { - if (b.enable_darling) { + if (graph.enable_darling) { try interp_argv.append(bin_name); try interp_argv.appendSlice(argv); } else { @@ -1716,7 +1790,7 @@ fn runCommand( } }, .wasmtime => |bin_name| { - if (b.enable_wasmtime) { + if (graph.enable_wasmtime) { try interp_argv.append(bin_name); try interp_argv.append("--dir=."); // Wasmtime doeesn't inherit environment variables from the parent process @@ -1743,8 +1817,8 @@ fn runCommand( .bad_os_or_cpu => { if (allow_skip) return error.MakeSkipped; - const host_name = try graph.host.result.zigTriple(b.allocator); - const foreign_name = try root_target.zigTriple(b.allocator); + const host_name = try graph.host.result.zigTriple(arena); + const foreign_name = try root_target.zigTriple(arena); return step.fail(maker, "the host system ({s}) is unable to execute binaries from the target ({s})", .{ host_name, foreign_name, @@ -1761,7 +1835,7 @@ fn runCommand( step.result_failed_command = null; try Step.handleVerbose(step.owner, cwd, run.environ_map, interp_argv.items); - break :term spawnChildAndCollect(run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { + break :term spawnChildAndCollect(run_index, run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; if (e == error.MakeFailed) return error.MakeFailed; // error already reported return step.fail(maker, "unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); @@ -1800,14 +1874,14 @@ fn runCommand( }) |stream| { if (stream.captured) |captured| { const output_components = .{ output_dir_path, captured.output.basename }; - const output_path = try b.cache_root.join(arena, &output_components); + const output_path = try cache_root.join(arena, &output_components); captured.output.generated_file.path = output_path; - const sub_path = b.pathJoin(&output_components); + const sub_path = try Dir.path.join(arena, &output_components); const sub_path_dirname = Dir.path.dirname(sub_path).?; - b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail(maker, "unable to make path '{f}{s}': {s}", .{ - b.cache_root, sub_path_dirname, @errorName(err), + cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { + return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ + cache_root, sub_path_dirname, err, }); }; const data = switch (captured.trim_whitespace) { @@ -1816,9 +1890,9 @@ fn runCommand( .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), }; - b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { - return step.fail(maker, "unable to write file '{f}{s}': {s}", .{ - b.cache_root, sub_path, @errorName(err), + cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { + return step.fail(maker, "unable to write file '{f}{s}': {t}", .{ + cache_root, sub_path, err, }); }; } @@ -1912,6 +1986,7 @@ const EvalGenericResult = struct { }; fn spawnChildAndCollect( + run_index: Configuration.Step.Index, run: *Run, maker: *Maker, progress_node: std.Progress.Node, @@ -1920,25 +1995,34 @@ fn spawnChildAndCollect( has_side_effects: bool, fuzz_context: ?FuzzContext, ) !?EvalGenericResult { - const b = run.step.owner; + const step = run.step; const graph = maker.graph; const gpa = maker.gpa; const io = graph.io; + const arena = graph.arena; // TODO don't leak into process arena + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; if (fuzz_context != null) { assert(!has_side_effects); assert(run.stdio == .zig_test); } - const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; + const child_cwd: process.Child.Cwd = if (conf_run.cwd) |lazy_cwd| + .{ .path = try maker.resolveLazyPathIndexAbs(arena, lazy_cwd, run_index) } + else + .inherit; // If an error occurs, it's caused by this command: - assert(run.step.result_failed_command == null); - run.step.result_failed_command = try Step.allocPrintCmd(gpa, child_cwd, .{ + assert(step.result_failed_command == null); + step.result_failed_command = try Step.allocPrintCmd(gpa, child_cwd, .{ .child = environ_map, .parent = &graph.environ_map, }, argv); + try step.handleChildProcUnsupported(maker); + var spawn_options: process.SpawnOptions = .{ .argv = argv, .cwd = child_cwd, @@ -1973,7 +2057,7 @@ fn spawnChildAndCollect( error.Canceled => |e| return e, else => |e| e, }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); try result; return null; } else { @@ -1993,7 +2077,7 @@ fn spawnChildAndCollect( error.Canceled => |e| return e, else => |e| e, }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); return try result; } } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 2feac82180f0f9b3df5d65677d2dde5a0133af6c..7815198bcaceadaea247f7ea5889c8a832511aaf 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -455,7 +455,7 @@ const Serialize = struct { .producer = .{ .value = null }, .generated = .{ .value = null }, }, - .output_file => |a| .{ + .output_file, .output_file_dep => |a, tag| .{ .flags = .{ .tag = .output_file, .prefix = a.prefix.len != 0, @@ -464,7 +464,7 @@ const Serialize = struct { .path = false, .producer = false, .generated = true, - .dep_file = false, + .dep_file = tag == .output_file_dep, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -1023,7 +1023,6 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { break :e @enumFromInt(extra_index); }, .check_file => @panic("TODO"), - .check_object => @panic("TODO"), .config_header => @panic("TODO"), .objcopy => @panic("TODO"), .options => @panic("TODO"), diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index cd45617de667a88b631dd2f80c25dd3895a7fe22..4c81bdb9eeff53d1528a0097ab161e3ce90ad3e2 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -427,7 +427,6 @@ pub const Step = extern struct { max_rss: MaxRss, extended: Storage.Extended(Flags, union(Tag) { check_file: CheckFile, - check_object: CheckObject, compile: Compile, config_header: ConfigHeader, fail: Fail, @@ -462,7 +461,6 @@ pub const Step = extern struct { pub const Tag = enum(u5) { check_file, - check_object, compile, config_header, fail, @@ -997,15 +995,6 @@ pub const Step = extern struct { }; }; - pub const CheckObject = struct { - flags: @This().Flags, - - pub const Flags = packed struct(u32) { - tag: Tag = .check_object, - _: u27 = 0, - }; - }; - pub const ConfigHeader = struct { flags: @This().Flags, diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index bfb540a0eec2f32cceb9e44c47ac49801e3f81d3..c91d8334fe40c0ddf3ccdc015643e3021b6e40ea 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -85,8 +85,6 @@ stdio_limit: std.Io.Limit, captured_stdout: ?*CapturedStdIo, captured_stderr: ?*CapturedStdIo, -dep_output_file: ?*Output, - has_side_effects: bool, test_runner_mode: bool = false, @@ -141,6 +139,7 @@ pub const Arg = union(enum) { file_content: PrefixedLazyPath, bytes: []const u8, output_file: *Output, + output_file_dep: *Output, output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. cli_rest_positionals, @@ -203,7 +202,6 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .stdio_limit = .unlimited, .captured_stdout = null, .captured_stderr = null, - .dep_output_file = null, .has_side_effects = false, .producer = null, }; @@ -476,12 +474,10 @@ pub fn addDepFileOutputArg(run: *Run, basename: []const u8) std.Build.LazyPath { /// Add a prefixed path argument to a dep file (.d) for the child process to /// write its discovered additional dependencies. -/// Only one dep file argument is allowed by instance. pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []const u8) std.Build.LazyPath { const b = run.step.owner; const graph = b.graph; const arena = graph.arena; - assert(run.dep_output_file == null); const dep_file = arena.create(Output) catch @panic("OOM"); dep_file.* = .{ @@ -490,9 +486,7 @@ pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []co .generated_file = graph.addGeneratedFile(&run.step), }; - run.dep_output_file = dep_file; - - run.argv.append(arena, .{ .output_file = dep_file }) catch @panic("OOM"); + run.argv.append(arena, .{ .output_file_dep = dep_file }) catch @panic("OOM"); return .{ .generated = .{ .index = dep_file.generated_file } }; } @@ -544,7 +538,7 @@ pub fn addPathDir(run: *Run, search_path: []const u8) void { .decorated_directory => false, .file_content => unreachable, // not allowed as first arg .bytes => |bytes| std.mem.endsWith(u8, bytes, ".exe"), - .output_file, .output_directory => false, + .output_file, .output_file_dep, .output_directory => false, }; const key = if (use_wine) "WINEPATH" else "PATH"; const prev_path = environ_map.get(key); -- 2.54.0 From 4e3d14f590160013e655f69e249997ff597f8e93 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Apr 2026 22:15:09 -0700 Subject: [PATCH 063/179] maker: update more Run step logic --- lib/compiler/Maker/Step.zig | 8 +- lib/compiler/Maker/Step/Run.zig | 626 ++++++++++++++++---------------- 2 files changed, 327 insertions(+), 307 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 8651ec86673677922789ee8eac0672db3bb6826c..9d4257a2967ddb3ea44ceaf437b09cb87bc776b4 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -18,10 +18,10 @@ const assert = std.debug.assert; const WebServer = @import("WebServer.zig"); const Maker = @import("../Maker.zig"); -const Compile = @import("Step/Compile.zig"); -const Run = @import("Step/Run.zig"); -const InstallArtifact = @import("Step/InstallArtifact.zig"); -const InstallFile = @import("Step/InstallFile.zig"); +pub const Compile = @import("Step/Compile.zig"); +pub const Run = @import("Step/Run.zig"); +pub const InstallArtifact = @import("Step/InstallArtifact.zig"); +pub const InstallFile = @import("Step/InstallFile.zig"); /// Avoid false sharing. _: void align(std.atomic.cache_line) = {}, diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 4d6d491f168d9092ff2e8cda77a3a9f3b0c8a250..81b46e031cf132e8c6bf5f1971624686eeede7d6 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -325,9 +325,10 @@ pub fn make( /// * The wait fails, indicating the child closed stdout and stderr fn waitZigTest( run: *Run, + run_index: Configuration.Step.Index, maker: *Maker, child: *process.Child, - options: Step.MakeOptions, + progress_node: std.Progress.Node, multi_reader: *Io.File.MultiReader, opt_metadata: *?TestMetadata, results: *Step.TestResults, @@ -342,9 +343,11 @@ fn waitZigTest( ns_elapsed: u64, }, } { - const gpa = run.step.owner.allocator; - const arena = run.step.owner.allocator; - const io = run.step.owner.graph.io; + const graph = maker.graph; + const gpa = maker.gpa; + const io = graph.io; + const arena = graph.arena; // TODO don't leak into the process arena + const step = maker.stepByIndex(run_index); var sub_prog_node: ?std.Progress.Node = null; defer if (sub_prog_node) |n| n.end(); @@ -367,10 +370,10 @@ fn waitZigTest( // start and it acknowledging the test starting, we terminate the child and raise an error. This // *should* never happen, but could in theory be caused by some very unlucky IB in a test. const response_timeout: Io.Clock.Duration = t: { - const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); + const ns = @max(maker.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; }; - const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ + const test_timeout: ?Io.Clock.Duration = if (maker.unit_test_timeout_ns) |ns| .{ .clock = .awake, .raw = .fromNanoseconds(ns), } else null; @@ -428,7 +431,7 @@ fn waitZigTest( var body_r: std.Io.Reader = .fixed(body); switch (header.tag) { .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( + if (!std.mem.eql(u8, builtin.zig_version_string, body)) return step.fail( maker, "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", .{ builtin.zig_version_string, body }, @@ -451,14 +454,14 @@ fn waitZigTest( const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; - options.progress_node.setEstimatedTotalItems(names.len); + progress_node.setEstimatedTotalItems(names.len); opt_metadata.* = .{ .string_bytes = try arena.dupe(u8, string_bytes), .ns_per_test = try arena.alloc(u64, results.test_count), .names = names, .expected_panic_msgs = expected_panic_msgs, .next_index = 0, - .prog_node = options.progress_node, + .prog_node = progress_node, }; @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); @@ -494,20 +497,20 @@ fn waitZigTest( const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); stderr.tossBuffered(); if (stderr_bytes.len == 0) { - try run.step.addError("'{s}' failed without output", .{name}); + try step.addError(maker, "'{s}' failed without output", .{name}); } else { - try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); + try step.addError(maker, "'{s}' failed:\n{s}", .{ name, stderr_bytes }); } } else if (leak_count > 0) { const name = md.testName(tr_hdr.index); const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); stderr.tossBuffered(); - try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); + try step.addError(maker, "'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); } else if (log_err_count > 0) { const name = md.testName(tr_hdr.index); const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); stderr.tossBuffered(); - try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); + try step.addError(maker, "'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); } active_test_index = null; @@ -525,6 +528,7 @@ fn waitZigTest( const FuzzTestRunner = struct { run: *Run, + run_index: Configuration.Step.Index, ctx: FuzzContext, coverage_id: ?u64, @@ -572,16 +576,18 @@ const FuzzTestRunner = struct { fn init( run: *Run, + run_index: Configuration.Step.Index, ctx: FuzzContext, progress_node: std.Progress.Node, spawn_options: process.SpawnOptions, ) !FuzzTestRunner { - const step_owner = run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; + const maker = ctx.fuzz.maker; + const graph = maker.graph; + const gpa = maker.gpa; + const io = graph.io; const n_instances = switch (ctx.fuzz.mode) { - .forever => step_owner.graph.max_jobs orelse @min( + .forever => graph.max_jobs orelse @min( std.Thread.getCpuCount() catch 1, (std.math.maxInt(u32) - 2) / 3, ), @@ -613,6 +619,7 @@ const FuzzTestRunner = struct { return .{ .run = run, + .run_index = run_index, .ctx = ctx, .coverage_id = null, @@ -625,9 +632,13 @@ const FuzzTestRunner = struct { } fn deinit(f: *FuzzTestRunner) void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; + const maker = f.ctx.fuzz.maker; + const run_index = f.run_index; + + const graph = maker.graph; + const gpa = maker.gpa; + const io = graph.io; + const step = maker.stepByIndex(run_index); f.batch.cancel(io); gpa.free(f.batch.storage); @@ -639,13 +650,18 @@ const FuzzTestRunner = struct { instance.progress_node.end(); total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; } - f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); + step.result_peak_rss = @max(step.result_peak_rss, total_rss); gpa.free(f.instances); } fn startInstances(f: *FuzzTestRunner) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; + const maker = f.ctx.fuzz.maker; + const run_index = f.run_index; + const run = f.run; + + const graph = maker.graph; + const io = graph.io; + const step = maker.stepByIndex(run_index); for (0.., f.instances) |id, *instance| { const id32: u32 = @intCast(id); @@ -653,14 +669,14 @@ const FuzzTestRunner = struct { .forever => sendRunFuzzTestMessage( io, instance.child.stdin.?, - f.run.fuzz_tests.items, + run.fuzz_tests.items, .forever, id32, ), .limit => |limit| sendRunFuzzTestMessage( io, instance.child.stdin.?, - f.run.fuzz_tests.items, + run.fuzz_tests.items, .iterations, limit.amount, ), @@ -670,7 +686,8 @@ const FuzzTestRunner = struct { instance.child.stdin.?.close(io); instance.child.stdin = null; const term = try instance.child.wait(io); - return f.run.step.fail( + return step.fail( + maker, "unable to write stdin ({t}); test process unexpectedly {f}", .{ write_err, fmtTerm(term) }, ); @@ -682,8 +699,9 @@ const FuzzTestRunner = struct { } fn listen(f: *FuzzTestRunner) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; + const maker = f.ctx.fuzz.maker; + const graph = maker.graph; + const io = graph.io; while (true) { try f.batch.awaitConcurrent(io, .none); @@ -714,10 +732,15 @@ const FuzzTestRunner = struct { } fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; + const maker = f.ctx.fuzz.maker; const instance = &f.instances[id]; + const run_index = f.run_index; + const run = f.run; + + const graph = maker.graph; + const gpa = maker.gpa; + const io = graph.io; + const step = maker.stepByIndex(run_index); instance.message.items.len += n; const total_read = instance.message.items.len; @@ -735,7 +758,8 @@ const FuzzTestRunner = struct { switch (header.tag) { .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( + if (!std.mem.eql(u8, builtin.zig_version_string, body)) return step.fail( + maker, "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", .{ builtin.zig_version_string, body }, ); @@ -750,14 +774,14 @@ const FuzzTestRunner = struct { const fuzz = f.ctx.fuzz; fuzz.queue_mutex.lockUncancelable(io); defer fuzz.queue_mutex.unlock(io); - try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ + try fuzz.msg_queue.append(gpa, .{ .coverage = .{ .id = f.coverage_id.?, .cumulative = .{ .runs = cumulative_runs, .unique = cumulative_unique, .coverage = cumulative_coverage, }, - .run = f.run, + .run = run_index, } }); fuzz.queue_cond.signal(io); }, @@ -768,7 +792,7 @@ const FuzzTestRunner = struct { fuzz.queue_mutex.lockUncancelable(io); defer fuzz.queue_mutex.unlock(io); - try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ + try fuzz.msg_queue.append(gpa, .{ .entry_point = .{ .addr = addr, .coverage_id = f.coverage_id.?, } }); @@ -776,7 +800,7 @@ const FuzzTestRunner = struct { }, .fuzz_test_change => { const test_i = std.mem.readInt(u32, body[0..4], .little); - instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); + instance.progress_node.setName(run.fuzz_tests.items[test_i]); }, .broadcast_fuzz_input => { if (f.instances.len == 1) { @@ -823,8 +847,8 @@ const FuzzTestRunner = struct { } fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; + const maker = f.ctx.fuzz.maker; + const gpa = maker.gpa; const instance = &f.instances[id]; try instance.message.ensureTotalCapacity(gpa, end); @@ -837,8 +861,8 @@ const FuzzTestRunner = struct { } fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; + const maker = f.ctx.fuzz.maker; + const gpa = maker.gpa; const instance = &f.instances[id]; try instance.stderr.ensureUnusedCapacity(gpa, 1); @@ -861,24 +885,31 @@ const FuzzTestRunner = struct { } fn instanceEos(f: *FuzzTestRunner, id: u32) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; + const maker = f.ctx.fuzz.maker; const instance = &f.instances[id]; + const run_index = f.run_index; + + const graph = maker.graph; + const io = graph.io; + const step = maker.stepByIndex(run_index); instance.child.stdin.?.close(io); instance.child.stdin = null; const term = try instance.child.wait(io); if (!termMatches(.{ .exited = 0 }, term)) { - f.run.step.result_stderr = try f.mergedStderr(); + step.result_stderr = try f.mergedStderr(); try f.saveCrash(id, term); - return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); + return step.fail(maker, "test process unexpectedly {f}", .{fmtTerm(term)}); } } fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { - const fuzz = f.context.fuzz; + const fuzz = f.ctx.fuzz; + const run_index = f.run_index; + const run = f.run; + const maker = fuzz.maker; - const step = &f.run.step; + const step = maker.stepByIndex(run_index); const graph = maker.graph; const io = graph.io; const cache_root = graph.local_cache_root; @@ -906,7 +937,7 @@ const FuzzTestRunner = struct { error.FileNotFound => return, error.WouldBlock => continue, // Can not be from // the crashed instance since it is still locked. - else => return step.fail("failed to open file '{f}{s}': {t}", .{ + else => return step.fail(maker, "failed to open file '{f}{s}': {t}", .{ cache_root, in_name, e, }), }; @@ -915,7 +946,7 @@ const FuzzTestRunner = struct { const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { in_f.close(io); switch (e) { - error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ + error.ReadFailed => return step.fail(maker, "failed to read file '{f}{s}': {t}", .{ cache_root, in_name, in_r.err.?, }), error.EndOfStream => continue, @@ -924,7 +955,7 @@ const FuzzTestRunner = struct { if (header.pc_digest == f.coverage_id.? and header.instance_id == id and - header.test_i < f.run.fuzz_tests.items.len) + header.test_i < run.fuzz_tests.items.len) { break header; } @@ -937,7 +968,7 @@ const FuzzTestRunner = struct { const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; const out = cache_root.handle.createFile(io, crash_name, .{ .lock = .exclusive, // Multiple run steps could have found a crash at the same time - }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ + }) catch |e| return step.fail(maker, "failed to create file '{f}{s}': {t}", .{ cache_root, crash_name, e, }); defer out.close(io); @@ -945,16 +976,16 @@ const FuzzTestRunner = struct { var out_w_buf: [512]u8 = undefined; var out_w = out.writerStreaming(io, &out_w_buf); _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { - error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ + error.ReadFailed => return step.fail(maker, "failed to read file '{f}{s}': {t}", .{ cache_root, in_name, in_r.err.?, }), - error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ + error.WriteFailed => return step.fail(maker, "failed to write file '{f}{s}': {t}", .{ cache_root, crash_name, out_w.err.?, }), }; - return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ - f.run.fuzz_tests.items[header.test_i], + return step.fail(maker, "test '{s}' {f}; input saved to '{f}{s}'", .{ + run.fuzz_tests.items[header.test_i], fmtTerm(term), cache_root, crash_name, @@ -967,8 +998,8 @@ const FuzzTestRunner = struct { assert(f.broadcast.items.len == 0); assert(from_id < f.instances.len); - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; + const maker = f.ctx.fuzz.maker; + const gpa = maker.gpa; var out_header: OutHeader = .{ .tag = .new_fuzz_input, @@ -1010,8 +1041,9 @@ const FuzzTestRunner = struct { } fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { - const step_owner = f.run.step.owner; - const arena = step_owner.allocator; + const maker = f.ctx.fuzz.maker; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena // Collect any available stderr while (f.batch.next()) |completion| { @@ -1035,11 +1067,12 @@ const FuzzTestRunner = struct { fn evalFuzzTest( run: *Run, + run_index: Configuration.Step.Index, + progress_node: std.Progress.Node, spawn_options: process.SpawnOptions, - options: Step.MakeOptions, fuzz_context: FuzzContext, ) !void { - var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); + var f: FuzzTestRunner = try .init(run, run_index, fuzz_context, progress_node, spawn_options); defer f.deinit(); try f.startInstances(); try f.listen(); @@ -1049,23 +1082,25 @@ const StdioPollEnum = enum { stdout, stderr }; fn evalZigTest( run: *Run, + run_index: Configuration.Step.Index, maker: *Maker, + progress_node: std.Progress.Node, spawn_options: process.SpawnOptions, - options: Step.MakeOptions, fuzz_context: ?FuzzContext, ) !void { if (fuzz_context != null) { - try evalFuzzTest(run, spawn_options, options, fuzz_context.?); + try evalFuzzTest(run, run_index, progress_node, spawn_options, fuzz_context.?); return; } - const step_owner = run.step.owner; - const gpa = step_owner.allocator; - const arena = step_owner.allocator; - const io = step_owner.graph.io; + const graph = maker.graph; + const gpa = maker.gpa; + const io = graph.io; + const arena = graph.arena; // TODO don't leak into the process arena + const step = maker.stepByIndex(run_index); // We will update this every time a child runs. - run.step.result_peak_rss = 0; + step.result_peak_rss = 0; var test_results: Step.TestResults = .{ .test_count = 0, @@ -1087,16 +1122,18 @@ fn evalZigTest( defer if (!child_killed) { child.kill(io); multi_reader.deinit(); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, + step.result_peak_rss = @max( + step.result_peak_rss, child.resource_usage_statistics.getMaxRss() orelse 0, ); }; switch (try waitZigTest( run, + run_index, + maker, &child, - options, + progress_node, &multi_reader, &test_metadata, &test_results, @@ -1109,7 +1146,7 @@ fn evalZigTest( error.ReadFailed => return stderr_fr.err.?, error.EndOfStream => {}, } - run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); + step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); // Clean up everything and wait for the child to exit. child.stdin.?.close(io); @@ -1117,14 +1154,16 @@ fn evalZigTest( multi_reader.deinit(); child_killed = true; const term = try child.wait(io); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, + step.result_peak_rss = @max( + step.result_peak_rss, child.resource_usage_statistics.getMaxRss() orelse 0, ); // The individual unit test results are irrelevant: the test runner itself broke! // Fail immediately without populating `s.test_results`. - return run.step.fail(maker, "unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); + return step.fail(maker, "unable to write stdin ({t}); test process unexpectedly {f}", .{ + err, fmtTerm(term), + }); }, .no_poll => |no_poll| { // This might be a success (we requested exit and the child dutifully closed stdout) or @@ -1138,8 +1177,8 @@ fn evalZigTest( multi_reader.deinit(); child_killed = true; const term = try child.wait(io); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, + step.result_peak_rss = @max( + step.result_peak_rss, child.resource_usage_statistics.getMaxRss() orelse 0, ); @@ -1148,7 +1187,7 @@ fn evalZigTest( // test, and continue to the next test. test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; test_results.crash_count += 1; - try run.step.addError("'{s}' {f}{s}{s}", .{ + try step.addError(maker, "'{s}' {f}{s}{s}", .{ test_metadata.?.testName(test_index), fmtTerm(term), if (stderr_owned.len != 0) " with stderr:\n" else "", @@ -1158,22 +1197,22 @@ fn evalZigTest( } // Report an error if the child terminated uncleanly or if we were still trying to run more tests. - run.step.result_stderr = stderr_owned; + step.result_stderr = stderr_owned; const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { // The individual unit test results are irrelevant: the test runner itself broke! // Fail immediately without populating `s.test_results`. - return run.step.fail(maker, "test process unexpectedly {f}", .{fmtTerm(term)}); + return step.fail(maker, "test process unexpectedly {f}", .{fmtTerm(term)}); } // We're done with all of the tests! Commit the test results and return. - run.step.test_results = test_results; + step.test_results = test_results; if (test_metadata) |tm| { run.cached_test_metadata = tm.toCachedTestMetadata(); - if (options.web_server) |ws| { - if (run.step.owner.graph.time_report) { + if (maker.web_server) |*ws| { + if (graph.time_report) { ws.updateTimeReportRunTest( - run, + run_index, &run.cached_test_metadata.?, tm.ns_per_test, ); @@ -1191,7 +1230,7 @@ fn evalZigTest( // the next test. test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; test_results.timeout_count += 1; - try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ + try step.addError(maker, "'{s}' timed out after {f}{s}{s}", .{ test_metadata.?.testName(test_index), Io.Duration{ .nanoseconds = timeout.ns_elapsed }, if (stderr.len != 0) " with stderr:\n" else "", @@ -1200,10 +1239,10 @@ fn evalZigTest( continue; } // Just log an error and let the child be killed. - run.step.result_stderr = try arena.dupe(u8, stderr); + step.result_stderr = try arena.dupe(u8, stderr); // The individual unit test results in `results` are irrelevant: the test runner // is broken! Fail immediately without populating `s.test_results`. - return run.step.fail(maker, "test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); + return step.fail(maker, "test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); }, } comptime unreachable; @@ -1323,27 +1362,35 @@ fn sendRunFuzzTestMessage( } } -fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !EvalGenericResult { +fn evalGeneric( + run_index: Configuration.Step.Index, + maker: *Maker, + spawn_options: process.SpawnOptions, +) !EvalGenericResult { const graph = maker.graph; const io = graph.io; - const arena = graph.allocator; // TODO don't leak into the process arena + const arena = graph.arena; // TODO don't leak into the process arena const gpa = maker.gpa; + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; + const step = maker.stepByIndex(run_index); var child = try process.spawn(io, spawn_options); defer child.kill(io); - switch (run.stdin) { + switch (conf_run.stdin.u) { .bytes => |bytes| { - child.stdin.?.writeStreamingAll(io, bytes) catch |err| { - return run.step.fail(maker, "unable to write stdin: {t}", .{err}); + child.stdin.?.writeStreamingAll(io, bytes.slice(conf)) catch |err| { + return step.fail(maker, "failed to write stdin: {t}", .{err}); }; child.stdin.?.close(io); child.stdin = null; }, .lazy_path => |lazy_path| { - const path = lazy_path.getPath3(graph, &run.step); + const path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index); const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { - return run.step.fail(maker, "unable to open stdin file: {t}", .{err}); + return step.fail(maker, "failed to open stdin file: {t}", .{err}); }; defer file.close(io); // TODO https://github.com/ziglang/zig/issues/23955 @@ -1352,15 +1399,15 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E var write_buffer: [1024]u8 = undefined; var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { - error.ReadFailed => return run.step.fail(maker, "failed to read from {f}: {t}", .{ + error.ReadFailed => return step.fail(maker, "failed to read from {f}: {t}", .{ path, file_reader.err.?, }), - error.WriteFailed => return run.step.fail(maker, "failed to write to stdin: {t}", .{ + error.WriteFailed => return step.fail(maker, "failed to write to stdin: {t}", .{ stdin_writer.err.?, }), }; stdin_writer.interface.flush() catch |err| switch (err) { - error.WriteFailed => return run.step.fail(maker, "failed to write to stdin: {t}", .{ + error.WriteFailed => return step.fail(maker, "failed to write to stdin: {t}", .{ stdin_writer.err.?, }), }; @@ -1384,7 +1431,7 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E const stderr_reader = multi_reader.reader(1); while (multi_reader.fill(64, .none)) |_| { - if (run.stdio_limit.toInt()) |limit| { + if (conf_run.stdio_limit.value) |limit| { if (stdout_reader.buffered().len > limit) return error.StdoutStreamTooLong; if (stderr_reader.buffered().len > limit) @@ -1404,7 +1451,8 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E stderr_bytes = try multi_reader.toOwnedSlice(1); } else { var stdout_reader = stdout.readerStreaming(io, &.{}); - stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { + const stdio_limit: Io.Limit = if (conf_run.stdio_limit.value) |x| .limited(x) else .unlimited; + stdout_bytes = stdout_reader.interface.allocRemaining(arena, stdio_limit) catch |err| switch (err) { error.OutOfMemory => |e| return e, error.ReadFailed => return stdout_reader.err.?, error.StreamTooLong => return error.StdoutStreamTooLong, @@ -1412,7 +1460,8 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E } } else if (child.stderr) |stderr| { var stderr_reader = stderr.readerStreaming(io, &.{}); - stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { + const stdio_limit: Io.Limit = if (conf_run.stdio_limit.value) |x| .limited(x) else .unlimited; + stderr_bytes = stderr_reader.interface.allocRemaining(arena, stdio_limit) catch |err| switch (err) { error.OutOfMemory => |e| return e, error.ReadFailed => return stderr_reader.err.?, error.StreamTooLong => return error.StderrStreamTooLong, @@ -1421,16 +1470,16 @@ fn evalGeneric(run: *Run, maker: *Maker, spawn_options: process.SpawnOptions) !E if (stderr_bytes) |bytes| if (bytes.len > 0) { // Treat stderr as an error message. - const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { - .check => |checks| !checksContainStderr(checks.items), + const stderr_is_diagnostic = conf_run.captured_stderr.value == null and switch (conf_run.flags.stdio) { + .check => !checksContainStderr(&conf_run), else => true, }; if (stderr_is_diagnostic) { - run.step.result_stderr = bytes; + step.result_stderr = bytes; } }; - run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; + step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; return .{ .term = try child.wait(io), @@ -1452,7 +1501,7 @@ pub fn rerunInFuzzMode( ) !void { const maker = fuzz.maker; const graph = maker.graph; - const step = &run.step; + const step = maker.stepByIndex(run_index); const io = graph.io; const arena = graph.arena; // TODO don't leak into the process arena const gpa = maker.gpa; @@ -1535,9 +1584,9 @@ pub fn rerunInFuzzMode( } } - if (run.step.result_failed_command) |cmd| { - fuzz.gpa.free(cmd); - run.step.result_failed_command = null; + if (step.result_failed_command) |cmd| { + gpa.free(cmd); + step.result_failed_command = null; } const has_side_effects = false; @@ -1549,8 +1598,6 @@ pub fn rerunInFuzzMode( }); } -const CapturedStdIo = void; // TODO get it from Configuration - fn populateGeneratedPaths( maker: *Maker, output_placeholders: []const IndexedOutput, @@ -1712,142 +1759,153 @@ fn runCommand( if (true) @panic("TODO"); - const opt_generic_result = spawnChildAndCollect(run_index, run, maker, progress_node, argv, &environ_map, has_side_effects, fuzz_context) catch |err| term: { - // InvalidExe: cpu arch mismatch - // FileNotFound: can happen with a wrong dynamic linker path - if (err == error.InvalidExe or err == error.FileNotFound) interpret: { - // TODO: learn the target from the binary directly rather than from - // relying on it being a Compile step. This will make this logic - // work even for the edge case that the binary was produced by a - // third party. - const exe = switch (run.argv.items[0]) { - .artifact => |exe| exe.artifact, - else => break :interpret, - }; - switch (exe.kind) { - .exe, .@"test" => {}, - else => break :interpret, - } + const opt_generic_result = spawnChildAndCollect( + run_index, + run, + maker, + progress_node, + argv, + environ_map, + has_side_effects, + fuzz_context, + ) catch |err| term: { + switch (err) { + error.InvalidExe, // cpu arch mismatch + error.FileNotFound, // can happen with a wrong dynamic linker path + => interpret: { + // TODO: learn the target from the binary directly rather than from + // relying on it being a Compile step. This will make this logic + // work even for the edge case that the binary was produced by a + // third party. + const exe = switch (run.argv.items[0]) { + .artifact => |exe| exe.artifact, + else => break :interpret, + }; + switch (exe.kind) { + .exe, .@"test" => {}, + else => break :interpret, + } - const root_target = exe.rootModuleTarget(); - const need_cross_libc = exe.is_linking_libc and - (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); - const other_target = exe.root_module.resolved_target.?.result; - switch (std.zig.system.getExternalExecutor(io, &graph.host.result, &other_target, .{ - .qemu_fixes_dl = need_cross_libc and graph.libc_runtimes_dir != null, - .link_libc = exe.is_linking_libc, - })) { - .native, .rosetta => { - if (allow_skip) return error.MakeSkipped; - break :interpret; - }, - .wine => |bin_name| { - if (graph.enable_wine) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); + const root_target = exe.rootModuleTarget(); + const need_cross_libc = exe.is_linking_libc and + (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); + const other_target = exe.root_module.resolved_target.?.result; + switch (std.zig.system.getExternalExecutor(io, &graph.host.result, &other_target, .{ + .qemu_fixes_dl = need_cross_libc and graph.libc_runtimes_dir != null, + .link_libc = exe.is_linking_libc, + })) { + .native, .rosetta => { + if (allow_skip) return error.MakeSkipped; + break :interpret; + }, + .wine => |bin_name| { + if (graph.enable_wine) { + try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); - // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but - // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. - if (environ_map.get("WINEDEBUG") == null) { - try environ_map.put("WINEDEBUG", "-all"); + // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but + // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. + if (environ_map.get("WINEDEBUG") == null) { + try environ_map.put("WINEDEBUG", "-all"); + } + } else { + return failForeign(conf_run, maker, run_index, "-fwine", argv[0], exe); } - } else { - return failForeign(run, "-fwine", argv[0], exe); - } - }, - .qemu => |bin_name| { - if (graph.enable_qemu) { - try interp_argv.append(bin_name); + }, + .qemu => |bin_name| { + if (graph.enable_qemu) { + try interp_argv.append(bin_name); - if (need_cross_libc) { - if (graph.libc_runtimes_dir) |dir| { - try interp_argv.append("-L"); - try interp_argv.append(try Dir.path.join(arena, &.{ - dir, - try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( - arena, - root_target.cpu.arch, - root_target.os.tag, - root_target.abi, - ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( - arena, - root_target.cpu.arch, - root_target.abi, - ) else unreachable, - })); - } else return failForeign(run, "--libc-runtimes", argv[0], exe); - } + if (need_cross_libc) { + if (graph.libc_runtimes_dir) |dir| { + try interp_argv.append("-L"); + try interp_argv.append(try Dir.path.join(arena, &.{ + dir, + try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( + arena, + root_target.cpu.arch, + root_target.os.tag, + root_target.abi, + ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( + arena, + root_target.cpu.arch, + root_target.abi, + ) else unreachable, + })); + } else return failForeign(conf_run, maker, run_index, "--libc-runtimes", argv[0], exe); + } - try interp_argv.appendSlice(argv); - } else return failForeign(run, "-fqemu", argv[0], exe); - }, - .darling => |bin_name| { - if (graph.enable_darling) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); - } else { - return failForeign(run, "-fdarling", argv[0], exe); - } - }, - .wasmtime => |bin_name| { - if (graph.enable_wasmtime) { - try interp_argv.append(bin_name); - try interp_argv.append("--dir=."); - // Wasmtime doeesn't inherit environment variables from the parent process - // by default. '-S inherit-env' was added in Wasmtime version 20. - try interp_argv.append("-Sinherit-env"); - try interp_argv.append(argv[0]); - try interp_argv.appendSlice(argv[1..]); - } else { - return failForeign(run, "-fwasmtime", argv[0], exe); - } - }, - .bad_dl => |foreign_dl| { - if (allow_skip) return error.MakeSkipped; + try interp_argv.appendSlice(argv); + } else return failForeign(conf_run, maker, run_index, "-fqemu", argv[0], exe); + }, + .darling => |bin_name| { + if (graph.enable_darling) { + try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); + } else { + return failForeign(conf_run, maker, run_index, "-fdarling", argv[0], exe); + } + }, + .wasmtime => |bin_name| { + if (graph.enable_wasmtime) { + try interp_argv.append(bin_name); + try interp_argv.append("--dir=."); + // Wasmtime doeesn't inherit environment variables from the parent process + // by default. '-S inherit-env' was added in Wasmtime version 20. + try interp_argv.append("-Sinherit-env"); + try interp_argv.append(argv[0]); + try interp_argv.appendSlice(argv[1..]); + } else { + return failForeign(conf_run, maker, run_index, "-fwasmtime", argv[0], exe); + } + }, + .bad_dl => |foreign_dl| { + if (allow_skip) return error.MakeSkipped; - const host_dl = graph.host.result.dynamic_linker.get() orelse "(none)"; + const host_dl = graph.host.result.dynamic_linker.get() orelse "(none)"; - return step.fail(maker, - \\the host system is unable to execute binaries from the target - \\ because the host dynamic linker is '{s}', - \\ while the target dynamic linker is '{s}'. - \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step - , .{ host_dl, foreign_dl }); - }, - .bad_os_or_cpu => { - if (allow_skip) return error.MakeSkipped; + return step.fail(maker, + \\the host system is unable to execute binaries from the target + \\ because the host dynamic linker is '{s}', + \\ while the target dynamic linker is '{s}'. + \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step + , .{ host_dl, foreign_dl }); + }, + .bad_os_or_cpu => { + if (allow_skip) return error.MakeSkipped; - const host_name = try graph.host.result.zigTriple(arena); - const foreign_name = try root_target.zigTriple(arena); + const host_name = try graph.host.result.zigTriple(arena); + const foreign_name = try root_target.zigTriple(arena); - return step.fail(maker, "the host system ({s}) is unable to execute binaries from the target ({s})", .{ - host_name, foreign_name, - }); - }, - } + return step.fail(maker, "the host system ({s}) is unable to execute binaries from the target ({s})", .{ + host_name, foreign_name, + }); + }, + } - if (root_target.os.tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - addPathForDynLibs(exe); - } + if (root_target.os.tag == .windows) { + // On Windows we don't have rpaths so we have to add .dll search paths to PATH + addPathForDynLibs(exe); + } - gpa.free(step.result_failed_command.?); - step.result_failed_command = null; - try Step.handleVerbose(step.owner, cwd, run.environ_map, interp_argv.items); + gpa.free(step.result_failed_command.?); + step.result_failed_command = null; + try graph.handleVerbose(cwd, run.environ_map, interp_argv.items); - break :term spawnChildAndCollect(run_index, run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { - if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; - if (e == error.MakeFailed) return error.MakeFailed; // error already reported - return step.fail(maker, "unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); - }; + break :term spawnChildAndCollect(run_index, run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { + if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; + if (e == error.MakeFailed) return error.MakeFailed; // error already reported + return step.fail(maker, "unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); + }; + }, + error.MakeFailed, error.OutOfMemory, error.Canceled => |e| return e, + else => {}, } - if (err == error.MakeFailed) return error.MakeFailed; // error already reported - return step.fail(maker, "failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); }; const generic_result = opt_generic_result orelse { - assert(run.stdio == .zig_test); + assert(conf_run.flags.stdio == .zig_test); // Specific errors have already been reported, and test results are populated. All we need // to do is report step failure if any test failed. if (!step.test_results.isSuccess()) return error.MakeFailed; @@ -1855,20 +1913,20 @@ fn runCommand( }; assert(fuzz_context == null); - assert(run.stdio != .zig_test); + assert(conf_run.flags.stdio != .zig_test); // Capture stdout and stderr to GeneratedFile objects. const Stream = struct { - captured: ?*CapturedStdIo, + captured: ?Configuration.Step.Run.CapturedStream, bytes: ?[]const u8, }; for ([_]Stream{ .{ - .captured = run.captured_stdout, + .captured = conf_run.captured_stdout.value, .bytes = generic_result.stdout, }, .{ - .captured = run.captured_stderr, + .captured = conf_run.captured_stderr.value, .bytes = generic_result.stderr, }, }) |stream| { @@ -1898,7 +1956,7 @@ fn runCommand( } } - switch (run.stdio) { + switch (conf_run.flags.stdio) { .zig_test => unreachable, .check => |checks| for (checks.items) |check| switch (check) { .expect_stderr_exact => |expected_bytes| { @@ -1970,7 +2028,7 @@ fn runCommand( }; if (bad_exit) { if (generic_result.stderr) |bytes| { - run.step.result_stderr = bytes; + step.result_stderr = bytes; } } @@ -1995,9 +2053,8 @@ fn spawnChildAndCollect( has_side_effects: bool, fuzz_context: ?FuzzContext, ) !?EvalGenericResult { - const step = run.step; + const step = maker.stepByIndex(run_index); const graph = maker.graph; - const gpa = maker.gpa; const io = graph.io; const arena = graph.arena; // TODO don't leak into process arena const conf = &maker.scanned_config.configuration; @@ -2006,17 +2063,17 @@ fn spawnChildAndCollect( if (fuzz_context != null) { assert(!has_side_effects); - assert(run.stdio == .zig_test); + assert(conf_run.flags.stdio == .zig_test); } - const child_cwd: process.Child.Cwd = if (conf_run.cwd) |lazy_cwd| + const child_cwd: process.Child.Cwd = if (conf_run.cwd.value) |lazy_cwd| .{ .path = try maker.resolveLazyPathIndexAbs(arena, lazy_cwd, run_index) } else .inherit; // If an error occurs, it's caused by this command: assert(step.result_failed_command == null); - step.result_failed_command = try Step.allocPrintCmd(gpa, child_cwd, .{ + step.result_failed_command = try std.zig.allocPrintCmd(arena, child_cwd, .{ .child = environ_map, .parent = &graph.environ_map, }, argv); @@ -2028,22 +2085,22 @@ fn spawnChildAndCollect( .cwd = child_cwd, .environ_map = environ_map, .request_resource_usage_statistics = true, - .stdin = if (run.stdin != .none) s: { - assert(run.stdio != .inherit); + .stdin = if (conf_run.stdin.u != .none) s: { + assert(conf_run.flags.stdio != .inherit); break :s .pipe; - } else switch (run.stdio) { + } else switch (conf_run.flags.stdio) { .infer_from_args => if (has_side_effects) .inherit else .ignore, .inherit => .inherit, .check => .ignore, .zig_test => .pipe, }, - .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { + .stdout = if (conf_run.captured_stdout.value != null) .pipe else switch (conf_run.flags.stdio) { .infer_from_args => if (has_side_effects) .inherit else .ignore, .inherit => .inherit, - .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, + .check => if (checksContainStdout(&conf_run)) .pipe else .ignore, .zig_test => .pipe, }, - .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { + .stderr = if (conf_run.captured_stderr.value != null) .pipe else switch (conf_run.flags.stdio) { .infer_from_args => if (has_side_effects) .inherit else .pipe, .inherit => .inherit, .check => .pipe, @@ -2051,9 +2108,9 @@ fn spawnChildAndCollect( }, }; - if (run.stdio == .zig_test) { + if (conf_run.flags.stdio == .zig_test) { const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalZigTest(run, maker, progress_node, spawn_options, fuzz_context) catch |err| switch (err) { + const result = evalZigTest(run, run_index, maker, progress_node, spawn_options, fuzz_context) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| e, }; @@ -2062,7 +2119,7 @@ fn spawnChildAndCollect( return null; } else { const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; - if (!run.disable_zig_progress and !inherit) { + if (!conf_run.flags.disable_zig_progress and !inherit) { spawn_options.progress_node = progress_node; } const terminal_mode: Io.Terminal.Mode = if (inherit) m: { @@ -2070,10 +2127,10 @@ fn spawnChildAndCollect( break :m stderr.terminal_mode; } else .no_color; defer if (inherit) io.unlockStderr(); - try setColorEnvironmentVariables(run, environ_map, terminal_mode); + try setColorEnvironmentVariables(&conf_run, environ_map, terminal_mode); const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalGeneric(run, maker, spawn_options) catch |err| switch (err) { + const result = evalGeneric(run_index, maker, spawn_options) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| e, }; @@ -2106,8 +2163,12 @@ fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { }; } -fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { - color: switch (run.color) { +fn setColorEnvironmentVariables( + conf_run: *const Configuration.Step.Run, + environ_map: *EnvMap, + terminal_mode: Io.Terminal.Mode, +) !void { + color: switch (conf_run.flags.color) { .manual => {}, .enable => { try environ_map.put("CLICOLOR_FORCE", "1"); @@ -2122,8 +2183,8 @@ fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: .escape_codes => continue :color .enable, }, .auto => { - const capture_stderr = run.captured_stderr != null or switch (run.stdio) { - .check => |checks| checksContainStderr(checks.items), + const capture_stderr = conf_run.captured_stderr.value != null or switch (conf_run.flags.stdio) { + .check => checksContainStderr(conf_run), .infer_from_args, .inherit, .zig_test => false, }; if (capture_stderr) { @@ -2135,53 +2196,12 @@ fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: } } -fn checksContainStdout(checks: []const @This().StdIo.Check) bool { - for (checks) |check| switch (check) { - .expect_stderr_exact, - .expect_stderr_match, - .expect_term, - => continue, - - .expect_stdout_exact, - .expect_stdout_match, - => return true, - }; - return false; -} - -fn checksContainStderr(checks: []const @This().StdIo.Check) bool { - for (checks) |check| switch (check) { - .expect_stdout_exact, - .expect_stdout_match, - .expect_term, - => continue, - - .expect_stderr_exact, - .expect_stderr_match, - => return true, - }; - return false; -} - -/// Returns whether the Run step has side effects *other than* updating the output arguments. -fn hasSideEffects(run: Run) bool { - if (run.has_side_effects) return true; - return switch (run.stdio) { - .infer_from_args => !run.hasAnyOutputArgs(), - .inherit => true, - .check => false, - .zig_test => false, - }; +fn checksContainStdout(conf_run: *const Configuration.Step.Run) bool { + return conf_run.expect_stdout_exact.value != null or conf_run.expect_stdout_match.slice.len != 0; } -fn hasAnyOutputArgs(run: Run) bool { - if (run.captured_stdout != null) return true; - if (run.captured_stderr != null) return true; - for (run.argv.items) |arg| switch (arg) { - .output_file, .output_directory => return true, - else => continue, - }; - return false; +fn checksContainStderr(conf_run: *const Configuration.Step.Run) bool { + return conf_run.expect_stderr_exact.value != null or conf_run.expect_stderr_match.slice.len != 0; } /// If `path` is cwd-relative, make it relative to the cwd of the child instead. @@ -2225,13 +2245,13 @@ fn addPathForDynLibs(artifact: Configuration.Step.Index) void { compile.isDynamicLibrary()) { @panic("TODO"); - //addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); + //addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, step)).?); } } } fn failForeign( - run: *Run, + conf_run: *const Configuration.Step.Run, maker: *Maker, step_index: Configuration.Step.Index, suggested_flag: []const u8, @@ -2239,9 +2259,9 @@ fn failForeign( exe: *Step.Compile, ) Step.ExtendedMakeError { const step = maker.stepByIndex(step_index); - switch (run.stdio) { + switch (conf_run.flags.stdio) { .check, .zig_test => { - if (run.skip_foreign_checks) return error.MakeSkipped; + if (conf_run.flags.skip_foreign_checks) return error.MakeSkipped; const graph = maker.graph; const process_arena = graph.arena; // TODO don't leak into process arena -- 2.54.0 From 0f3471eb6643140c67587c205aea6582f415dd06 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Apr 2026 15:36:33 -0700 Subject: [PATCH 064/179] maker: finish porting over run step --- BRANCH_TODO | 1 + lib/compiler/Maker/Graph.zig | 9 +- lib/compiler/Maker/Step/Compile.zig | 6 +- lib/compiler/Maker/Step/Run.zig | 199 ++++++++++++++++------------ lib/compiler/aro/aro/Driver.zig | 7 +- lib/compiler/configurer.zig | 1 + lib/std/Build/Configuration.zig | 37 +++++- lib/std/Target.zig | 4 +- lib/std/Target/Query.zig | 2 +- lib/std/zig.zig | 2 +- lib/std/zig/system.zig | 33 +++-- src/main.zig | 6 +- 12 files changed, 194 insertions(+), 113 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 7b28f34c27eaf74e21e7457de00cb0ac9b4606af..66c225ddadd80f507483992de56e2f60da790ac7 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -16,6 +16,7 @@ * restore the generated_compiler_rt_dyn_lib hack? * run args * https://codeberg.org/ziglang/zig/pulls/30762 +* get the target from the parent process instead ## Followup Issues * reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index cb963003491d0a29969402029cd2c868e93d2f4c..3a3136d8783c12e12424cceff48085aa6784f026 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -6,6 +6,7 @@ const Io = std.Io; const Allocator = std.mem.Allocator; const Configuration = std.Build.Configuration; const Path = std.Build.Cache.Path; +const Directory = std.Build.Cache.Directory; io: Io, /// Process lifetime. @@ -13,10 +14,10 @@ arena: Allocator, cache: std.Build.Cache, zig_exe: []const u8, environ_map: std.process.Environ.Map, -global_cache_root: std.Build.Cache.Directory, -local_cache_root: std.Build.Cache.Directory, -zig_lib_directory: std.Build.Cache.Directory, -build_root_directory: std.Build.Cache.Directory, +global_cache_root: Directory, +local_cache_root: Directory, +zig_lib_directory: Directory, +build_root_directory: Directory, pkg_root: Path, debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index d4d370bb80421b6d277ed1b120a059eac118f616..5beb76ae5078ee6ec336421c4c35feef09fe815d 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -22,6 +22,8 @@ zig_process: ?*Step.ZigProcess = null, zig_args: std.ArrayList([]const u8) = .empty, /// Populated by InstallArtifact. installed_path: ?Path = null, +/// Populated by `make`, used by `Run`. +is_linking_libc: bool = false, pub fn make( compile: *Compile, @@ -144,7 +146,7 @@ const ModuleListContext = struct { }; fn lowerZigArgs( - compile: *const Compile, + compile: *Compile, compile_index: Configuration.Step.Index, maker: *const Maker, zig_args: *std.ArrayList([]const u8), @@ -564,6 +566,8 @@ fn lowerZigArgs( try zig_args.ensureUnusedCapacity(gpa, 2); if (is_linking_libcpp) zig_args.appendAssumeCapacity("-lc++"); if (is_linking_libc) zig_args.appendAssumeCapacity("-lc"); + + compile.is_linking_libc = is_linking_libc; } if (conf_comp.win32_manifest.value) |manifest_file| { diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 81b46e031cf132e8c6bf5f1971624686eeede7d6..53d9c8cdb427508ba0fe89b4410c854df7666b38 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -1717,7 +1717,7 @@ fn runCommand( has_side_effects: bool, output_dir_path: []const u8, fuzz_context: ?FuzzContext, -) !void { +) Step.ExtendedMakeError!void { const graph = maker.graph; const arena = graph.arena; // TODO don't leak into process arena const gpa = maker.gpa; @@ -1757,8 +1757,6 @@ fn runCommand( } try graph.handleVerbose(cwd, environ_map, argv); - if (true) @panic("TODO"); - const opt_generic_result = spawnChildAndCollect( run_index, run, @@ -1777,22 +1775,33 @@ fn runCommand( // relying on it being a Compile step. This will make this logic // work even for the edge case that the binary was produced by a // third party. - const exe = switch (run.argv.items[0]) { - .artifact => |exe| exe.artifact, - else => break :interpret, - }; - switch (exe.kind) { + const arg0 = conf_run.args.slice[0].get(conf); + const producer_index = arg0.producer.value orelse break :interpret; + const producer_step = producer_index.ptr(conf); + const producer = producer_step.extended.get(conf.extra).compile; + switch (producer.flags3.kind) { .exe, .@"test" => {}, else => break :interpret, } + const root_module = producer.root_module.get(conf); + const root_module_target = root_module.resolved_target.get(conf).?.result.get(conf); + const other_target_query = root_module_target.unwrap(conf); + const root_target = std.zig.system.resolveTargetQuery(io, other_target_query) catch unreachable; + const link_libc = maker.stepByIndex(producer_index).extended.compile.is_linking_libc; - const root_target = exe.rootModuleTarget(); - const need_cross_libc = exe.is_linking_libc and - (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); - const other_target = exe.root_module.resolved_target.?.result; - switch (std.zig.system.getExternalExecutor(io, &graph.host.result, &other_target, .{ + // TODO get this from the parent process instead + const host: std.Target = std.zig.system.resolveTargetQuery(io, .{}) catch |he| switch (he) { + error.Canceled => |e| return e, + else => builtin.target, + }; + + const need_cross_libc = link_libc and root_target.os.tag == .linux and + producer.flags2.linkage == .dynamic; + switch (std.zig.system.getExternalExecutor(io, &root_target, .{ + .host_cpu_arch = host.cpu.arch, + .host_os_tag = host.os.tag, .qemu_fixes_dl = need_cross_libc and graph.libc_runtimes_dir != null, - .link_libc = exe.is_linking_libc, + .link_libc = link_libc, })) { .native, .rosetta => { if (allow_skip) return error.MakeSkipped; @@ -1800,8 +1809,9 @@ fn runCommand( }, .wine => |bin_name| { if (graph.enable_wine) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); + try interp_argv.ensureUnusedCapacity(arena, 1 + argv.len); + interp_argv.appendAssumeCapacity(bin_name); + interp_argv.appendSliceAssumeCapacity(argv); // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. @@ -1809,17 +1819,18 @@ fn runCommand( try environ_map.put("WINEDEBUG", "-all"); } } else { - return failForeign(conf_run, maker, run_index, "-fwine", argv[0], exe); + return failForeign(&conf_run, maker, run_index, "-fwine", argv[0], &root_target, &host); } }, .qemu => |bin_name| { if (graph.enable_qemu) { - try interp_argv.append(bin_name); + try interp_argv.ensureUnusedCapacity(arena, 3 + argv.len); + interp_argv.appendAssumeCapacity(bin_name); if (need_cross_libc) { if (graph.libc_runtimes_dir) |dir| { - try interp_argv.append("-L"); - try interp_argv.append(try Dir.path.join(arena, &.{ + interp_argv.appendAssumeCapacity("-L"); + interp_argv.appendAssumeCapacity(try Dir.path.join(arena, &.{ dir, try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( arena, @@ -1832,37 +1843,38 @@ fn runCommand( root_target.abi, ) else unreachable, })); - } else return failForeign(conf_run, maker, run_index, "--libc-runtimes", argv[0], exe); + } else return failForeign(&conf_run, maker, run_index, "--libc-runtimes", argv[0], &root_target, &host); } - try interp_argv.appendSlice(argv); - } else return failForeign(conf_run, maker, run_index, "-fqemu", argv[0], exe); + interp_argv.appendSliceAssumeCapacity(argv); + } else return failForeign(&conf_run, maker, run_index, "-fqemu", argv[0], &root_target, &host); }, .darling => |bin_name| { if (graph.enable_darling) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); + try interp_argv.ensureUnusedCapacity(arena, 1 + argv.len); + interp_argv.appendAssumeCapacity(bin_name); + interp_argv.appendSliceAssumeCapacity(argv); } else { - return failForeign(conf_run, maker, run_index, "-fdarling", argv[0], exe); + return failForeign(&conf_run, maker, run_index, "-fdarling", argv[0], &root_target, &host); } }, .wasmtime => |bin_name| { if (graph.enable_wasmtime) { - try interp_argv.append(bin_name); - try interp_argv.append("--dir=."); + try interp_argv.ensureUnusedCapacity(arena, 3 + argv.len); + interp_argv.appendAssumeCapacity(bin_name); + interp_argv.appendAssumeCapacity("--dir=."); // Wasmtime doeesn't inherit environment variables from the parent process // by default. '-S inherit-env' was added in Wasmtime version 20. - try interp_argv.append("-Sinherit-env"); - try interp_argv.append(argv[0]); - try interp_argv.appendSlice(argv[1..]); + interp_argv.appendAssumeCapacity("-Sinherit-env"); + interp_argv.appendSliceAssumeCapacity(argv); } else { - return failForeign(conf_run, maker, run_index, "-fwasmtime", argv[0], exe); + return failForeign(&conf_run, maker, run_index, "-fwasmtime", argv[0], &root_target, &host); } }, .bad_dl => |foreign_dl| { if (allow_skip) return error.MakeSkipped; - const host_dl = graph.host.result.dynamic_linker.get() orelse "(none)"; + const host_dl = host.dynamic_linker.get() orelse "(none)"; return step.fail(maker, \\the host system is unable to execute binaries from the target @@ -1874,7 +1886,7 @@ fn runCommand( .bad_os_or_cpu => { if (allow_skip) return error.MakeSkipped; - const host_name = try graph.host.result.zigTriple(arena); + const host_name = try host.zigTriple(arena); const foreign_name = try root_target.zigTriple(arena); return step.fail(maker, "the host system ({s}) is unable to execute binaries from the target ({s})", .{ @@ -1885,15 +1897,24 @@ fn runCommand( if (root_target.os.tag == .windows) { // On Windows we don't have rpaths so we have to add .dll search paths to PATH - addPathForDynLibs(exe); + addPathForDynLibs(producer_index); } gpa.free(step.result_failed_command.?); step.result_failed_command = null; - try graph.handleVerbose(cwd, run.environ_map, interp_argv.items); + try graph.handleVerbose(cwd, environ_map, interp_argv.items); - break :term spawnChildAndCollect(run_index, run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { - if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; + break :term spawnChildAndCollect( + run_index, + run, + maker, + progress_node, + interp_argv.items, + environ_map, + has_side_effects, + fuzz_context, + ) catch |e| { + if (!conf_run.flags.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; if (e == error.MakeFailed) return error.MakeFailed; // error already reported return step.fail(maker, "unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); }; @@ -1919,47 +1940,51 @@ fn runCommand( const Stream = struct { captured: ?Configuration.Step.Run.CapturedStream, bytes: ?[]const u8, + trim_whitespace: Configuration.Step.Run.TrimWhitespace, }; - for ([_]Stream{ + for (&[_]Stream{ .{ .captured = conf_run.captured_stdout.value, .bytes = generic_result.stdout, + .trim_whitespace = conf_run.flags.stdout_trim_whitespace, }, .{ .captured = conf_run.captured_stderr.value, .bytes = generic_result.stderr, + .trim_whitespace = conf_run.flags.stderr_trim_whitespace, }, - }) |stream| { + }) |*stream| { if (stream.captured) |captured| { - const output_components = .{ output_dir_path, captured.output.basename }; - const output_path = try cache_root.join(arena, &output_components); - captured.output.generated_file.path = output_path; - - const sub_path = try Dir.path.join(arena, &output_components); - const sub_path_dirname = Dir.path.dirname(sub_path).?; - cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ - cache_root, sub_path_dirname, err, - }); + const output_path: Path = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ + output_dir_path, captured.basename.slice(conf), + }), }; - const data = switch (captured.trim_whitespace) { + maker.generatedPath(captured.generated_file).* = output_path; + + const sub_path_parent = output_path.dirname().?; + sub_path_parent.root_dir.handle.createDirPath(io, sub_path_parent.sub_path) catch |err| + return step.fail(maker, "unable to make path {f}: {t}", .{ sub_path_parent, err }); + + const data = switch (stream.trim_whitespace) { .none => stream.bytes.?, .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), }; - cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { - return step.fail(maker, "unable to write file '{f}{s}': {t}", .{ - cache_root, sub_path, err, - }); - }; + output_path.root_dir.handle.writeFile(io, .{ + .sub_path = output_path.sub_path, + .data = data, + }) catch |err| return step.fail(maker, "unable to write file {f}: {t}", .{ output_path, err }); } } switch (conf_run.flags.stdio) { .zig_test => unreachable, - .check => |checks| for (checks.items) |check| switch (check) { - .expect_stderr_exact => |expected_bytes| { + .check => { + if (conf_run.expect_stderr_exact.value) |bytes| { + const expected_bytes = bytes.slice(conf); if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { return step.fail(maker, \\========= expected this stderr: ========= @@ -1971,8 +1996,23 @@ fn runCommand( generic_result.stderr.?, }); } - }, - .expect_stderr_match => |match| { + } + if (conf_run.expect_stdout_exact.value) |bytes| { + const expected_bytes = bytes.slice(conf); + if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { + return step.fail(maker, + \\========= expected this stdout: ========= + \\{s} + \\========= but found: ==================== + \\{s} + , .{ + expected_bytes, + generic_result.stdout.?, + }); + } + } + for (conf_run.expect_stderr_match.slice) |bytes| { + const match = bytes.slice(conf); if (mem.find(u8, generic_result.stderr.?, match) == null) { return step.fail(maker, \\========= expected to find in stderr: ========= @@ -1984,21 +2024,9 @@ fn runCommand( generic_result.stderr.?, }); } - }, - .expect_stdout_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { - return step.fail(maker, - \\========= expected this stdout: ========= - \\{s} - \\========= but found: ==================== - \\{s} - , .{ - expected_bytes, - generic_result.stdout.?, - }); - } - }, - .expect_stdout_match => |match| { + } + for (conf_run.expect_stdout_match.slice) |bytes| { + const match = bytes.slice(conf); if (mem.find(u8, generic_result.stdout.?, match) == null) { return step.fail(maker, \\========= expected to find in stdout: ========= @@ -2010,15 +2038,21 @@ fn runCommand( generic_result.stdout.?, }); } - }, - .expect_term => |expected_term| { + } + if (conf_run.expect_term_value.value) |expected_term_value| { + const expected_term: process.Child.Term = switch (conf_run.flags2.expect_term_status) { + .exited => .{ .exited = @intCast(expected_term_value) }, + .signal => .{ .signal = @enumFromInt(expected_term_value) }, + .stopped => .{ .stopped = @enumFromInt(expected_term_value) }, + .unknown => .{ .unknown = expected_term_value }, + }; if (!termMatches(expected_term, generic_result.term)) { return step.fail(maker, "process {f} (expected {f})", .{ fmtTerm(generic_result.term), fmtTerm(expected_term), }); } - }, + } }, else => { // On failure, report captured stderr like normal standard error output. @@ -2032,7 +2066,7 @@ fn runCommand( } } - try step.handleChildProcessTerm(generic_result.term); + try step.handleChildProcessTerm(maker, generic_result.term); }, } } @@ -2256,7 +2290,8 @@ fn failForeign( step_index: Configuration.Step.Index, suggested_flag: []const u8, argv0: []const u8, - exe: *Step.Compile, + artifact_target: *const std.Target, + host_target: *const std.Target, ) Step.ExtendedMakeError { const step = maker.stepByIndex(step_index); switch (conf_run.flags.stdio) { @@ -2265,8 +2300,8 @@ fn failForeign( const graph = maker.graph; const process_arena = graph.arena; // TODO don't leak into process arena - const host_name = try graph.host.result.zigTriple(process_arena); - const foreign_name = try exe.rootModuleTarget().zigTriple(process_arena); + const host_name = try host_target.zigTriple(process_arena); + const foreign_name = try artifact_target.zigTriple(process_arena); return step.fail(maker, \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) diff --git a/lib/compiler/aro/aro/Driver.zig b/lib/compiler/aro/aro/Driver.zig index f1bf6b8d4e62e56a924ef4954eddc784296e5239..269be075fb3b225e6bf1cd923b2e022e13d1f8b3 100644 --- a/lib/compiler/aro/aro/Driver.zig +++ b/lib/compiler/aro/aro/Driver.zig @@ -1041,9 +1041,10 @@ fn parseTarget(d: *Driver, arch_os_abi: []const u8, opt_cpu_features: ?[]const u } else if (mem.eql(u8, cpu_name, "baseline")) { query.cpu_model = .baseline; } else { - query.cpu_model = .{ .explicit = arch.parseCpuModel(cpu_name) catch |er| switch (er) { - error.UnknownCpuModel => return d.fatal("unknown CPU model: '{s}'", .{cpu_name}), - } }; + query.cpu_model = .{ + .explicit = arch.parseCpuModel(cpu_name) orelse + return d.fatal("unknown CPU model: '{s}'", .{cpu_name}), + }; } if (opt_sub_arch) |sub_arch| { diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 7815198bcaceadaea247f7ea5889c8a832511aaf..e99f6384eff5e0fb08a6769a99e51f32a4ca8b4b 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -83,6 +83,7 @@ pub fn main(init: process.Init.Minimal) !void { .environ_map = try init.environ.createMap(arena), .global_cache_root = global_cache_directory, .zig_lib_directory = zig_lib_directory, + // TODO get this from parent process instead .host = .{ .query = .{}, .result = try std.zig.system.resolveTargetQuery(io, .{}), diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 4c81bdb9eeff53d1528a0097ab161e3ce90ad3e2..eb1736db1d26d6505d04846abffc2b4b18f5c2c0 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1797,7 +1797,7 @@ pub const TargetQuery = struct { } pub fn get(this: @This(), c: *const Configuration) ?TargetQuery { - return (unwrap(this) orelse return null).get(c); + return (this.unwrap() orelse return null).get(c); } }; @@ -1831,6 +1831,15 @@ pub const TargetQuery = struct { .windows => .windows, }; } + + pub fn unwrap(this: @This(), c: *const Configuration) ?std.Target.Query.OsVersion { + return switch (this) { + .none => .none, + .semver => |sv| .{ .semver = std.SemanticVersion.parse(sv.slice(c)) catch unreachable }, + .windows => |wv| .{ .windows = wv }, + .default => null, + }; + } }; pub const Abi = enum(u5) { none, @@ -2052,6 +2061,32 @@ pub const TargetQuery = struct { android_api_level: bool, dynamic_linker: bool, }; + + pub fn unwrap(tq: *const TargetQuery, c: *const Configuration) std.Target.Query { + const cpu_arch = tq.flags.cpu_arch.unwrap(); + return .{ + .cpu_arch = cpu_arch, + .cpu_model = switch (tq.flags.cpu_model) { + .native => .native, + .baseline => .baseline, + .determined_by_arch_os => .determined_by_arch_os, + .explicit => .{ .explicit = cpu_arch.?.parseCpuModel(tq.cpu_name.value.?.slice(c)).? }, + }, + .cpu_features_add = tq.cpu_features_add.value orelse .empty, + .cpu_features_sub = tq.cpu_features_sub.value orelse .empty, + .os_tag = tq.flags.os_tag.unwrap(), + .os_version_min = tq.os_version_min.u.unwrap(c), + .os_version_max = tq.os_version_max.u.unwrap(c), + .glibc_version = if (tq.glibc_version.value) |s| + std.SemanticVersion.parse(s.slice(c)) catch unreachable + else + null, + .android_api_level = tq.android_api_level.value, + .abi = tq.flags.abi.unwrap(), + .dynamic_linker = .init(if (tq.dynamic_linker.value) |s| s.slice(c) else null), + .ofmt = tq.flags.object_format.unwrap(), + }; + } }; pub const Storage = enum { diff --git a/lib/std/Target.zig b/lib/std/Target.zig index aaed0a2855a27d352b1cef23b739ee5f156b9282..49ad49b0b13fae90d34d196d28f39e83523e80cd 100644 --- a/lib/std/Target.zig +++ b/lib/std/Target.zig @@ -1668,13 +1668,13 @@ pub const Cpu = struct { }; } - pub fn parseCpuModel(arch: Arch, cpu_name: []const u8) !*const Cpu.Model { + pub fn parseCpuModel(arch: Arch, cpu_name: []const u8) ?*const Cpu.Model { for (arch.allCpuModels()) |cpu| { if (std.mem.eql(u8, cpu_name, cpu.name)) { return cpu; } } - return error.UnknownCpuModel; + return null; } pub fn endian(arch: Arch) std.builtin.Endian { diff --git a/lib/std/Target/Query.zig b/lib/std/Target/Query.zig index 2f6b9fb718b8452065d972395342f453261d09ee..c1a533020b2de773e1c04544ff61a46eb865d583 100644 --- a/lib/std/Target/Query.zig +++ b/lib/std/Target/Query.zig @@ -282,7 +282,7 @@ pub fn parse(args: ParseOptions) !Query { } else if (mem.eql(u8, cpu_name, "baseline")) { result.cpu_model = .baseline; } else { - result.cpu_model = .{ .explicit = try arch.parseCpuModel(cpu_name) }; + result.cpu_model = .{ .explicit = arch.parseCpuModel(cpu_name) orelse return error.UnknownCpuModel }; } while (index < cpu_features.len) { diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 3abf43b3258452a24cdebb12f92e0d5085e338dc..700978a94d51c0e4071c24156c9869d347e6fac4 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -680,7 +680,7 @@ pub fn putAstErrorsIntoBundle( pub fn resolveTargetQueryOrFatal(io: Io, target_query: std.Target.Query) std.Target { return std.zig.system.resolveTargetQuery(io, target_query) catch |err| - std.process.fatal("unable to resolve target: {s}", .{@errorName(err)}); + std.process.fatal("unable to resolve target: {t}", .{err}); } pub fn parseTargetQueryOrReportFatalError( diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig index ff4692db91f77e80257b81d85d986d5de69099c8..9d47215a99562ded1fdb437242d9c5729d921e81 100644 --- a/lib/std/zig/system.zig +++ b/lib/std/zig/system.zig @@ -28,6 +28,8 @@ pub const Executor = union(enum) { }; pub const GetExternalExecutorOptions = struct { + host_cpu_arch: std.Target.Cpu.Arch, + host_os_tag: std.Target.Os.Tag, allow_darling: bool = true, allow_qemu: bool = true, allow_rosetta: bool = true, @@ -39,24 +41,21 @@ pub const GetExternalExecutorOptions = struct { /// Return whether or not the given host is capable of running executables of /// the other target. -pub fn getExternalExecutor( - io: Io, - host: *const std.Target, - candidate: *const std.Target, - options: GetExternalExecutorOptions, -) Executor { - const os_match = host.os.tag == candidate.os.tag; +pub fn getExternalExecutor(io: Io, candidate: *const std.Target, options: GetExternalExecutorOptions) Executor { + const host_os_tag = options.host_os_tag; + const host_cpu_arch = options.host_cpu_arch; + const os_match = host_os_tag == candidate.os.tag; const cpu_ok = cpu_ok: { - if (host.cpu.arch == candidate.cpu.arch) + if (host_cpu_arch == candidate.cpu.arch) break :cpu_ok true; - if (host.cpu.arch == .x86_64 and candidate.cpu.arch == .x86) + if (host_cpu_arch == .x86_64 and candidate.cpu.arch == .x86) break :cpu_ok true; - if (host.cpu.arch == .aarch64 and candidate.cpu.arch == .arm) + if (host_cpu_arch == .aarch64 and candidate.cpu.arch == .arm) break :cpu_ok true; - if (host.cpu.arch == .aarch64_be and candidate.cpu.arch == .armeb) + if (host_cpu_arch == .aarch64_be and candidate.cpu.arch == .armeb) break :cpu_ok true; // TODO additionally detect incompatible CPU features. @@ -83,7 +82,7 @@ pub fn getExternalExecutor( // If the OS match and OS is macOS and CPU is arm64, we can use Rosetta 2 // to emulate the foreign architecture. if (options.allow_rosetta and os_match and - (host.os.tag == .maccatalyst or host.os.tag == .macos) and host.cpu.arch == .aarch64) + (host_os_tag == .maccatalyst or host_os_tag == .macos) and host_cpu_arch == .aarch64) { switch (candidate.cpu.arch) { .x86_64 => return .rosetta, @@ -173,13 +172,13 @@ pub fn getExternalExecutor( .windows => { if (options.allow_wine) { const wine_supported = switch (candidate.cpu.arch) { - .thumb => switch (host.cpu.arch) { + .thumb => switch (host_cpu_arch) { .arm, .thumb, .aarch64 => true, else => false, }, - .aarch64 => host.cpu.arch == .aarch64, - .x86 => host.cpu.arch.isX86(), - .x86_64 => host.cpu.arch == .x86_64, + .aarch64 => host_cpu_arch == .aarch64, + .x86 => host_cpu_arch.isX86(), + .x86_64 => host_cpu_arch == .x86_64, else => false, }; return if (wine_supported) .{ .wine = "wine" } else bad_result; @@ -191,7 +190,7 @@ pub fn getExternalExecutor( // This check can be loosened once darling adds a QEMU-based emulation // layer for non-host architectures: // https://github.com/darlinghq/darling/issues/863 - if (candidate.cpu.arch != host.cpu.arch) { + if (candidate.cpu.arch != host_cpu_arch) { return bad_result; } return .{ .darling = "darling" }; diff --git a/src/main.zig b/src/main.zig index f774eb522fecf9c9dd093be05d3a4d80dbb4d5c2..6a4768c3db1bc3e835aabe99e2fa50ccbf65052b 100644 --- a/src/main.zig +++ b/src/main.zig @@ -6825,7 +6825,11 @@ fn warnAboutForeignBinaries( const host_query: std.Target.Query = .{}; const host_target = std.zig.resolveTargetQueryOrFatal(io, host_query); - switch (std.zig.system.getExternalExecutor(io, &host_target, target, .{ .link_libc = link_libc })) { + switch (std.zig.system.getExternalExecutor(io, target, .{ + .host_cpu_arch = host_target.cpu.arch, + .host_os_tag = host_target.os.tag, + .link_libc = link_libc, + })) { .native => return, .rosetta => { const host_name = try host_target.zigTriple(arena); -- 2.54.0 From 1e956fda90a9fe6d832c044838a1e6278a775c50 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Apr 2026 15:53:30 -0700 Subject: [PATCH 065/179] maker: add the --listen and --seed args back to run --- BRANCH_TODO | 1 - lib/compiler/Maker/Step/Run.zig | 7 +++++++ lib/std/Build.zig | 13 ++++++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 66c225ddadd80f507483992de56e2f60da790ac7..5fece6c3c729095a095433f0d6afb1ebb175748b 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -2,7 +2,6 @@ * make addExtra return Index using reflection * remove Cache from configurer * implement the build options -* don't forget to add -listen arg back * get zig init template working * finish migrating the rest of the build steps * make zig-pkg path root configurable in maker (make sure --system still works) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 53d9c8cdb427508ba0fe89b4410c854df7666b38..d3fe6f48ed712499be57a433c5b9f7035e03a992 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -183,6 +183,13 @@ pub fn make( } } + man.hash.add(conf_run.flags.test_runner_mode); + if (conf_run.flags.test_runner_mode) { + try argv_list.ensureUnusedCapacity(gpa, 2); + argv_list.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{graph.random_seed})); + argv_list.appendAssumeCapacity("--listen=-"); + } + switch (conf_run.stdin.u) { .bytes => |bytes| { man.hash.addBytes(bytes.slice(conf)); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 4e958a47faa437fdd342fdbf9d298d4d39a2e56f..774d7563d246d29f50b0226744b87512fc116da8 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -796,16 +796,19 @@ pub fn addSystemCommand(b: *Build, argv: []const []const u8) *Step.Run { /// Creates a `Step.Run` with an executable built with `addExecutable`. /// Add command line arguments with methods of `Step.Run`. +/// +/// It doesn't have to target the host. In some cases cross-compiled binaries +/// can even be executed. +/// +/// This is declarative; it constructs a build step that may or may not be run +/// depending on the options provided by the user to the build command. pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run { - // It doesn't have to be native. We catch that if you actually try to run it. - // Consider that this is declarative; the run step may not be run unless a user - // option is supplied. // Avoid the common case of the step name looking like "run test test". const step_name = if (exe.kind.isTest() and mem.eql(u8, exe.name, "test")) - b.fmt("run {s}", .{@tagName(exe.kind)}) + b.fmt("run {t}", .{exe.kind}) else - b.fmt("run {s} {s}", .{ @tagName(exe.kind), exe.name }); + b.fmt("run {t} {s}", .{ exe.kind, exe.name }); const run_step = Step.Run.create(b, step_name); run_step.producer = exe; -- 2.54.0 From 5f626d28c14bfded775ae779dcde5b2435d4ac13 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Apr 2026 17:17:30 -0700 Subject: [PATCH 066/179] maker: report when result_oom flag is set --- lib/compiler/Maker.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 0ba2cfbbfc3b6fa0ea103ba93e9ed35f6a0b3a8b..99cfc2c436906ee5e20651626518ac70b010b8d4 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1528,7 +1528,6 @@ pub fn printErrorMessages( ) !void { const c = &maker.scanned_config.configuration; const gpa = maker.gpa; - log.err("TODO also report if result_oom flag is set", .{}); const writer = stderr.writer; if (error_style.verboseContext()) { // Provide context for where these error messages are coming from by @@ -1608,6 +1607,13 @@ pub fn printErrorMessages( } } + if (failing_step.result_oom) { + try stderr.setColor(.red); + try writer.writeAll("error information missing due to allocation failure"); + try stderr.setColor(.reset); + try writer.writeByte('\n'); + } + try writer.writeByte('\n'); } -- 2.54.0 From fa235757671490746d31a61a9b836ddc6320d3dc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Apr 2026 18:18:22 -0700 Subject: [PATCH 067/179] maker: fix the has side effects logic in run step --- lib/compiler/Maker.zig | 4 +++- lib/compiler/Maker/Step/Run.zig | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 99cfc2c436906ee5e20651626518ac70b010b8d4..81fb2fa71c2080d5200d82897f324530cce39e78 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1669,8 +1669,10 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { } fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { + std.log.err("TODO implement cleanTmpFiles", .{}); + if (true) return; + for (steps) |step_index| { - if (true) @panic("TODO"); const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; if (wf.mode != .tmp) continue; const path = wf.generated_directory.path orelse continue; diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index d3fe6f48ed712499be57a433c5b9f7035e03a992..45030305a53fe32cc8ced68b94a21d914097b6c3 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -70,7 +70,8 @@ pub fn make( man.hash.add(conf_run.flags.color); man.hash.add(conf_run.flags.disable_zig_progress); - var dep_file_count: usize = 0; + var any_dep_files = false; + var any_output_args = false; for (conf_run.args.slice) |arg_index| { const arg = arg_index.get(conf); @@ -161,9 +162,10 @@ pub fn make( man.hash.addBytesZ(prefix); man.hash.addBytesZ(basename); man.hash.addBytesZ(suffix); - man.hash.add(arg.flags.dep_file); - dep_file_count += @intFromBool(arg.flags.dep_file); + + any_dep_files = any_dep_files or arg.flags.dep_file; + any_output_args = true; // Add a placeholder into the argument list because we need the // manifest hash to be updated with all arguments before the @@ -233,7 +235,14 @@ pub fn make( _ = man.hash.addBytes(try cwd_path.toString(arena)); } - const has_side_effects = conf_run.flags.has_side_effects; + // Whether the Run step has side effects *other than* updating the output arguments. + const has_side_effects = conf_run.flags.has_side_effects or switch (conf_run.flags.stdio) { + .infer_from_args => !any_output_args and + conf_run.captured_stdout.value == null and + conf_run.captured_stderr.value == null, + .inherit => true, + .check, .zig_test => false, + }; if (!has_side_effects and try step.cacheHitAndWatch(maker, &man)) { // Cache hit; skip running command. @@ -244,7 +253,7 @@ pub fn make( return; } - if (dep_file_count == 0) { + if (!any_dep_files) { // We already know the final output paths; use them directly. const digest = if (has_side_effects) man.hash.final() else man.final(); const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; -- 2.54.0 From a23050bf733793c40428bb77a8cd08bd1781306e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 29 Apr 2026 22:32:13 -0700 Subject: [PATCH 068/179] compiler: update callsites of std.zig.binNameAlloc --- src/Compilation.zig | 7 ++++++- src/libs/libtsan.zig | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 5445710ea4a7f01d2175306417b4e3eb179e1362..43e0ebb7fdede91b2ea5027c33db1233c0de87bb 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -7465,9 +7465,14 @@ pub fn build_crt_file( defer arena_allocator.deinit(); const arena = arena_allocator.allocator(); + const target = &comp.root_mod.resolved_target.result; + const basename = try std.zig.binNameAlloc(gpa, .{ .root_name = root_name, - .target = &comp.root_mod.resolved_target.result, + .cpu_arch = target.cpu.arch, + .os_tag = target.os.tag, + .ofmt = target.ofmt, + .abi = target.abi, .output_mode = output_mode, }); diff --git a/src/libs/libtsan.zig b/src/libs/libtsan.zig index 588217738ad2021a67075c36e6b13d05821ae347..6f83d106455905eb17d0c19adff84b295fb5e8ed 100644 --- a/src/libs/libtsan.zig +++ b/src/libs/libtsan.zig @@ -41,7 +41,10 @@ pub fn buildTsan(comp: *Compilation, prog_node: std.Progress.Node) BuildError!vo const output_mode = .Lib; const basename = try std.zig.binNameAlloc(arena, .{ .root_name = root_name, - .target = target, + .cpu_arch = target.cpu.arch, + .os_tag = target.os.tag, + .ofmt = target.ofmt, + .abi = target.abi, .output_mode = output_mode, .link_mode = link_mode, }); -- 2.54.0 From 0d95b44a1c9e483beb82f7b70515185159d79bdc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 11:39:28 -0700 Subject: [PATCH 069/179] build system: implement cli positionals --- BRANCH_TODO | 33 +++++++++++++++++++++++---------- lib/compiler/Maker/Step/Run.zig | 23 +++++++++++++---------- lib/compiler/configurer.zig | 4 ++-- lib/init/build.zig | 4 +--- lib/std/Build/Configuration.zig | 2 +- lib/std/Build/Step/Run.zig | 30 +++++++++++++++++++++++++++++- 6 files changed, 69 insertions(+), 27 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 5fece6c3c729095a095433f0d6afb1ebb175748b..83accc79d0f31ada93ec3dab277d9a49d3fac410 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,21 +1,21 @@ -* make more stuff use IndexType -* make addExtra return Index using reflection * remove Cache from configurer * implement the build options -* get zig init template working * finish migrating the rest of the build steps +* inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) * make zig-pkg path root configurable in maker (make sure --system still works) * eliminate calls to getPath, getPath2, getPath3 +* [build system compile step data races with getGraph function](https://codeberg.org/ziglang/zig/issues/31397) * solve the TODOs added in this branch * get zig tests passing * test a bunch of third party projects / help people migrate -* refactor with DefaultingEnum -* inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) -* https://codeberg.org/ziglang/zig/issues/31397 -* restore the generated_compiler_rt_dyn_lib hack? -* run args -* https://codeberg.org/ziglang/zig/pulls/30762 + * get the target from the parent process instead +* [handle missing cache hits when chaining two run steps](https://codeberg.org/ziglang/zig/pulls/30762) +* [Absolute and cwd-relative paths in build cache](https://codeberg.org/ziglang/zig/issues/32097) + +* make more stuff use IndexType +* make addExtra return Index using reflection +* refactor with DefaultingEnum ## Followup Issues * reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make @@ -28,4 +28,17 @@ ## Release Notes -* run args are all together now, not observable in configure phase whether run args are provided + +run args are all together now, not observable in configure phase whether run args are provided. + +```zig +if (b.args) |args| { + run_cmd.addArgs(args); +} +``` + +⬇️ + +```zig +run_cmd.addBuildPositionals(); +``` diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 45030305a53fe32cc8ced68b94a21d914097b6c3..0c2c6c70db9fd7953780165cb6541f761bba8052 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -72,6 +72,7 @@ pub fn make( var any_dep_files = false; var any_output_args = false; + var any_cli_positionals = false; for (conf_run.args.slice) |arg_index| { const arg = arg_index.get(conf); @@ -176,10 +177,11 @@ pub fn make( }); argv_list.items.len += 1; }, - .cli_rest_positionals => { + .cli_positionals => { + any_cli_positionals = true; if (maker.run_args) |run_args| { try argv_list.appendSlice(gpa, run_args); - for (run_args) |s| man.hash.addBytes(s); + man.hash.addListOfBytes(run_args); } }, } @@ -236,13 +238,14 @@ pub fn make( } // Whether the Run step has side effects *other than* updating the output arguments. - const has_side_effects = conf_run.flags.has_side_effects or switch (conf_run.flags.stdio) { - .infer_from_args => !any_output_args and - conf_run.captured_stdout.value == null and - conf_run.captured_stderr.value == null, - .inherit => true, - .check, .zig_test => false, - }; + const has_side_effects = conf_run.flags.has_side_effects or any_cli_positionals or + switch (conf_run.flags.stdio) { + .infer_from_args => !any_output_args and + conf_run.captured_stdout.value == null and + conf_run.captured_stderr.value == null, + .inherit => true, + .check, .zig_test => false, + }; if (!has_side_effects and try step.cacheHitAndWatch(maker, &man)) { // Cache hit; skip running command. @@ -1596,7 +1599,7 @@ pub fn rerunInFuzzMode( }, .output_file => unreachable, .output_directory => unreachable, - .cli_rest_positionals => unreachable, + .cli_positionals => unreachable, } } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index e99f6384eff5e0fb08a6769a99e51f32a4ca8b4b..9feb803677b63865f3136fd51b76d3c3b511d045 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -492,9 +492,9 @@ const Serialize = struct { .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, }, - .cli_rest_positionals => .{ + .cli_positionals => .{ .flags = .{ - .tag = .cli_rest_positionals, + .tag = .cli_positionals, .prefix = false, .suffix = false, .basename = false, diff --git a/lib/init/build.zig b/lib/init/build.zig index 88c42f760edbfd93979f34b4a40e0e46a79ee40b..d4477c46a9db58da05f18dcca149979f5128a503 100644 --- a/lib/init/build.zig +++ b/lib/init/build.zig @@ -111,9 +111,7 @@ pub fn build(b: *std.Build) void { // This allows the user to pass arguments to the application in the build // command itself, like this: `zig build run -- arg1 arg2 etc` - if (b.args) |args| { - run_cmd.addArgs(args); - } + run_cmd.addCliPositionals(); // Creates an executable that will run `test` blocks from the provided module. // Here `mod` needs to define a target, which is why earlier we made sure to diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index eb1736db1d26d6505d04846abffc2b4b18f5c2c0..47aa0566af6236539e73872e9c0ccac9725d597c 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -567,7 +567,7 @@ pub const Step = extern struct { file_content, output_file, output_directory, - cli_rest_positionals, + cli_positionals, }; pub const Index = IndexType(@This()); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index c91d8334fe40c0ddf3ccdc015643e3021b6e40ea..478df688f34a84ae07284718e31dcf29061ee35a 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -142,7 +142,7 @@ pub const Arg = union(enum) { output_file_dep: *Output, output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. - cli_rest_positionals, + cli_positionals, }; pub const PrefixedArtifact = struct { @@ -491,16 +491,44 @@ pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []co return .{ .generated = .{ .index = dep_file.generated_file } }; } +/// Appends the contents of `arg`, verbatim, to the command line that will be +/// passed to the process being run. +/// +/// If `arg` is an input file, `addFileInput` (or related function) must be +/// used instead to ensure correct cache behavior. +/// +/// If `arg` is an output file, `addOutputFileArg` (or related function) must +/// be used instead to ensure correct cache behavior. pub fn addArg(run: *Run, arg: []const u8) void { const graph = run.step.owner.graph; const arena = graph.arena; run.argv.append(arena, .{ .bytes = graph.dupeString(arg) }) catch @panic("OOM"); } +/// Appends each of `args`, verbatim, to the command line that will be passed +/// to the process being run. +/// +/// If any element of `args` is an input file, `addFileInput` must be used +/// instead to ensure correct cache behavior. +/// +/// If any element of `args` is an output file, `addOutputFileArg` (or related +/// function) must be used instead to ensure correct cache behavior. pub fn addArgs(run: *Run, args: []const []const u8) void { for (args) |arg| run.addArg(arg); } +/// Any extra positional args are provided to the `zig build` command, they are +/// appended here. This causes the step to be considered to have side effects, +/// disabling caching. +/// +/// In the example command `zig build run -- arg1 arg2`, "arg1" and "arg2" will +/// be passed to the process being run. +pub fn addCliPositionals(run: *Run) void { + const graph = run.step.owner.graph; + const arena = graph.arena; + run.argv.append(arena, .cli_positionals) catch @panic("OOM"); +} + pub fn setStdIn(run: *Run, stdin: StdIn) void { switch (stdin) { .lazy_path => |lazy_path| lazy_path.addStepDependencies(&run.step), -- 2.54.0 From 8f224bc3f0065aaf971126de65b5d56d2d6d522d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 11:46:56 -0700 Subject: [PATCH 070/179] rename addBuildPositionals to addCliExtras they don't have to be positionals --- BRANCH_TODO | 2 +- lib/compiler/Maker/Step/Run.zig | 4 ++-- lib/compiler/configurer.zig | 4 ++-- lib/init/build.zig | 2 +- lib/std/Build/Configuration.zig | 2 +- lib/std/Build/Step/Run.zig | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 83accc79d0f31ada93ec3dab277d9a49d3fac410..8cbceb355c7e9c787904df86dddf2adffdd2aee0 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -40,5 +40,5 @@ if (b.args) |args| { ⬇️ ```zig -run_cmd.addBuildPositionals(); +run_cmd.addCliExtras(); ``` diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 0c2c6c70db9fd7953780165cb6541f761bba8052..bbba65ef913573400921996fd4d7e340fdaa39fd 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -177,7 +177,7 @@ pub fn make( }); argv_list.items.len += 1; }, - .cli_positionals => { + .cli_extras => { any_cli_positionals = true; if (maker.run_args) |run_args| { try argv_list.appendSlice(gpa, run_args); @@ -1599,7 +1599,7 @@ pub fn rerunInFuzzMode( }, .output_file => unreachable, .output_directory => unreachable, - .cli_positionals => unreachable, + .cli_extras => unreachable, } } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 9feb803677b63865f3136fd51b76d3c3b511d045..ba6afb0db80d8541a2cab143a8b2ed1893b98346 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -492,9 +492,9 @@ const Serialize = struct { .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, }, - .cli_positionals => .{ + .cli_extras => .{ .flags = .{ - .tag = .cli_positionals, + .tag = .cli_extras, .prefix = false, .suffix = false, .basename = false, diff --git a/lib/init/build.zig b/lib/init/build.zig index d4477c46a9db58da05f18dcca149979f5128a503..8de488f2e0c1965c32ea76c9a704104fb42c3e58 100644 --- a/lib/init/build.zig +++ b/lib/init/build.zig @@ -111,7 +111,7 @@ pub fn build(b: *std.Build) void { // This allows the user to pass arguments to the application in the build // command itself, like this: `zig build run -- arg1 arg2 etc` - run_cmd.addCliPositionals(); + run_cmd.addCliExtras(); // Creates an executable that will run `test` blocks from the provided module. // Here `mod` needs to define a target, which is why earlier we made sure to diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 47aa0566af6236539e73872e9c0ccac9725d597c..be5748922507ec8de344e6f111b7349c2421c7ae 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -567,7 +567,7 @@ pub const Step = extern struct { file_content, output_file, output_directory, - cli_positionals, + cli_extras, }; pub const Index = IndexType(@This()); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 478df688f34a84ae07284718e31dcf29061ee35a..1db2600f4f7cc61f0aeba76d283bbc16a5191a17 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -142,7 +142,7 @@ pub const Arg = union(enum) { output_file_dep: *Output, output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. - cli_positionals, + cli_extras, }; pub const PrefixedArtifact = struct { @@ -523,10 +523,10 @@ pub fn addArgs(run: *Run, args: []const []const u8) void { /// /// In the example command `zig build run -- arg1 arg2`, "arg1" and "arg2" will /// be passed to the process being run. -pub fn addCliPositionals(run: *Run) void { +pub fn addCliExtras(run: *Run) void { const graph = run.step.owner.graph; const arena = graph.arena; - run.argv.append(arena, .cli_positionals) catch @panic("OOM"); + run.argv.append(arena, .cli_extras) catch @panic("OOM"); } pub fn setStdIn(run: *Run, stdin: StdIn) void { -- 2.54.0 From 7819e4dea7d1bd75ee1bbb8d76bb8a8ee7672962 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 13:55:56 -0700 Subject: [PATCH 071/179] rename addCliExtras to addPassthruArgs finally, a good name --- BRANCH_TODO | 2 +- lib/compiler/Maker/Step/Run.zig | 4 ++-- lib/compiler/configurer.zig | 4 ++-- lib/init/build.zig | 2 +- lib/std/Build/Configuration.zig | 2 +- lib/std/Build/Step/Run.zig | 14 ++++++++------ 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 8cbceb355c7e9c787904df86dddf2adffdd2aee0..eda7ffa77e11baa73ab405995c5234b09f7ce105 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -40,5 +40,5 @@ if (b.args) |args| { ⬇️ ```zig -run_cmd.addCliExtras(); +run_cmd.addPassthruArgs(); ``` diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index bbba65ef913573400921996fd4d7e340fdaa39fd..0cdbec1cecc589b6a6e4fd8f8af3b2ffe4ad56b5 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -177,7 +177,7 @@ pub fn make( }); argv_list.items.len += 1; }, - .cli_extras => { + .passthru => { any_cli_positionals = true; if (maker.run_args) |run_args| { try argv_list.appendSlice(gpa, run_args); @@ -1599,7 +1599,7 @@ pub fn rerunInFuzzMode( }, .output_file => unreachable, .output_directory => unreachable, - .cli_extras => unreachable, + .passthru => unreachable, } } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index ba6afb0db80d8541a2cab143a8b2ed1893b98346..a4e37b50c1a7b40f12a22f23af52d088b34289ec 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -492,9 +492,9 @@ const Serialize = struct { .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, }, - .cli_extras => .{ + .passthru => .{ .flags = .{ - .tag = .cli_extras, + .tag = .passthru, .prefix = false, .suffix = false, .basename = false, diff --git a/lib/init/build.zig b/lib/init/build.zig index 8de488f2e0c1965c32ea76c9a704104fb42c3e58..99cee233218d23472c2846b9c5120fa99a09406b 100644 --- a/lib/init/build.zig +++ b/lib/init/build.zig @@ -111,7 +111,7 @@ pub fn build(b: *std.Build) void { // This allows the user to pass arguments to the application in the build // command itself, like this: `zig build run -- arg1 arg2 etc` - run_cmd.addCliExtras(); + run_cmd.addPassthruArgs(); // Creates an executable that will run `test` blocks from the provided module. // Here `mod` needs to define a target, which is why earlier we made sure to diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index be5748922507ec8de344e6f111b7349c2421c7ae..16e91bed505f285baf1ce393f0855636b783de93 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -567,7 +567,7 @@ pub const Step = extern struct { file_content, output_file, output_directory, - cli_extras, + passthru, }; pub const Index = IndexType(@This()); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 1db2600f4f7cc61f0aeba76d283bbc16a5191a17..a92bb5f1f3c19ae5679ad4627d4907817197f72d 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -142,7 +142,7 @@ pub const Arg = union(enum) { output_file_dep: *Output, output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. - cli_extras, + passthru, }; pub const PrefixedArtifact = struct { @@ -517,16 +517,18 @@ pub fn addArgs(run: *Run, args: []const []const u8) void { for (args) |arg| run.addArg(arg); } -/// Any extra positional args are provided to the `zig build` command, they are -/// appended here. This causes the step to be considered to have side effects, -/// disabling caching. +/// Appends the extra arguments provided to `zig build` to the command line +/// that will be passed to the process being run. +/// +/// This causes the step to be considered to have side effects, disabling +/// caching. /// /// In the example command `zig build run -- arg1 arg2`, "arg1" and "arg2" will /// be passed to the process being run. -pub fn addCliExtras(run: *Run) void { +pub fn addPassthruArgs(run: *Run) void { const graph = run.step.owner.graph; const arena = graph.arena; - run.argv.append(arena, .cli_extras) catch @panic("OOM"); + run.argv.append(arena, .passthru) catch @panic("OOM"); } pub fn setStdIn(run: *Run, stdin: StdIn) void { -- 2.54.0 From 1dc82c13280ae327faf9448dcaf69960f5f5cad4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 14:49:23 -0700 Subject: [PATCH 072/179] configurer: remove Cache --- BRANCH_TODO | 8 ++++++-- lib/compiler/configurer.zig | 12 ------------ lib/std/Build.zig | 1 - 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index eda7ffa77e11baa73ab405995c5234b09f7ce105..2c256f09e8992587d84fa409d02817ab2d95084a 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,3 @@ -* remove Cache from configurer * implement the build options * finish migrating the rest of the build steps * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) @@ -29,7 +28,8 @@ ## Release Notes -run args are all together now, not observable in configure phase whether run args are provided. +In the Run step, passthru args are all together now, not observable in +configure phase whether run args are provided. ```zig if (b.args) |args| { @@ -42,3 +42,7 @@ if (b.args) |args| { ```zig run_cmd.addPassthruArgs(); ``` + +This removes a capability from build scripts since they can no longer observe +those arguments. In exchange, it means that when changing those arguments, +build scripts need not be rebuilt from source. diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index a4e37b50c1a7b40f12a22f23af52d088b34289ec..1b0a87a1e5d160b4ff3cd00c57656584df3fbcad 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -73,12 +73,6 @@ pub fn main(init: process.Init.Minimal) !void { var graph: std.Build.Graph = .{ .io = io, .arena = arena, - .cache = .{ - .io = io, - .gpa = arena, - .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), - .cwd = try process.currentPathAlloc(io, arena), - }, .zig_exe = zig_exe, .environ_map = try init.environ.createMap(arena), .global_cache_root = global_cache_directory, @@ -102,12 +96,6 @@ pub fn main(init: process.Init.Minimal) !void { assert(try graph.wip_configuration.addString("") == .empty); assert(try graph.wip_configuration.addString("root") == .root); - graph.cache.addPrefix(.{ .path = null, .handle = cwd }); - graph.cache.addPrefix(build_root_directory); - graph.cache.addPrefix(local_cache_directory); - graph.cache.addPrefix(global_cache_directory); - graph.cache.hash.addBytes(builtin.zig_version_string); - const builder = try std.Build.create( &graph, build_root_directory, diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 774d7563d246d29f50b0226744b87512fc116da8..edec451b627f79d2f9c09cef9339f3a789c9edb6 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -83,7 +83,6 @@ pub const Graph = struct { arena: Allocator, system_integration_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .empty, system_package_mode: bool = false, - cache: Cache, zig_exe: []const u8, environ_map: process.Environ.Map, global_cache_root: Cache.Directory, -- 2.54.0 From aec708ce25409f7e5aa9f39568cdf6281e857742 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 15:35:25 -0700 Subject: [PATCH 073/179] build system: remove unneeded args from configurer not needed: * zig exe path * zig lib dir * build root * local cache root * global cache root --- lib/compiler/Maker/Step/Run.zig | 5 ++- lib/compiler/configurer.zig | 66 ++++----------------------------- lib/std/Build.zig | 34 ++++------------- lib/std/Build/Step/Run.zig | 2 - src/main.zig | 56 ++++++++-------------------- 5 files changed, 34 insertions(+), 129 deletions(-) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 0cdbec1cecc589b6a6e4fd8f8af3b2ffe4ad56b5..c381c480cc1fc0a24101af657d6351ee0aad4918 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -189,7 +189,10 @@ pub fn make( man.hash.add(conf_run.flags.test_runner_mode); if (conf_run.flags.test_runner_mode) { - try argv_list.ensureUnusedCapacity(gpa, 2); + const cache_dir_string = try convertPathArg(run_index, maker, .{ .root_dir = cache_root }); + + try argv_list.ensureUnusedCapacity(gpa, 3); + argv_list.appendAssumeCapacity(try allocPrint(arena, "--cache-dir={s}", .{cache_dir_string})); argv_list.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{graph.random_seed})); argv_list.appendAssumeCapacity("--listen=-"); } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 1b0a87a1e5d160b4ff3cd00c57656584df3fbcad..bd1c1d07eba9a61129a1884723ccecf12272ba83 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -39,44 +39,10 @@ pub fn main(init: process.Init.Minimal) !void { const args = try init.args.toSlice(arena); - // Skip own executable name. - var arg_idx: usize = 1; - - const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); - const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); - const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); - const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); - const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); - - const cwd: Io.Dir = .cwd(); - - const zig_lib_directory: std.Build.Cache.Directory = .{ - .path = zig_lib_dir, - .handle = try cwd.openDir(io, zig_lib_dir, .{}), - }; - - const build_root_directory: std.Build.Cache.Directory = .{ - .path = build_root, - .handle = try cwd.openDir(io, build_root, .{}), - }; - - const local_cache_directory: std.Build.Cache.Directory = .{ - .path = local_cache_root, - .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), - }; - - const global_cache_directory: std.Build.Cache.Directory = .{ - .path = global_cache_root, - .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), - }; - var graph: std.Build.Graph = .{ .io = io, .arena = arena, - .zig_exe = zig_exe, .environ_map = try init.environ.createMap(arena), - .global_cache_root = global_cache_directory, - .zig_lib_directory = zig_lib_directory, // TODO get this from parent process instead .host = .{ .query = .{}, @@ -96,12 +62,7 @@ pub fn main(init: process.Init.Minimal) !void { assert(try graph.wip_configuration.addString("") == .empty); assert(try graph.wip_configuration.addString("root") == .root); - const builder = try std.Build.create( - &graph, - build_root_directory, - local_cache_directory, - dependencies.root_deps, - ); + const builder = try std.Build.create(&graph, dependencies.root_deps); var error_style: ErrorStyle = .verbose; var multiline_errors: MultilineErrors = .indent; @@ -119,7 +80,9 @@ pub fn main(init: process.Init.Minimal) !void { } } - while (nextArg(args, &arg_idx)) |arg| { + var arg_i: usize = 1; // Skip own executable name. + + while (nextArg(args, &arg_i)) |arg| { if (mem.cutPrefix(u8, arg, "-D")) |option_contents| { if (option_contents.len == 0) fatalWithHint("expected option name after '-D'", .{}); @@ -145,7 +108,7 @@ pub fn main(init: process.Init.Minimal) !void { }); }; } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected [auto|on|off] after {q}", .{arg}); color = std.meta.stringToEnum(Color, next_arg) orelse { fatalWithHint("expected [auto|on|off] after {q}, found {q}", .{ @@ -153,13 +116,13 @@ pub fn main(init: process.Init.Minimal) !void { }); }; } else if (mem.eql(u8, arg, "--error-style")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected style after {q}", .{arg}); error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--multiline-errors")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected style after {q}", .{arg}); multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); @@ -169,8 +132,6 @@ pub fn main(init: process.Init.Minimal) !void { // but it is handled by the parent process. The build runner // only sees this flag. graph.system_package_mode = true; - } else if (mem.eql(u8, arg, "--have-run-args")) { - graph.have_run_args = true; } else { fatalWithHint("unrecognized argument: {q}", .{arg}); } @@ -1112,19 +1073,6 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { return args[idx.*]; } -fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - fatalWithHint("expected argument after: {s}", .{args[idx.* - 1]}); - }; -} - -fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { - const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); - if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); - const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); - return arg; -} - const ErrorStyle = enum { verbose, minimal, diff --git a/lib/std/Build.zig b/lib/std/Build.zig index edec451b627f79d2f9c09cef9339f3a789c9edb6..5a8df579806b34651d6f0a109191738020237c66 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -36,9 +36,6 @@ invalid_user_input: bool, default_step: *Step, top_level_steps: std.StringArrayHashMapUnmanaged(*Step.TopLevel), install_prefix: []const u8, -/// Path to the directory containing build.zig. -build_root: Cache.Directory, -cache_root: Cache.Directory, debug_log_scopes: []const []const u8 = &.{}, /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, /// in particular at `Step` creation. @@ -83,10 +80,7 @@ pub const Graph = struct { arena: Allocator, system_integration_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .empty, system_package_mode: bool = false, - zig_exe: []const u8, environ_map: process.Environ.Map, - global_cache_root: Cache.Directory, - zig_lib_directory: Cache.Directory, needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .empty, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, @@ -97,10 +91,6 @@ pub const Graph = struct { /// respects the '--color' flag. stderr_mode: ?Io.Terminal.Mode = null, release_mode: ReleaseMode = .off, - /// Whether the user passed in "--" arguments. They can be added to a child - /// process via `Step.Run` API but cannot be observed in the configure - /// phase. - have_run_args: bool = false, /// Indexes correspond to `Configuration.GeneratedFileIndex`. generated_files: std.ArrayList(*Step), @@ -230,8 +220,6 @@ const TypeId = enum { pub fn create( graph: *Graph, - build_root: Cache.Directory, - cache_root: Cache.Directory, available_deps: AvailableDeps, ) error{OutOfMemory}!*Build { const arena = graph.arena; @@ -239,8 +227,6 @@ pub fn create( const b = try arena.create(Build); b.* = .{ .graph = graph, - .build_root = build_root, - .cache_root = cache_root, .invalid_user_input = false, .allocator = arena, .user_input_options = UserInputOptionsMap.init(arena), @@ -280,7 +266,6 @@ pub fn create( fn createChild( parent: *Build, dep_name: []const u8, - build_root: Cache.Directory, pkg_hash: []const u8, pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap, @@ -312,8 +297,6 @@ fn createChild( .invalid_user_input = false, .default_step = undefined, .top_level_steps = .{}, - .build_root = build_root, - .cache_root = parent.cache_root, .debug_log_scopes = parent.debug_log_scopes, .enable_darling = parent.enable_darling, .enable_qemu = parent.enable_qemu, @@ -2036,15 +2019,12 @@ fn dependencyInner( const build_root: std.Build.Cache.Directory = .{ .path = build_root_string, - .handle = Io.Dir.cwd().openDir(io, build_root_string, .{}) catch |err| { - std.debug.print("unable to open '{s}': {s}\n", .{ - build_root_string, @errorName(err), - }); - process.exit(1); - }, + .handle = Io.Dir.cwd().openDir(io, build_root_string, .{}) catch |err| + process.fatal("unable to open {s}: {t}", .{ build_root_string, err }), }; - const sub_builder = b.createChild(name, build_root, pkg_hash, pkg_deps, user_input_options) catch @panic("unhandled error"); + const sub_builder = b.createChild(name, build_root, pkg_hash, pkg_deps, user_input_options) catch + @panic("unhandled error"); if (build_zig) |bz| { sub_builder.runBuild(bz) catch @panic("unhandled error"); @@ -2330,9 +2310,9 @@ pub const InstallDir = union(enum) { } }; -/// Creates a path leading to a directory inside "tmp" subdirectory of -/// `cache_root` which is created on demand and cleaned up by the build runner -/// upon success. +/// Creates a path leading to a directory inside "tmp" subdirectory of local +/// cache which is created on demand and cleaned up by the build runner upon +/// success. pub fn tmpPath(b: *Build) LazyPath { const wf = b.addTempFiles(); return wf.getDirectory(); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index a92bb5f1f3c19ae5679ad4627d4907817197f72d..1b9b519f9b70a338040c6917db083744f0cf32dd 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -215,9 +215,7 @@ pub fn setName(run: *Run, name: []const u8) void { pub fn enableTestRunnerMode(run: *Run) void { if (run.test_runner_mode) return; - const b = run.step.owner; run.stdio = .zig_test; - run.addPrefixedDirectoryArg("--cache-dir=", .{ .cwd_relative = b.cache_root.path orelse "." }); run.test_runner_mode = true; } diff --git a/src/main.zig b/src/main.zig index 6a4768c3db1bc3e835aabe99e2fa50ccbf65052b..912952b174eb139183ff35dcaf4f4ff582ed8d25 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4976,43 +4976,27 @@ fn cmdBuild( try configure_argv.ensureUnusedCapacity(arena, 16); try make_argv.ensureUnusedCapacity(arena, 16); - _ = configure_argv.addOneAssumeCapacity(); - _ = make_argv.addOneAssumeCapacity(); + _ = configure_argv.addOneAssumeCapacity(); // configurer executable + _ = make_argv.addOneAssumeCapacity(); // maker executable - configure_argv.appendAssumeCapacity("--zig"); - configure_argv.appendAssumeCapacity(self_exe_path); + make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; - make_argv.appendAssumeCapacity("--zig"); - make_argv.appendAssumeCapacity(self_exe_path); + make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig-lib-dir", undefined }; + const argv_index_zig_lib_dir = make_argv.items.len - 1; - configure_argv.appendAssumeCapacity("--zig-lib-dir"); - make_argv.appendAssumeCapacity("--zig-lib-dir"); - const argv_index_zig_lib_dir = configure_argv.items.len; - _ = configure_argv.addOneAssumeCapacity(); - _ = make_argv.addOneAssumeCapacity(); + make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; + const argv_index_build_file = make_argv.items.len - 1; - configure_argv.appendAssumeCapacity("--build-root"); - make_argv.appendAssumeCapacity("--build-root"); - const argv_index_build_file = configure_argv.items.len; - _ = configure_argv.addOneAssumeCapacity(); - _ = make_argv.addOneAssumeCapacity(); + make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--local-cache", undefined }; + const argv_index_cache_dir = make_argv.items.len - 1; - configure_argv.appendAssumeCapacity("--local-cache"); - make_argv.appendAssumeCapacity("--local-cache"); - const argv_index_cache_dir = configure_argv.items.len; - _ = configure_argv.addOneAssumeCapacity(); - _ = make_argv.addOneAssumeCapacity(); + make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--global-cache", undefined }; + const argv_index_global_cache_dir = make_argv.items.len - 1; - configure_argv.appendAssumeCapacity("--global-cache"); - make_argv.appendAssumeCapacity("--global-cache"); - const argv_index_global_cache_dir = configure_argv.items.len; - _ = configure_argv.addOneAssumeCapacity(); - _ = make_argv.addOneAssumeCapacity(); - - make_argv.appendSliceAssumeCapacity(&.{ "--configuration", undefined }); + make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--configuration", undefined }; const argv_index_configuration_file = make_argv.items.len - 1; - make_argv.appendSliceAssumeCapacity(&.{ "--seed", default_seed }); + make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--seed", default_seed }; const argv_index_seed = make_argv.items.len - 1; var color: Color = .auto; @@ -5147,13 +5131,10 @@ fn cmdBuild( try configure_argv.appendSlice(arena, &.{ arg, args[i] }); continue; } else if (mem.cutPrefix(u8, arg, "-j")) |str| { - const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| { - fatal("unable to parse jobs count '{s}': {s}", .{ - str, @errorName(err), - }); - }; + const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| + fatal("unable to parse jobs count {s}: {t}", .{ str, err }); if (num < 1) { - fatal("number of jobs must be at least 1\n", .{}); + fatal("number of jobs must be at least 1", .{}); } n_jobs = num; } else if (mem.eql(u8, arg, "--seed")) { @@ -5287,11 +5268,6 @@ fn cmdBuild( } }); defer _ = make_runner_task.cancel(io) catch {}; - configure_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; - configure_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; - configure_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; - configure_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; - make_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; make_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; make_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; -- 2.54.0 From ac0b1bfda2c8ac6093b86a000c3515c57535ef07 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 18:08:57 -0700 Subject: [PATCH 074/179] build system: implement options options which are passed to configurer and therefore observable by the build script are added to the cache hash. A sorted list is hashed since they are unordered. --- BRANCH_TODO | 7 +- lib/compiler/configurer.zig | 63 ++++++--------- lib/std/Build.zig | 131 ++++++++++---------------------- lib/std/Build/Configuration.zig | 28 ++++++- src/main.zig | 63 ++++++++++----- 5 files changed, 139 insertions(+), 153 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 2c256f09e8992587d84fa409d02817ab2d95084a..2b02293e9a8f20d66192d1710d32d93aa4304a08 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,9 +1,9 @@ -* implement the build options * finish migrating the rest of the build steps * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) * make zig-pkg path root configurable in maker (make sure --system still works) * eliminate calls to getPath, getPath2, getPath3 * [build system compile step data races with getGraph function](https://codeberg.org/ziglang/zig/issues/31397) +* test lazyImport * solve the TODOs added in this branch * get zig tests passing * test a bunch of third party projects / help people migrate @@ -16,6 +16,8 @@ * make addExtra return Index using reflection * refactor with DefaultingEnum +* implement {q} or delete {q} uses + ## Followup Issues * reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make * link_eh_frame_hdr should be DefaultingBool @@ -24,6 +26,7 @@ - but artifact install steps also add paths for dyn libs on windows * no more "artifact arg" to run step. if you want to run the post-install binary, get the lazy path from the install step. +* -D options which are files need to be accounted for in the configure cache ## Release Notes @@ -45,4 +48,4 @@ run_cmd.addPassthruArgs(); This removes a capability from build scripts since they can no longer observe those arguments. In exchange, it means that when changing those arguments, -build scripts need not be rebuilt from source. +build scripts no longer must be rebuilt from source. diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index bd1c1d07eba9a61129a1884723ccecf12272ba83..3fde1e989bde460555f53d0f8bd2150dc541de08 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -64,22 +64,7 @@ pub fn main(init: process.Init.Minimal) !void { const builder = try std.Build.create(&graph, dependencies.root_deps); - var error_style: ErrorStyle = .verbose; - var multiline_errors: MultilineErrors = .indent; var color: Color = .auto; - - if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(ErrorStyle, str)) |style| { - error_style = style; - } - } - - if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(MultilineErrors, str)) |style| { - multiline_errors = style; - } - } - var arg_i: usize = 1; // Skip own executable name. while (nextArg(args, &arg_i)) |arg| { @@ -101,39 +86,20 @@ pub fn main(init: process.Init.Minimal) !void { try graph.system_integration_options.put(arena, name, .user_disabled); } else if (mem.eql(u8, arg, "--release")) { graph.release_mode = .any; - } else if (mem.cutPrefix(u8, arg, "--release=")) |text| { - graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { - fatalWithHint("expected [off|any|fast|safe|small] in {q}, found {q}", .{ - arg, text, - }); - }; - } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArg(args, &arg_i) orelse - fatalWithHint("expected [auto|on|off] after {q}", .{arg}); - color = std.meta.stringToEnum(Color, next_arg) orelse { - fatalWithHint("expected [auto|on|off] after {q}, found {q}", .{ - arg, next_arg, - }); - }; - } else if (mem.eql(u8, arg, "--error-style")) { - const next_arg = nextArg(args, &arg_i) orelse - fatalWithHint("expected style after {q}", .{arg}); - error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { - fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); - }; - } else if (mem.eql(u8, arg, "--multiline-errors")) { - const next_arg = nextArg(args, &arg_i) orelse - fatalWithHint("expected style after {q}", .{arg}); - multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { - fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); + } else if (mem.cutPrefix(u8, arg, "--release=")) |rest| { + graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, rest) orelse { + fatalWithHint("expected --release=[off|any|fast|safe|small]; found: {s}", .{arg}); }; + } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { + color = std.meta.stringToEnum(Color, rest) orelse + fatalWithHint("expected --color=[auto|on|off]; found: {s}", .{arg}); } else if (mem.eql(u8, arg, "--system")) { // The usage text shows another argument after this parameter // but it is handled by the parent process. The build runner // only sees this flag. graph.system_package_mode = true; } else { - fatalWithHint("unrecognized argument: {q}", .{arg}); + fatalWithHint("unrecognized argument: {s}", .{arg}); } } @@ -152,6 +118,7 @@ pub fn main(init: process.Init.Minimal) !void { fatal(" access the help menu with 'zig build -h'", .{}); } + try serializePackageOptions(builder, &graph.wip_configuration); try serializeSystemIntegrationOptions(&graph, &graph.wip_configuration); var stdout_buffer: [1024]u8 = undefined; @@ -1125,3 +1092,17 @@ fn serializeSystemIntegrationOptions(graph: *std.Build.Graph, wc: *Configuration process.exit(1); } } + +fn serializePackageOptions(b: *std.Build, wc: *Configuration.Wip) Allocator.Error!void { + const gpa = wc.gpa; + + try wc.available_options.ensureTotalCapacityPrecise(gpa, b.available_options_map.count()); + for (b.available_options_map.keys(), b.available_options_map.values()) |name, *opt| { + wc.available_options.appendAssumeCapacity(.{ + .name = try wc.addString(name), + .description = try wc.addString(opt.description), + .type = opt.type_id, + .enum_options = if (opt.enum_options) |enum_vals| .init(try wc.addStringList(enum_vals)) else .none, + }); + } +} diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 5a8df579806b34651d6f0a109191738020237c66..c03df8d552dc7fda7b7ef72a8b19bcf3d58fd614 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -30,8 +30,7 @@ install_tls: Step.TopLevel, uninstall_tls: Step.TopLevel, allocator: Allocator, user_input_options: UserInputOptionsMap, -available_options_map: AvailableOptionsMap, -available_options_list: std.array_list.Managed(AvailableOption), +available_options_map: std.array_hash_map.String(AvailableOption) = .empty, invalid_user_input: bool, default_step: *Step, top_level_steps: std.StringArrayHashMapUnmanaged(*Step.TopLevel), @@ -180,11 +179,10 @@ pub const RunError = error{ } || std.process.SpawnError; const UserInputOptionsMap = StringHashMap(UserInputOption); -const AvailableOptionsMap = StringHashMap(AvailableOption); const AvailableOption = struct { name: []const u8, - type_id: TypeId, + type_id: Configuration.AvailableOption.Type, description: []const u8, /// If the `type_id` is `enum` or `enum_list` this provides the list of enum options enum_options: ?[]const []const u8, @@ -205,19 +203,6 @@ const UserValue = union(enum) { lazy_path_list: std.array_list.Managed(LazyPath), }; -const TypeId = enum { - bool, - int, - float, - @"enum", - enum_list, - string, - list, - build_id, - lazy_path, - lazy_path_list, -}; - pub fn create( graph: *Graph, available_deps: AvailableDeps, @@ -230,8 +215,6 @@ pub fn create( .invalid_user_input = false, .allocator = arena, .user_input_options = UserInputOptionsMap.init(arena), - .available_options_map = AvailableOptionsMap.init(arena), - .available_options_list = std.array_list.Managed(AvailableOption).init(arena), .top_level_steps = .{}, .default_step = undefined, .install_prefix = undefined, @@ -292,8 +275,6 @@ fn createChild( .description = "Remove build artifacts from prefix path", }, .user_input_options = user_input_options, - .available_options_map = AvailableOptionsMap.init(allocator), - .available_options_list = std.array_list.Managed(AvailableOption).init(allocator), .invalid_user_input = false, .default_step = undefined, .top_level_steps = .{}, @@ -960,13 +941,14 @@ pub fn getUninstallStep(b: *Build) *Step { /// these options when calling the dependency's build.zig script as a function. /// `null` is returned when an option is left to default. pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw: []const u8) ?T { + const arena = b.allocator; const name = b.dupe(name_raw); const description = b.dupe(description_raw); const type_id = comptime typeToEnum(T); const enum_options = if (type_id == .@"enum" or type_id == .enum_list) blk: { const EnumType = if (type_id == .enum_list) @typeInfo(T).pointer.child else T; const fields = comptime std.meta.fields(EnumType); - var options = std.array_list.Managed([]const u8).initCapacity(b.allocator, fields.len) catch @panic("OOM"); + var options = std.array_list.Managed([]const u8).initCapacity(arena, fields.len) catch @panic("OOM"); inline for (fields) |field| { options.appendAssumeCapacity(field.name); @@ -980,10 +962,9 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw .description = description, .enum_options = enum_options, }; - if ((b.available_options_map.fetchPut(name, available_option) catch @panic("OOM")) != null) { - panic("Option '{s}' declared twice", .{name}); + if ((b.available_options_map.fetchPut(arena, name, available_option) catch @panic("OOM")) != null) { + panic("option '{s}' declared twice", .{name}); } - b.available_options_list.append(available_option) catch @panic("OOM"); const option_ptr = b.user_input_options.getPtr(name) orelse return null; option_ptr.used = true; @@ -996,36 +977,32 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw } else if (mem.eql(u8, s, "false")) { return false; } else { - log.err("Expected -D{s} to be a boolean, but received '{s}'", .{ name, s }); + log.err("expected -D{s} to be a boolean; received: {s}", .{ name, s }); b.markInvalidUserInput(); return null; } }, .list, .map, .lazy_path, .lazy_path_list => { - log.err("Expected -D{s} to be a boolean, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be a boolean; received: {t}", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, }, .int => switch (option_ptr.value) { .flag, .list, .map, .lazy_path, .lazy_path_list => { - log.err("Expected -D{s} to be an integer, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be an integer; received: {t}", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, .scalar => |s| { const n = std.fmt.parseInt(T, s, 10) catch |err| switch (err) { error.Overflow => { - log.err("-D{s} value {s} cannot fit into type {s}.", .{ name, s, @typeName(T) }); + log.err("-D{s} value {s} cannot fit into type {s}", .{ name, s, @typeName(T) }); b.markInvalidUserInput(); return null; }, else => { - log.err("Expected -D{s} to be an integer of type {s}.", .{ name, @typeName(T) }); + log.err("expected -D{s} to be an integer of type {s}", .{ name, @typeName(T) }); b.markInvalidUserInput(); return null; }, @@ -1035,15 +1012,13 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw }, .float => switch (option_ptr.value) { .flag, .map, .list, .lazy_path, .lazy_path_list => { - log.err("Expected -D{s} to be a float, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be a float; received: {t}", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, .scalar => |s| { const n = std.fmt.parseFloat(T, s) catch { - log.err("Expected -D{s} to be a float of type {s}.", .{ name, @typeName(T) }); + log.err("expected -D{s} to be a float of type {s}", .{ name, @typeName(T) }); b.markInvalidUserInput(); return null; }; @@ -1052,9 +1027,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw }, .@"enum" => switch (option_ptr.value) { .flag, .map, .list, .lazy_path, .lazy_path_list => { - log.err("Expected -D{s} to be an enum, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be an enum; received: {t}.", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, @@ -1062,7 +1035,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw if (std.meta.stringToEnum(T, s)) |enum_lit| { return enum_lit; } else { - log.err("Expected -D{s} to be of type {s}.", .{ name, @typeName(T) }); + log.err("expected -D{s} to be of type {s}", .{ name, @typeName(T) }); b.markInvalidUserInput(); return null; } @@ -1070,9 +1043,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw }, .string => switch (option_ptr.value) { .flag, .list, .map, .lazy_path, .lazy_path_list => { - log.err("Expected -D{s} to be a string, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be a string; received: {t}", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, @@ -1080,9 +1051,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw }, .build_id => switch (option_ptr.value) { .flag, .map, .list, .lazy_path, .lazy_path_list => { - log.err("Expected -D{s} to be an enum, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be an enum; received: {t}.", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, @@ -1090,7 +1059,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw if (std.zig.BuildId.parse(s)) |build_id| { return build_id; } else |err| { - log.err("unable to parse option '-D{s}': {t}", .{ name, err }); + log.err("failed to parse option -D{s}: {t}", .{ name, err }); b.markInvalidUserInput(); return null; } @@ -1098,42 +1067,38 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw }, .list => switch (option_ptr.value) { .flag, .map, .lazy_path, .lazy_path_list => { - log.err("Expected -D{s} to be a list, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be a list; received: {t}", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, .scalar => |s| { - return b.allocator.dupe([]const u8, &[_][]const u8{s}) catch @panic("OOM"); + return arena.dupe([]const u8, &[_][]const u8{s}) catch @panic("OOM"); }, .list => |lst| return lst.items, }, .enum_list => switch (option_ptr.value) { .flag, .map, .lazy_path, .lazy_path_list => { - log.err("Expected -D{s} to be a list, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be a list; received: {t}", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, .scalar => |s| { const Child = @typeInfo(T).pointer.child; const value = std.meta.stringToEnum(Child, s) orelse { - log.err("Expected -D{s} to be of type {s}.", .{ name, @typeName(Child) }); + log.err("expected -D{s} to be of type {s}", .{ name, @typeName(Child) }); b.markInvalidUserInput(); return null; }; - return b.allocator.dupe(Child, &[_]Child{value}) catch @panic("OOM"); + return arena.dupe(Child, &[_]Child{value}) catch @panic("OOM"); }, .list => |lst| { const Child = @typeInfo(T).pointer.child; - const new_list = b.allocator.alloc(Child, lst.items.len) catch @panic("OOM"); + const new_list = arena.alloc(Child, lst.items.len) catch @panic("OOM"); for (new_list, lst.items) |*new_item, str| { new_item.* = std.meta.stringToEnum(Child, str) orelse { - log.err("Expected -D{s} to be of type {s}.", .{ name, @typeName(Child) }); + log.err("expected -D{s} to be of type {s}", .{ name, @typeName(Child) }); b.markInvalidUserInput(); - b.allocator.free(new_list); + arena.free(new_list); return null; }; } @@ -1144,18 +1109,16 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw .scalar => |s| return .{ .cwd_relative = s }, .lazy_path => |lp| return lp, .flag, .map, .list, .lazy_path_list => { - log.err("Expected -D{s} to be a path, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be a path; received: {t}", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, }, .lazy_path_list => switch (option_ptr.value) { - .scalar => |s| return b.allocator.dupe(LazyPath, &[_]LazyPath{.{ .cwd_relative = s }}) catch @panic("OOM"), - .lazy_path => |lp| return b.allocator.dupe(LazyPath, &[_]LazyPath{lp}) catch @panic("OOM"), + .scalar => |s| return arena.dupe(LazyPath, &[_]LazyPath{.{ .cwd_relative = s }}) catch @panic("OOM"), + .lazy_path => |lp| return arena.dupe(LazyPath, &[_]LazyPath{lp}) catch @panic("OOM"), .list => |lst| { - const new_list = b.allocator.alloc(LazyPath, lst.items.len) catch @panic("OOM"); + const new_list = arena.alloc(LazyPath, lst.items.len) catch @panic("OOM"); for (new_list, lst.items) |*new_item, str| { new_item.* = .{ .cwd_relative = str }; } @@ -1163,9 +1126,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw }, .lazy_path_list => |lp_list| return lp_list.items, .flag, .map => { - log.err("Expected -D{s} to be a path, but received a {s}.", .{ - name, @tagName(option_ptr.value), - }); + log.err("expected -D{s} to be a path; received: {t}", .{ name, option_ptr.value }); b.markInvalidUserInput(); return null; }, @@ -1250,8 +1211,8 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile opts_copy.diagnostics = &diags; return std.Target.Query.parse(opts_copy) catch |err| switch (err) { error.UnknownCpuModel => { - std.debug.print("unknown CPU: '{s}'\navailable CPUs for architecture '{s}':\n", .{ - diags.cpu_name.?, @tagName(diags.arch.?), + std.debug.print("unknown CPU: '{s}'\navailable CPUs for architecture '{t}':\n", .{ + diags.cpu_name.?, diags.arch.?, }); for (diags.arch.?.allCpuModels()) |cpu| { std.debug.print(" {s}\n", .{cpu.name}); @@ -1261,11 +1222,10 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile error.UnknownCpuFeature => { std.debug.print( \\unknown CPU feature: '{s}' - \\available CPU features for architecture '{s}': + \\available CPU features for architecture '{t}': \\ , .{ - diags.unknown_feature_name.?, - @tagName(diags.arch.?), + diags.unknown_feature_name.?, diags.arch.?, }); for (diags.arch.?.allFeaturesList()) |feature| { std.debug.print(" {s}: {s}\n", .{ feature.name, feature.description }); @@ -1398,7 +1358,9 @@ pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8 return true; }, .lazy_path, .lazy_path_list => { - log.warn("the lazy path value type isn't added from the CLI, but somehow '{s}' is a .{f}", .{ name, std.zig.fmtId(@tagName(gop.value_ptr.value)) }); + log.warn("the lazy path value type isn't added from the CLI, but somehow '{s}' is a .{f}", .{ + name, std.zig.fmtId(@tagName(gop.value_ptr.value)), + }); return true; }, } @@ -1437,7 +1399,7 @@ pub fn addUserInputFlag(b: *Build, name_raw: []const u8) error{OutOfMemory}!bool return false; } -fn typeToEnum(comptime T: type) TypeId { +fn typeToEnum(comptime T: type) Configuration.AvailableOption.Type { return switch (T) { std.zig.BuildId => .build_id, LazyPath => .lazy_path, @@ -1588,17 +1550,6 @@ pub fn path(b: *Build, sub_path: []const u8) LazyPath { } }; } -/// This is low-level implementation details of the build system, not meant to -/// be called by users' build scripts. Even in the build system itself it is a -/// code smell to call this function. -pub fn pathFromRoot(b: *Build, sub_path: []const u8) []u8 { - return b.pathResolve(&.{ b.build_root.path orelse ".", sub_path }); -} - -fn pathFromCwd(b: *Build, sub_path: []const u8) []u8 { - return b.pathResolve(&.{ b.graph.cache.cwd, sub_path }); -} - pub fn pathJoin(b: *Build, paths: []const []const u8) []u8 { return fs.path.join(b.allocator, paths) catch @panic("OOM"); } @@ -1792,7 +1743,9 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c if (@hasDecl(pkg, "build_zig") and pkg.build_zig == asking_build_zig) break .{ pkg_hash, pkg.deps }; } else .{ "", deps.root_deps }; if (!std.mem.eql(u8, b_pkg_hash, b.pkg_hash)) { - std.debug.panic("'{}' is not the struct that corresponds to '{s}'", .{ asking_build_zig, b.pathFromRoot("build.zig") }); + std.debug.panic("'{}' is not the struct that corresponds to '{s}'", .{ + asking_build_zig, b.pathFromRoot("build.zig"), + }); } comptime for (b_pkg_deps) |dep| { if (std.mem.eql(u8, dep[0], dep_name)) return dep[1]; diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 16e91bed505f285baf1ce393f0855636b783de93..4ce04aa05af0f2b590ca479186e0104b3d9d9577 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -189,9 +189,24 @@ pub const Wip = struct { } pub fn addStringList(wip: *Wip, list: []const []const u8) Allocator.Error!StringList { - _ = wip; - _ = list; - @panic("TODO"); + // Increase size of extra to support the list. Add the string list + // there. Then check for duplicate, reverting list if already found. + const gpa = wip.gpa; + const revert_index: u32 = @intCast(wip.extra.items.len); + const added = try wip.extra.addManyAsSlice(gpa, list.len + 1); + added[0] = @intCast(list.len); + for (added[1..], list) |*d, s| d.* = @intFromEnum(try addString(wip, s)); + const gop = try wip.dedupe_table.getOrPutContext(gpa, .{ + .index = revert_index, + .len = @intCast(added.len), + }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items })); + + if (gop.found_existing) { + wip.extra.items.len = revert_index; + return @enumFromInt(gop.key_ptr.index); + } + + return @enumFromInt(revert_index); } pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes { @@ -1456,6 +1471,13 @@ pub const OptionalStringList = enum(u32) { none = max_u32, _, + pub fn init(opt_string_list: ?StringList) OptionalStringList { + const sl = opt_string_list orelse return .none; + const result: OptionalStringList = @enumFromInt(@intFromEnum(sl)); + assert(result != .none); + return result; + } + pub fn unwrap(this: @This()) ?StringList { if (this == .none) return null; return @enumFromInt(@intFromEnum(this)); diff --git a/src/main.zig b/src/main.zig index 912952b174eb139183ff35dcaf4f4ff582ed8d25..1a17195e80de0f124d1d7849e5781e1692dc0a97 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4951,6 +4951,7 @@ fn cmdBuild( .ReleaseSafe; var configure_argv: std.ArrayList([]const u8) = .empty; var make_argv: std.ArrayList([]const u8) = .empty; + var cached_unordered_passthru_configure: std.ArrayList(u32) = .empty; var forks: std.ArrayList(Fork) = .empty; var reference_trace: ?u32 = null; var debug_compile_errors = false; @@ -4975,6 +4976,7 @@ fn cmdBuild( try configure_argv.ensureUnusedCapacity(arena, 16); try make_argv.ensureUnusedCapacity(arena, 16); + try cached_unordered_passthru_configure.ensureUnusedCapacity(arena, 16); _ = configure_argv.addOneAssumeCapacity(); // configurer executable _ = make_argv.addOneAssumeCapacity(); // maker executable @@ -5007,7 +5009,33 @@ fn cmdBuild( while (i < args.len) : (i += 1) { const arg = args[i]; if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "--build-file")) { + try configure_argv.ensureUnusedCapacity(arena, 1); + + if (mem.startsWith(u8, arg, "-D") or + mem.startsWith(u8, arg, "-fsys=") or + mem.startsWith(u8, arg, "-fno-sys=") or + mem.startsWith(u8, arg, "--release=") or + mem.eql(u8, arg, "--release")) + { + try cached_unordered_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--system")) { + if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); + i += 1; + system_pkg_dir_path = args[i]; + + try cached_unordered_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path. + continue; + } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { + color = std.meta.stringToEnum(Color, rest) orelse + fatal("expected --color=[auto|on|off]; found: {s}", .{arg}); + + try cached_unordered_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--build-file")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; build_file = args[i]; @@ -5058,12 +5086,6 @@ fn cmdBuild( .failed = false, }); continue; - } else if (mem.eql(u8, arg, "--system")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - system_pkg_dir_path = args[i]; - try configure_argv.append(arena, "--system"); - continue; } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { fatal("unable to parse reference_trace count '{s}': {t}", .{ num, err }); @@ -5122,14 +5144,6 @@ fn cmdBuild( verbose_llvm_bc = rest; } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { verbose_llvm_cpu_features = true; - } else if (mem.eql(u8, arg, "--color")) { - if (i + 1 >= args.len) fatal("expected [auto|on|off] after {s}", .{arg}); - i += 1; - color = std.meta.stringToEnum(Color, args[i]) orelse { - fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] }); - }; - try configure_argv.appendSlice(arena, &.{ arg, args[i] }); - continue; } else if (mem.cutPrefix(u8, arg, "-j")) |str| { const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| fatal("unable to parse jobs count {s}: {t}", .{ str, err }); @@ -5143,9 +5157,6 @@ fn cmdBuild( make_argv.items[argv_index_seed] = args[i]; continue; } else if (mem.eql(u8, arg, "--")) { - // The rest of the args are supposed to get passed onto - // build runner's `build.args` - try configure_argv.append(arena, "--have-run-args"); try make_argv.appendSlice(arena, args[i..]); break; } @@ -5212,6 +5223,22 @@ fn cmdBuild( defer config_man.deinit(); config_man.hash.addBytes(build_options.version); + const SortContext = struct { + list: []const []const u8, + fn lessThan(this: @This(), lhs: u32, rhs: u32) bool { + return mem.lessThan(u8, this.list[lhs], this.list[rhs]); + } + }; + mem.sortUnstable( + u32, + cached_unordered_passthru_configure.items, + @as(SortContext, .{ .list = configure_argv.items }), + SortContext.lessThan, + ); + for (cached_unordered_passthru_configure.items) |i| { + config_man.hash.addBytes(configure_argv.items[i]); + } + // Normally the build runner is compiled for the host target but here is // some code to help when debugging edits to the build runner so that you // can make sure it compiles successfully on other targets. -- 2.54.0 From 9ae410eec22dd0afaa93c3d2f33f3fa6ce6b7861 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 18:15:05 -0700 Subject: [PATCH 075/179] zig build: actually the configure passthru args are ordered --- src/main.zig | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/src/main.zig b/src/main.zig index 1a17195e80de0f124d1d7849e5781e1692dc0a97..0f048adaffefe7f6ff01cbfbc758f58cb918f893 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4951,7 +4951,7 @@ fn cmdBuild( .ReleaseSafe; var configure_argv: std.ArrayList([]const u8) = .empty; var make_argv: std.ArrayList([]const u8) = .empty; - var cached_unordered_passthru_configure: std.ArrayList(u32) = .empty; + var cached_passthru_configure: std.ArrayList(u32) = .empty; var forks: std.ArrayList(Fork) = .empty; var reference_trace: ?u32 = null; var debug_compile_errors = false; @@ -4976,7 +4976,7 @@ fn cmdBuild( try configure_argv.ensureUnusedCapacity(arena, 16); try make_argv.ensureUnusedCapacity(arena, 16); - try cached_unordered_passthru_configure.ensureUnusedCapacity(arena, 16); + try cached_passthru_configure.ensureUnusedCapacity(arena, 16); _ = configure_argv.addOneAssumeCapacity(); // configurer executable _ = make_argv.addOneAssumeCapacity(); // maker executable @@ -5017,7 +5017,7 @@ fn cmdBuild( mem.startsWith(u8, arg, "--release=") or mem.eql(u8, arg, "--release")) { - try cached_unordered_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); configure_argv.appendAssumeCapacity(arg); continue; } else if (mem.eql(u8, arg, "--system")) { @@ -5025,14 +5025,14 @@ fn cmdBuild( i += 1; system_pkg_dir_path = args[i]; - try cached_unordered_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path. continue; } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { color = std.meta.stringToEnum(Color, rest) orelse fatal("expected --color=[auto|on|off]; found: {s}", .{arg}); - try cached_unordered_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); configure_argv.appendAssumeCapacity(arg); continue; } else if (mem.eql(u8, arg, "--build-file")) { @@ -5223,21 +5223,8 @@ fn cmdBuild( defer config_man.deinit(); config_man.hash.addBytes(build_options.version); - const SortContext = struct { - list: []const []const u8, - fn lessThan(this: @This(), lhs: u32, rhs: u32) bool { - return mem.lessThan(u8, this.list[lhs], this.list[rhs]); - } - }; - mem.sortUnstable( - u32, - cached_unordered_passthru_configure.items, - @as(SortContext, .{ .list = configure_argv.items }), - SortContext.lessThan, - ); - for (cached_unordered_passthru_configure.items) |i| { + for (cached_passthru_configure.items) |i| config_man.hash.addBytes(configure_argv.items[i]); - } // Normally the build runner is compiled for the host target but here is // some code to help when debugging edits to the build runner so that you -- 2.54.0 From 364a1400ffb7c88539a34d86c83245478b096b84 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 18:35:20 -0700 Subject: [PATCH 076/179] configurer: fix compilation in the presence of dependencies --- lib/std/Build.zig | 27 +++++++++------------------ lib/std/Build/Step/Compile.zig | 6 +++--- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index c03df8d552dc7fda7b7ef72a8b19bcf3d58fd614..3fbc1357584105ed4ff1bc6b36803561b6baa0f9 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -34,7 +34,6 @@ available_options_map: std.array_hash_map.String(AvailableOption) = .empty, invalid_user_input: bool, default_step: *Step, top_level_steps: std.StringArrayHashMapUnmanaged(*Step.TopLevel), -install_prefix: []const u8, debug_log_scopes: []const []const u8 = &.{}, /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, /// in particular at `Step` creation. @@ -217,7 +216,6 @@ pub fn create( .user_input_options = UserInputOptionsMap.init(arena), .top_level_steps = .{}, .default_step = undefined, - .install_prefix = undefined, .install_tls = .{ .step = .init(.{ .tag = .top_level, @@ -506,7 +504,7 @@ const OrderedUserValue = union(enum) { hasher.update(sp.sub_path); }, .generated => |gen| { - hasher.update(gen.file.step.owner.pkg_hash); + hasher.update(std.mem.asBytes(&gen.index)); hasher.update(std.mem.asBytes(&gen.up)); hasher.update(gen.sub_path); }, @@ -1728,9 +1726,9 @@ fn findPkgHashOrFatal(b: *Build, name: []const u8) []const u8 { for (b.available_deps) |dep| { if (mem.eql(u8, dep[0], name)) return dep[1]; } - - const full_path = b.pathFromRoot("build.zig.zon"); - std.debug.panic("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file", .{ name, full_path }); + std.log.info("all dependencies used by build.zig must be declared in corresponding build.zig.zon", .{}); + if (b.pkg_hash.len == 0) std.debug.panic("no dependency named {s}", .{name}); + std.debug.panic("no dependency named {s} in {s} ({s})", .{ name, b.dep_prefix, b.pkg_hash }); } inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, comptime dep_name: []const u8) []const u8 { @@ -1931,10 +1929,10 @@ fn userLazyPathsAreTheSame(lhs_lp: LazyPath, rhs_lp: LazyPath) bool { if (lhs_sp.owner != rhs_sp.owner) return false; if (std.mem.eql(u8, lhs_sp.sub_path, rhs_sp.sub_path)) return false; }, - .generated => |lhs_gen| { - const rhs_gen = rhs_lp.generated; + .generated => |*lhs_gen| { + const rhs_gen = &rhs_lp.generated; - if (lhs_gen.file != rhs_gen.file) return false; + if (lhs_gen.index != rhs_gen.index) return false; if (lhs_gen.up != rhs_gen.up) return false; if (std.mem.eql(u8, lhs_gen.sub_path, rhs_gen.sub_path)) return false; }, @@ -1962,7 +1960,6 @@ fn dependencyInner( pkg_deps: AvailableDeps, args: anytype, ) *Dependency { - const io = b.graph.io; const user_input_options = userInputOptionsFromArgs(b.allocator, args); if (b.graph.dependency_cache.getContext(.{ .build_root_string = build_root_string, @@ -1970,13 +1967,7 @@ fn dependencyInner( }, .{ .allocator = b.graph.arena })) |dep| return dep; - const build_root: std.Build.Cache.Directory = .{ - .path = build_root_string, - .handle = Io.Dir.cwd().openDir(io, build_root_string, .{}) catch |err| - process.fatal("unable to open {s}: {t}", .{ build_root_string, err }), - }; - - const sub_builder = b.createChild(name, build_root, pkg_hash, pkg_deps, user_input_options) catch + const sub_builder = b.createChild(name, pkg_hash, pkg_deps, user_input_options) catch @panic("unhandled error"); if (build_zig) |bz| { sub_builder.runBuild(bz) catch @panic("unhandled error"); @@ -2142,7 +2133,7 @@ pub const LazyPath = union(enum) { .sub_path = try fs.path.resolve(arena, &.{ src.sub_path, sub_path }), } }, .generated => |gen| .{ .generated = .{ - .file = gen.file, + .index = gen.index, .up = gen.up, .sub_path = try fs.path.resolve(arena, &.{ gen.sub_path, sub_path }), } }, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index f16c4ee83b2e38b89577cd435b99294846d8621b..2d0cec2bfcdc466e52c3021edf63e2aa0154179d 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -317,10 +317,10 @@ pub const HeaderInstallation = union(enum) { /// `exclude_extensions` takes precedence over `include_extensions`. include_extensions: ?[]const []const u8 = &.{".h"}, - pub fn dupe(opts: Directory.Options, b: *std.Build) Directory.Options { + pub fn dupe(opts: Directory.Options, graph: *std.Build.Graph) Directory.Options { return .{ - .exclude_extensions = b.dupeStrings(opts.exclude_extensions), - .include_extensions = if (opts.include_extensions) |incs| b.dupeStrings(incs) else null, + .exclude_extensions = graph.dupeStrings(opts.exclude_extensions), + .include_extensions = if (opts.include_extensions) |incs| graph.dupeStrings(incs) else null, }; } }; -- 2.54.0 From c57bf9904396c365191d25b9ce6410bb47daa96a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 30 Apr 2026 21:19:04 -0700 Subject: [PATCH 077/179] Configuration: implement Storage.UnionList.tag --- BRANCH_TODO | 4 +++- lib/compiler/Maker/ScannedConfig.zig | 9 +++++++++ lib/compiler/configurer.zig | 2 -- lib/std/Build/Configuration.zig | 9 ++++----- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 2b02293e9a8f20d66192d1710d32d93aa4304a08..1b5f0d8e3df2bc55849c238d52a856e788c9945c 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -2,7 +2,6 @@ * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) * make zig-pkg path root configurable in maker (make sure --system still works) * eliminate calls to getPath, getPath2, getPath3 -* [build system compile step data races with getGraph function](https://codeberg.org/ziglang/zig/issues/31397) * test lazyImport * solve the TODOs added in this branch * get zig tests passing @@ -17,6 +16,7 @@ * refactor with DefaultingEnum * implement {q} or delete {q} uses +* make the generated dependencies.zig be dependencies.zon and don't put absolute paths in there ## Followup Issues * reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make @@ -49,3 +49,5 @@ run_cmd.addPassthruArgs(); This removes a capability from build scripts since they can no longer observe those arguments. In exchange, it means that when changing those arguments, build scripts no longer must be rebuilt from source. + +closes #31397 diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 183146af278abe426d194ed932ed83300525b5aa..487f0b16122ae7987ab048412cc86c7501b17ab6 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -79,6 +79,15 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try printStruct(sc, &sub_struct, Configuration.Step.Run.Arg, field_value.get(c)); try sub_struct.end(); }, + Configuration.LazyPath.Index => { + switch (field_value.get(c)) { + inline else => |u| { + var sub_struct = try s.beginStruct(.{}); + try printStruct(sc, &sub_struct, @TypeOf(u), u); + try sub_struct.end(); + }, + } + }, else => switch (@typeInfo(Field)) { .int => try s.int(field_value), .pointer => |info| switch (info.size) { diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 3fde1e989bde460555f53d0f8bd2150dc541de08..6b034e7cabfb82240e726fd88cc48a153197cc99 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -164,7 +164,6 @@ const Serialize = struct { .src_path => |src_path| i: { const sub_path = try wc.addString(src_path.sub_path); break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ - .flags = .{}, .owner = try s.builderToPackage(src_path.owner), .sub_path = sub_path, })); @@ -187,7 +186,6 @@ const Serialize = struct { .dependency => |dependency| i: { const sub_path = try wc.addString(dependency.sub_path); break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ - .flags = .{}, .owner = try s.builderToPackage(dependency.dependency.builder), .sub_path = sub_path, })); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 4ce04aa05af0f2b590ca479186e0104b3d9d9577..d3a5a2086b19ca8910b6318bd34d7bcf2890d226 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1173,7 +1173,7 @@ pub const LazyPath = union(@This().Tag) { }; pub const SourcePath = struct { - flags: @This().Flags, + flags: @This().Flags = .{}, owner: Package.Index, sub_path: String, @@ -2318,10 +2318,9 @@ pub const Storage = enum { /// Valid to call only when deserializing. pub fn tag(this: *const @This(), extra: []const u32, i: usize) Tag { - _ = this; - _ = extra; - _ = i; - @panic("TODO implement UnionList.tag"); + const start = @intFromPtr(this.data); + const meta_start = start - (this.len * @bitSizeOf(Meta) + 31) / 32; + return loadBits(u32, extra[meta_start..], i * @bitSizeOf(Meta), Meta).tag; } fn extraLen(len: usize) usize { -- 2.54.0 From fa26566867cddbb0cd067cbc1d41bd41e68002a1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 1 May 2026 18:26:19 -0700 Subject: [PATCH 078/179] configurer: get InstallDir and Options steps compiling --- BRANCH_TODO | 1 - build.zig | 13 +- lib/compiler/Maker/Step/InstallDir.zig | 66 +++++++ lib/compiler/Maker/Step/InstallFile.zig | 1 + lib/compiler/Maker/Step/Options.zig | 84 ++++++++ lib/std/Build.zig | 14 +- lib/std/Build/Configuration.zig | 23 ++- lib/std/Build/Step/InstallDir.zig | 75 ++----- lib/std/Build/Step/InstallFile.zig | 17 +- lib/std/Build/Step/Options.zig | 248 +----------------------- 10 files changed, 215 insertions(+), 327 deletions(-) create mode 100644 lib/compiler/Maker/Step/InstallDir.zig create mode 100644 lib/compiler/Maker/Step/Options.zig diff --git a/BRANCH_TODO b/BRANCH_TODO index 1b5f0d8e3df2bc55849c238d52a856e788c9945c..10c0b23bae73bc79ac03d1b365e1bea29ab36fdd 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -26,7 +26,6 @@ - but artifact install steps also add paths for dyn libs on windows * no more "artifact arg" to run step. if you want to run the post-install binary, get the lazy path from the install step. -* -D options which are files need to be accounted for in the configure cache ## Release Notes diff --git a/build.zig b/build.zig index fcbf1a46dc6df7d5dc2aa958488b559f0e891516..1b247074835009a59d202051e2d98d7d82d5bbf8 100644 --- a/build.zig +++ b/build.zig @@ -208,7 +208,8 @@ pub fn build(b: *std.Build) !void { .single_threaded = single_threaded, }); exe.pie = pie; - exe.entitlements = entitlements; + // https://codeberg.org/ziglang/zig/issues/32173 + exe.entitlements = if (entitlements) |p| .{ .cwd_relative = p } else null; exe.use_new_linker = b.option(bool, "new-linker", "Use the new linker"); const use_llvm = b.option(bool, "use-llvm", "Use the llvm backend"); @@ -1498,11 +1499,13 @@ fn generateLangRef(b: *std.Build) std.Build.LazyPath { }), }); - var dir = b.build_root.handle.openDir(io, "doc/langref", .{ .iterate = true }) catch |err| { - std.debug.panic("unable to open '{f}doc/langref' directory: {s}", .{ - b.build_root, @errorName(err), - }); + const langref_path: std.Build.Cache.Path = .{ + .root_dir = b.build_root, + .sub_path = "doc/langref", }; + + var dir = langref_path.root_dir.handle.openDir(io, langref_path.sub_path, .{ .iterate = true }) catch |err| + std.debug.panic("unable to open directory {f}: {t}", .{ langref_path, err }); defer dir.close(io); var wf = b.addWriteFiles(); diff --git a/lib/compiler/Maker/Step/InstallDir.zig b/lib/compiler/Maker/Step/InstallDir.zig new file mode 100644 index 0000000000000000000000000000000000000000..fb5f288c68404ad3307c2293bf5dc8942efcfa6e --- /dev/null +++ b/lib/compiler/Maker/Step/InstallDir.zig @@ -0,0 +1,66 @@ +const InstallDir = @This(); + +const std = @import("std"); +const Configuration = std.Build.Configuration; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + install_dir: *InstallDir, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) !void { + const graph = maker.graph; + const arena = maker.graph.arena; // TODO don't leak into process arena + const io = graph.io; + const step = maker.stepByIndex(step_index); + + step.clearWatchInputs(); + const dest_prefix = b.getInstallPath(install_dir.options.install_dir, install_dir.options.install_subdir); + const src_dir_path = install_dir.options.source_dir.getPath3(b, step); + const need_derived_inputs = try step.addDirectoryWatchInput(install_dir.options.source_dir); + var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{f}': {t}", .{ src_dir_path, err }); + }; + defer src_dir.close(io); + var it = try src_dir.walk(arena); + var all_cached = true; + next_entry: while (try it.next(io)) |entry| { + for (install_dir.options.exclude_extensions) |ext| { + if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; + } + if (install_dir.options.include_extensions) |incs| { + for (incs) |inc| { + if (std.mem.endsWith(u8, entry.path, inc)) break; + } else { + continue :next_entry; + } + } + + const src_path = try install_dir.options.source_dir.join(arena, entry.path); + const dest_path = b.pathJoin(&.{ dest_prefix, entry.path }); + switch (entry.kind) { + .directory => { + if (need_derived_inputs) _ = try step.addDirectoryWatchInput(src_path); + const p = try step.installDir(dest_path); + all_cached = all_cached and p == .existed; + }, + .file => { + for (install_dir.options.blank_extensions) |ext| { + if (std.mem.endsWith(u8, entry.path, ext)) { + try b.truncateFile(dest_path); + continue :next_entry; + } + } + + const p = try step.installFile(src_path, dest_path); + all_cached = all_cached and p == .fresh; + }, + else => continue, + } + } + + step.result_cached = all_cached; +} diff --git a/lib/compiler/Maker/Step/InstallFile.zig b/lib/compiler/Maker/Step/InstallFile.zig index 439c793c6f1c7ef5b8875b6913e2ad6f0785a6e8..eb9547f44ed7149b728e409580be1002329bbfe8 100644 --- a/lib/compiler/Maker/Step/InstallFile.zig +++ b/lib/compiler/Maker/Step/InstallFile.zig @@ -19,6 +19,7 @@ pub fn make( const conf = &maker.scanned_config.configuration; const conf_step = step_index.ptr(conf); const conf_if = conf_step.extended.get(conf.extra).install_file; + try step.singleUnchangingWatchInput(maker, arena, conf_if.source.get(conf)); const p = try maker.installLazyPathSub(arena, conf_if.source, conf_if.dest_dir, conf_if.dest_sub_path.slice(conf), step_index); step.result_cached = p == .fresh; diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig new file mode 100644 index 0000000000000000000000000000000000000000..dcdc49a4a0cc051cb67a64afd3fa20a88b098557 --- /dev/null +++ b/lib/compiler/Maker/Step/Options.zig @@ -0,0 +1,84 @@ +const Options = @This(); + +const std = @import("std"); +const Configuration = std.Build.Configuration; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + + +fn make( + options: *Options, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) !void { + // This step completes so quickly that no progress reporting is necessary. + _ = progress_node; + + const graph = maker.graph; + const step = maker.stepByIndex(step_index); + const io = graph.io; + const cache_root = graph.local_cache_root; + + for (options.args.items) |arg| { + options.addOption( + []const u8, + arg.name, + arg.path.getPath2(b, step), + ); + } + if (!step.inputs.populated()) for (options.args.items) |arg| { + try step.addWatchInput(arg.path); + }; + + const basename = "options.zig"; + + // Hash contents to file name. + var hash = graph.cache.hash; + // Random bytes to make unique. Refresh this with new random bytes when + // implementation is modified in a non-backwards-compatible way. + hash.add(@as(u32, 0xad95e922)); + hash.addBytes(options.contents.items); + const sub_path = "c" ++ fs.path.sep_str ++ hash.final() ++ fs.path.sep_str ++ basename; + + options.generated_file.path = try cache_root.join(arena, &.{sub_path}); + + // Optimize for the hot path. Stat the file, and if it already exists, + // cache hit. + if (cache_root.handle.access(io, sub_path, .{})) |_| { + // This is the hot path, success. + step.result_cached = true; + return; + } else |outer_err| switch (outer_err) { + error.FileNotFound => { + var atomic_file = cache_root.handle.createFileAtomic(io, sub_path, .{ + .replace = false, + .make_path = true, + }) catch |err| return step.fail("failed to create temporary path for '{f}{s}': {t}", .{ + cache_root, sub_path, err, + }); + defer atomic_file.deinit(io); + + atomic_file.file.writeStreamingAll(io, options.contents.items) catch |err| { + return step.fail("failed to write options to temporary path for '{f}{s}': {t}", .{ + cache_root, sub_path, err, + }); + }; + + atomic_file.link(io) catch |err| switch (err) { + error.PathAlreadyExists => { + step.result_cached = true; + return; + }, + else => return step.fail("failed to link temporary file into '{f}{s}': {t}", .{ + cache_root, sub_path, err, + }), + }; + }, + else => |e| return step.fail("unable to access options file '{f}{s}': {t}", .{ + cache_root, sub_path, e, + }), + } +} + diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 3fbc1357584105ed4ff1bc6b36803561b6baa0f9..b6b3263bfa5a57e0fdc89a24f1569932b8e866f4 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -99,19 +99,19 @@ pub const Graph = struct { return @enumFromInt(graph.generated_files.items.len - 1); } - pub fn dupeString(graph: *Graph, bytes: []const u8) []const u8 { + pub fn dupeString(graph: *const Graph, bytes: []const u8) []const u8 { return graph.arena.dupe(u8, bytes) catch @panic("OOM"); } - pub fn dupePath(graph: *Graph, bytes: []const u8) []const u8 { + pub fn dupePath(graph: *const Graph, bytes: []const u8) []const u8 { const arena = graph.arena; - if (builtin.os.tag != .windows) return graph.arena.dupe(u8, bytes) catch @panic("OOM"); + if (builtin.os.tag != .windows) return arena.dupe(u8, bytes) catch @panic("OOM"); const the_copy = arena.dupe(u8, bytes) catch @panic("OOM"); mem.replaceScalar(u8, the_copy, '/', '\\'); return the_copy; } - pub fn dupeStrings(graph: *Graph, strings: []const []const u8) []const []const u8 { + pub fn dupeStrings(graph: *const Graph, strings: []const []const u8) []const []const u8 { const arena = graph.arena; const array = arena.alloc([]const u8, strings.len) catch @panic("OOM"); for (array, strings) |*dest, source| dest.* = dupeString(graph, source); @@ -2186,7 +2186,7 @@ pub const LazyPath = union(enum) { /// /// The `b` parameter is only used for its allocator. All *Build instances /// share the same allocator. - pub fn dupe(lazy_path: LazyPath, graph: *Graph) LazyPath { + pub fn dupe(lazy_path: LazyPath, graph: *const Graph) LazyPath { return switch (lazy_path) { .src_path => |sp| .{ .src_path = .{ .owner = sp.owner, .sub_path = sp.owner.dupePath(sp.sub_path) } }, .cwd_relative => |p| .{ .cwd_relative = graph.dupePath(p) }, @@ -2245,9 +2245,9 @@ pub const InstallDir = union(enum) { custom: []const u8, /// Duplicates the install directory including the path if set to custom. - pub fn dupe(dir: InstallDir, builder: *Build) InstallDir { + pub fn dupe(dir: InstallDir, graph: *const Graph) InstallDir { if (dir == .custom) { - return .{ .custom = builder.dupe(dir.custom) }; + return .{ .custom = graph.dupeString(dir.custom) }; } else { return dir; } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index d3a5a2086b19ca8910b6318bd34d7bcf2890d226..ca1c82074aec6ffd2ed75b819f12799922690287 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1039,10 +1039,20 @@ pub const Step = extern struct { pub const InstallDir = struct { flags: @This().Flags, + source_dir: LazyPath.Index, + dest_dir: InstallDestDir, + dest_sub_path: Storage.FlagOptional(.flags, .dest_sub_path, String), + exclude_extensions: Storage.FlagLengthPrefixedList(.flags, .exclude_extensions, String), + include_extensions: Storage.FlagLengthPrefixedList(.flags, .include_extensions, String), + blank_extensions: Storage.FlagLengthPrefixedList(.flags, .blank_extensions, String), pub const Flags = packed struct(u32) { tag: Tag = .install_dir, - _: u27 = 0, + dest_sub_path: bool, + exclude_extensions: bool, + include_extensions: bool, + blank_extensions: bool, + _: u23 = 0, }; }; @@ -1069,10 +1079,19 @@ pub const Step = extern struct { pub const Options = struct { flags: @This().Flags, + generated_file: GeneratedFileIndex, + contents: Bytes, + args: Storage.FlagLengthPrefixedList(.flags, .args, Arg), + + pub const Arg = extern struct { + name: String, + path: LazyPath.Index, + }; pub const Flags = packed struct(u32) { tag: Tag = .options, - _: u27 = 0, + args: bool, + _: u26 = 0, }; }; diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index f755a28662b24f60a08157bef23a2a2b3b40c5e1..f1a904cba5d150a5c21b84bd4a69c1eb52721d63 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -1,9 +1,10 @@ +const InstallDir = @This(); + const std = @import("std"); const mem = std.mem; const fs = std.fs; const Step = std.Build.Step; const LazyPath = std.Build.LazyPath; -const InstallDir = @This(); step: Step, options: Options, @@ -28,83 +29,29 @@ pub const Options = struct { /// `@import("test.zig")` would be a compile error. blank_extensions: []const []const u8 = &.{}, - fn dupe(opts: Options, b: *std.Build) Options { + fn dupe(opts: Options, graph: *const std.Build.Graph) Options { return .{ - .source_dir = opts.source_dir.dupe(b), - .install_dir = opts.install_dir.dupe(b), - .install_subdir = b.dupe(opts.install_subdir), - .exclude_extensions = b.dupeStrings(opts.exclude_extensions), - .include_extensions = if (opts.include_extensions) |incs| b.dupeStrings(incs) else null, - .blank_extensions = b.dupeStrings(opts.blank_extensions), + .source_dir = opts.source_dir.dupe(graph), + .install_dir = opts.install_dir.dupe(graph), + .install_subdir = graph.dupeString(opts.install_subdir), + .exclude_extensions = graph.dupeStrings(opts.exclude_extensions), + .include_extensions = if (opts.include_extensions) |incs| graph.dupeStrings(incs) else null, + .blank_extensions = graph.dupeStrings(opts.blank_extensions), }; } }; pub fn create(owner: *std.Build, options: Options) *InstallDir { const install_dir = owner.allocator.create(InstallDir) catch @panic("OOM"); + const graph = owner.graph; install_dir.* = .{ .step = Step.init(.{ .tag = base_tag, .name = owner.fmt("install {s}/", .{options.source_dir.getDisplayName()}), .owner = owner, - .makeFn = make, }), - .options = options.dupe(owner), + .options = options.dupe(graph), }; options.source_dir.addStepDependencies(&install_dir.step); return install_dir; } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const io = b.graph.io; - const install_dir: *InstallDir = @fieldParentPtr("step", step); - step.clearWatchInputs(); - const arena = b.allocator; - const dest_prefix = b.getInstallPath(install_dir.options.install_dir, install_dir.options.install_subdir); - const src_dir_path = install_dir.options.source_dir.getPath3(b, step); - const need_derived_inputs = try step.addDirectoryWatchInput(install_dir.options.source_dir); - var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {t}", .{ src_dir_path, err }); - }; - defer src_dir.close(io); - var it = try src_dir.walk(arena); - var all_cached = true; - next_entry: while (try it.next(io)) |entry| { - for (install_dir.options.exclude_extensions) |ext| { - if (mem.endsWith(u8, entry.path, ext)) continue :next_entry; - } - if (install_dir.options.include_extensions) |incs| { - for (incs) |inc| { - if (mem.endsWith(u8, entry.path, inc)) break; - } else { - continue :next_entry; - } - } - - const src_path = try install_dir.options.source_dir.join(b.allocator, entry.path); - const dest_path = b.pathJoin(&.{ dest_prefix, entry.path }); - switch (entry.kind) { - .directory => { - if (need_derived_inputs) _ = try step.addDirectoryWatchInput(src_path); - const p = try step.installDir(dest_path); - all_cached = all_cached and p == .existed; - }, - .file => { - for (install_dir.options.blank_extensions) |ext| { - if (mem.endsWith(u8, entry.path, ext)) { - try b.truncateFile(dest_path); - continue :next_entry; - } - } - - const p = try step.installFile(src_path, dest_path); - all_cached = all_cached and p == .fresh; - }, - else => continue, - } - } - - step.result_cached = all_cached; -} diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index 5b80b3d1a1b1b7c2b7103e1e196fa8766e316de3..f143e43e4e0b30c1c088309adb43d7f1ef127c1c 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -1,17 +1,18 @@ +const InstallFile = @This(); + const std = @import("std"); const Step = std.Build.Step; const LazyPath = std.Build.LazyPath; const InstallDir = std.Build.InstallDir; -const InstallFile = @This(); const assert = std.debug.assert; -pub const base_tag: Step.Tag = .install_file; - step: Step, source: LazyPath, dir: InstallDir, dest_rel_path: []const u8, +pub const base_tag: Step.Tag = .install_file; + pub fn create( owner: *std.Build, source: LazyPath, @@ -19,16 +20,18 @@ pub fn create( dest_rel_path: []const u8, ) *InstallFile { assert(dest_rel_path.len != 0); - const install_file = owner.allocator.create(InstallFile) catch @panic("OOM"); + const graph = owner.graph; + const arena = graph.arena; + const install_file = arena.create(InstallFile) catch @panic("OOM"); install_file.* = .{ .step = Step.init(.{ .tag = base_tag, .name = owner.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), .owner = owner, }), - .source = source.dupe(owner), - .dir = dir.dupe(owner), - .dest_rel_path = owner.dupePath(dest_rel_path), + .source = source.dupe(graph), + .dir = dir.dupe(graph), + .dest_rel_path = graph.dupePath(dest_rel_path), }; source.addStepDependencies(&install_file.step); return install_file; diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 1bf8266ec0841c97c282fc310356fc4aaf7b6608..935aec618c3bc54f25970f7ef16b40afd0edc55a 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -9,15 +9,19 @@ const Step = std.Build.Step; const LazyPath = std.Build.LazyPath; const Configuration = std.Build.Configuration; -pub const base_tag: Step.Tag = .options; - step: Step, generated_file: Configuration.GeneratedFileIndex, - contents: std.ArrayList(u8), args: std.ArrayList(Arg), encountered_types: std.StringHashMapUnmanaged(void), +pub const base_tag: Step.Tag = .options; + +pub const Arg = struct { + name: []const u8, + path: LazyPath, +}; + pub fn create(owner: *std.Build) *Options { const graph = owner.graph; const arena = graph.arena; @@ -28,7 +32,6 @@ pub fn create(owner: *std.Build) *Options { .tag = base_tag, .name = "options", .owner = owner, - .makeFn = make, }), .generated_file = graph.addGeneratedFile(&options.step), .contents = .empty, @@ -439,240 +442,3 @@ pub fn createModule(options: *Options) *std.Build.Module { pub fn getOutput(options: *Options) LazyPath { return .{ .generated = .{ .index = options.generated_file } }; } - -fn make(step: *Step, make_options: Step.MakeOptions) !void { - // This step completes so quickly that no progress reporting is necessary. - _ = make_options; - - const b = step.owner; - const io = b.graph.io; - const options: *Options = @fieldParentPtr("step", step); - - for (options.args.items) |item| { - options.addOption( - []const u8, - item.name, - item.path.getPath2(b, step), - ); - } - if (!step.inputs.populated()) for (options.args.items) |item| { - try step.addWatchInput(item.path); - }; - - const basename = "options.zig"; - - // Hash contents to file name. - var hash = b.graph.cache.hash; - // Random bytes to make unique. Refresh this with new random bytes when - // implementation is modified in a non-backwards-compatible way. - hash.add(@as(u32, 0xad95e922)); - hash.addBytes(options.contents.items); - const sub_path = "c" ++ fs.path.sep_str ++ hash.final() ++ fs.path.sep_str ++ basename; - - options.generated_file.path = try b.cache_root.join(b.allocator, &.{sub_path}); - - // Optimize for the hot path. Stat the file, and if it already exists, - // cache hit. - if (b.cache_root.handle.access(io, sub_path, .{})) |_| { - // This is the hot path, success. - step.result_cached = true; - return; - } else |outer_err| switch (outer_err) { - error.FileNotFound => { - var atomic_file = b.cache_root.handle.createFileAtomic(io, sub_path, .{ - .replace = false, - .make_path = true, - }) catch |err| return step.fail("failed to create temporary path for '{f}{s}': {t}", .{ - b.cache_root, sub_path, err, - }); - defer atomic_file.deinit(io); - - atomic_file.file.writeStreamingAll(io, options.contents.items) catch |err| { - return step.fail("failed to write options to temporary path for '{f}{s}': {t}", .{ - b.cache_root, sub_path, err, - }); - }; - - atomic_file.link(io) catch |err| switch (err) { - error.PathAlreadyExists => { - step.result_cached = true; - return; - }, - else => return step.fail("failed to link temporary file into '{f}{s}': {t}", .{ - b.cache_root, sub_path, err, - }), - }; - }, - else => |e| return step.fail("unable to access options file '{f}{s}': {t}", .{ - b.cache_root, sub_path, e, - }), - } -} - -const Arg = struct { - name: []const u8, - path: LazyPath, -}; - -test Options { - if (builtin.os.tag == .wasi) return error.SkipZigTest; - - const io = std.testing.io; - - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - - const cwd = try std.process.currentPathAlloc(io, std.testing.allocator); - defer std.testing.allocator.free(cwd); - - var graph: std.Build.Graph = .{ - .io = io, - .arena = arena.allocator(), - .cache = .{ - .io = io, - .gpa = arena.allocator(), - .manifest_dir = Io.Dir.cwd(), - .cwd = cwd, - }, - .zig_exe = "test", - .environ_map = std.process.Environ.Map.init(arena.allocator()), - .global_cache_root = .{ .path = "test", .handle = Io.Dir.cwd() }, - .host = .{ - .query = .{}, - .result = try std.zig.system.resolveTargetQuery(io, .{}), - }, - .zig_lib_directory = std.Build.Cache.Directory.cwd(), - .time_report = false, - }; - - var builder = try std.Build.create( - &graph, - .{ .path = "test", .handle = Io.Dir.cwd() }, - .{ .path = "test", .handle = Io.Dir.cwd() }, - &.{}, - ); - - const options = builder.addOptions(); - - const KeywordEnum = enum { - @"0.8.1", - }; - - const NormalEnum = enum { - foo, - bar, - }; - - const nested_array = [2][2]u16{ - [2]u16{ 300, 200 }, - [2]u16{ 300, 200 }, - }; - const nested_slice: []const []const u16 = &[_][]const u16{ &nested_array[0], &nested_array[1] }; - - const NormalStruct = struct { - hello: ?[]const u8, - world: bool = true, - }; - - const NestedStruct = struct { - normal_struct: NormalStruct, - normal_enum: NormalEnum = .foo, - }; - - options.addOption(usize, "option1", 1); - options.addOption(?usize, "option2", null); - options.addOption(?usize, "option3", 3); - options.addOption(comptime_int, "option4", 4); - options.addOption(comptime_float, "option5", 5.01); - options.addOption([]const u8, "string", "zigisthebest"); - options.addOption(?[]const u8, "optional_string", null); - options.addOption([2][2]u16, "nested_array", nested_array); - options.addOption([]const []const u16, "nested_slice", nested_slice); - options.addOption(KeywordEnum, "keyword_enum", .@"0.8.1"); - options.addOption(std.SemanticVersion, "semantic_version", try std.SemanticVersion.parse("0.1.2-foo+bar")); - options.addOption(NormalEnum, "normal1_enum", NormalEnum.foo); - options.addOption(NormalEnum, "normal2_enum", NormalEnum.bar); - options.addOption(NormalStruct, "normal1_struct", NormalStruct{ - .hello = "foo", - }); - options.addOption(NormalStruct, "normal2_struct", NormalStruct{ - .hello = null, - .world = false, - }); - options.addOption(NestedStruct, "nested_struct", NestedStruct{ - .normal_struct = .{ .hello = "bar" }, - }); - - try std.testing.expectEqualStrings( - \\pub const option1: usize = 1; - \\pub const option2: ?usize = null; - \\pub const option3: ?usize = 3; - \\pub const option4: comptime_int = 4; - \\pub const option5: comptime_float = 5.01; - \\pub const string: []const u8 = "zigisthebest"; - \\pub const optional_string: ?[]const u8 = null; - \\pub const nested_array: [2][2]u16 = [2][2]u16 { - \\ [2]u16 { - \\ 300, - \\ 200, - \\ }, - \\ [2]u16 { - \\ 300, - \\ 200, - \\ }, - \\}; - \\pub const nested_slice: []const []const u16 = &[_][]const u16 { - \\ &[_]u16 { - \\ 300, - \\ 200, - \\ }, - \\ &[_]u16 { - \\ 300, - \\ 200, - \\ }, - \\}; - \\pub const @"Build.Step.Options.decltest.Options.KeywordEnum" = enum (u0) { - \\ @"0.8.1" = 0, - \\}; - \\pub const keyword_enum: @"Build.Step.Options.decltest.Options.KeywordEnum" = .@"0.8.1"; - \\pub const semantic_version: @import("std").SemanticVersion = .{ - \\ .major = 0, - \\ .minor = 1, - \\ .patch = 2, - \\ .pre = "foo", - \\ .build = "bar", - \\}; - \\pub const @"Build.Step.Options.decltest.Options.NormalEnum" = enum (u1) { - \\ foo = 0, - \\ bar = 1, - \\}; - \\pub const normal1_enum: @"Build.Step.Options.decltest.Options.NormalEnum" = .foo; - \\pub const normal2_enum: @"Build.Step.Options.decltest.Options.NormalEnum" = .bar; - \\pub const @"Build.Step.Options.decltest.Options.NormalStruct" = struct { - \\ hello: ?[]const u8, - \\ world: bool = true, - \\}; - \\pub const normal1_struct: @"Build.Step.Options.decltest.Options.NormalStruct" = .{ - \\ .hello = "foo", - \\ .world = true, - \\}; - \\pub const normal2_struct: @"Build.Step.Options.decltest.Options.NormalStruct" = .{ - \\ .hello = null, - \\ .world = false, - \\}; - \\pub const @"Build.Step.Options.decltest.Options.NestedStruct" = struct { - \\ normal_struct: @"Build.Step.Options.decltest.Options.NormalStruct", - \\ normal_enum: @"Build.Step.Options.decltest.Options.NormalEnum" = .foo, - \\}; - \\pub const nested_struct: @"Build.Step.Options.decltest.Options.NestedStruct" = .{ - \\ .normal_struct = .{ - \\ .hello = "bar", - \\ .world = true, - \\ }, - \\ .normal_enum = .foo, - \\}; - \\ - , options.contents.items); - - _ = try std.zig.Ast.parse(arena.allocator(), try options.contents.toOwnedSliceSentinel(arena.allocator(), 0), .zig); -} -- 2.54.0 From ddabd57743579818a05016e031b4919c47b4428a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 3 May 2026 22:21:13 -0700 Subject: [PATCH 079/179] progress towards compiling zig's build script --- BRANCH_TODO | 1 + build.zig | 54 +++-- lib/compiler/Maker.zig | 9 + lib/compiler/Maker/Graph.zig | 9 +- lib/compiler/Maker/Step.zig | 6 +- lib/compiler/Maker/Step/Run.zig | 9 +- lib/compiler/Maker/WebServer.zig | 12 +- lib/compiler/configurer.zig | 34 ++- lib/std/Build.zig | 342 ++++++++++++++++++---------- lib/std/Build/Configuration.zig | 21 ++ lib/std/Build/Module.zig | 136 ++++++----- lib/std/Build/Step/ConfigHeader.zig | 2 +- lib/std/Build/Step/InstallDir.zig | 2 +- lib/std/Build/Step/InstallFile.zig | 2 +- lib/std/Build/Step/ObjCopy.zig | 4 +- lib/std/zig.zig | 31 ++- src/main.zig | 13 +- 17 files changed, 438 insertions(+), 249 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 10c0b23bae73bc79ac03d1b365e1bea29ab36fdd..c51f87fd6916014f8f61687a37e4b21109fb5805 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -17,6 +17,7 @@ * implement {q} or delete {q} uses * make the generated dependencies.zig be dependencies.zon and don't put absolute paths in there + - and adjust dependencyInner to not openDir() ## Followup Issues * reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make diff --git a/build.zig b/build.zig index 1b247074835009a59d202051e2d98d7d82d5bbf8..5400169113ebc1ffac82a3d54df215ea1238dc03 100644 --- a/build.zig +++ b/build.zig @@ -16,6 +16,8 @@ const IoMode = enum { threaded, evented }; const ValueInterpretMode = enum { direct, by_name }; pub fn build(b: *std.Build) !void { + const arena = b.graph.arena; + const only_c = b.option(bool, "only-c", "Translate the Zig compiler to C code, with only the C backend enabled") orelse false; const target = b.standardTargetOptions(.{ .default_target = .{ @@ -35,7 +37,7 @@ pub fn build(b: *std.Build) !void { const no_bin = b.option(bool, "no-bin", "skip emitting compiler binary") orelse false; const enable_superhtml = b.option(bool, "enable-superhtml", "Check langref output HTML validity") orelse false; - const langref_file = generateLangRef(b); + const langref_file = try generateLangRef(b); const install_langref = b.addInstallFileWithDir(langref_file, .prefix, "doc/langref.html"); const check_langref = superHtmlCheck(b, langref_file); if (enable_superhtml) install_langref.step.dependOn(check_langref); @@ -262,7 +264,7 @@ pub fn build(b: *std.Build) !void { var code: u8 = undefined; const git_describe_untrimmed = b.runAllowFail(&[_][]const u8{ "git", - "-C", b.build_root.path orelse ".", // affects the --git-dir argument + "-C", b.fmt("{f}", .{b.root}), // affects the --git-dir argument "--git-dir", ".git", // affected by the -C argument "describe", "--match", "*.*.*", // "--tags", "--abbrev=9", @@ -308,7 +310,7 @@ pub fn build(b: *std.Build) !void { }, } }; - const version = try b.allocator.dupeSentinel(u8, version_slice, 0); + const version = try arena.dupeSentinel(u8, version_slice, 0); exe_options.addOption([:0]const u8, "version", version); if (enable_llvm) { @@ -316,7 +318,7 @@ pub fn build(b: *std.Build) !void { const io = b.graph.io; const cwd: Io.Dir = .cwd(); if (findConfigH(b, config_h_path_option)) |config_h_path| { - const file_contents = cwd.readFileAlloc(io, config_h_path, b.allocator, .limited(max_config_h_bytes)) catch unreachable; + const file_contents = cwd.readFileAlloc(io, config_h_path, arena, .limited(max_config_h_bytes)) catch unreachable; break :blk parseConfigH(b, file_contents); } else { std.log.warn("config.h could not be located automatically. Consider providing it explicitly via \"-Dconfig_h\"", .{}); @@ -976,11 +978,12 @@ fn addCxxKnownPath( errtxt: ?[]const u8, need_cpp_includes: bool, ) !void { - if (!std.process.can_spawn) - return error.RequiredLibraryNotFound; + if (!std.process.can_spawn) return error.RequiredLibraryNotFound; + + const arena = b.graph.arena; const path_padded = run: { - var args = std.array_list.Managed([]const u8).init(b.allocator); + var args = std.array_list.Managed([]const u8).init(arena); try args.append(ctx.cxx_compiler); var it = std.mem.tokenizeAny(u8, ctx.cxx_compiler_arg1, &std.ascii.whitespace); while (it.next()) |arg| try args.append(arg); @@ -1049,6 +1052,7 @@ const CMakeConfig = struct { const max_config_h_bytes = 1 * 1024 * 1024; fn findConfigH(b: *std.Build, config_h_path_option: ?[]const u8) ?[]const u8 { + const arena = b.graph.arena; const io = b.graph.io; const cwd: Io.Dir = .cwd(); @@ -1073,7 +1077,7 @@ fn findConfigH(b: *std.Build, config_h_path_option: ?[]const u8) ?[]const u8 { if (config_h_or_err) |*file| { file.close(io); return fs.path.join( - b.allocator, + arena, &[_][]const u8{ check_dir, "config.h" }, ) catch unreachable; } else |e| switch (e) { @@ -1198,7 +1202,8 @@ fn parseConfigH(b: *std.Build, config_h_text: []const u8) ?CMakeConfig { } fn toNativePathSep(b: *std.Build, s: []const u8) []u8 { - const duplicated = b.allocator.dupe(u8, s) catch unreachable; + const arena = b.graph.arena; + const duplicated = arena.dupe(u8, s) catch unreachable; for (duplicated) |*byte| switch (byte.*) { '/' => byte.* = fs.path.sep, else => {}, @@ -1487,8 +1492,9 @@ const llvm_libs_xtensa = [_][]const u8{ "LLVMXtensaInfo", }; -fn generateLangRef(b: *std.Build) std.Build.LazyPath { +fn generateLangRef(b: *std.Build) !std.Build.LazyPath { const io = b.graph.io; + const arena = b.graph.arena; const doctest_exe = b.addExecutable(.{ .name = "doctest", @@ -1499,10 +1505,7 @@ fn generateLangRef(b: *std.Build) std.Build.LazyPath { }), }); - const langref_path: std.Build.Cache.Path = .{ - .root_dir = b.build_root, - .sub_path = "doc/langref", - }; + const langref_path = try b.root.join(arena, "doc/langref"); var dir = langref_path.root_dir.handle.openDir(io, langref_path.sub_path, .{ .iterate = true }) catch |err| std.debug.panic("unable to open directory {f}: {t}", .{ langref_path, err }); @@ -1518,17 +1521,22 @@ fn generateLangRef(b: *std.Build) std.Build.LazyPath { const out_basename = b.fmt("{s}.out", .{std.fs.path.stem(entry.name)}); const cmd = b.addRunArtifact(doctest_exe); - cmd.addArgs(&.{ - "--zig", b.graph.zig_exe, - // TODO: enhance doctest to use "--listen=-" rather than operating - // in a temporary directory - "--cache-root", b.cache_root.path orelse ".", - }); - cmd.addArgs(&.{ "--zig-lib-dir", b.fmt("{f}", .{b.graph.zig_lib_directory}) }); - cmd.addArgs(&.{"-i"}); + + cmd.addArg("--zig"); + cmd.addFileArg(.zig_exe); + + // TODO: enhance doctest to use "--listen=-" rather than operating in a + // temporary directory + cmd.addArg("--cache-root"); + cmd.addFileArg(.cache_root); + + cmd.addArg("--zig-lib-dir"); + cmd.addFileArg(.zig_lib); + + cmd.addArg("-i"); cmd.addFileArg(b.path(b.fmt("doc/langref/{s}", .{entry.name}))); - cmd.addArgs(&.{"-o"}); + cmd.addArg("-o"); _ = wf.addCopyFile(cmd.addOutputFileArg(out_basename), out_basename); } diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 81fb2fa71c2080d5200d82897f324530cce39e78..e8c409980087043a114cd3dc91131b727e53f9bb 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1779,6 +1779,7 @@ pub fn relativePath(maker: *const Maker, relative: Configuration.LazyPath.Relati const graph = maker.graph; const c = &maker.scanned_config.configuration; const sub_path = relative.sub_path.slice(c); + if (relative.flags.base == .zig_exe and sub_path.len != 0) @panic("TODO"); return switch (relative.flags.base) { .cwd => .{ .root_dir = .cwd(), @@ -1796,6 +1797,14 @@ pub fn relativePath(maker: *const Maker, relative: Configuration.LazyPath.Relati .root_dir = graph.build_root_directory, .sub_path = sub_path, }, + .zig_exe => .{ + .root_dir = .cwd(), + .sub_path = graph.zig_exe, + }, + .zig_lib => .{ + .root_dir = graph.zig_lib_directory, + .sub_path = sub_path, + }, }; } diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index 3a3136d8783c12e12424cceff48085aa6784f026..4e1c05b8c56ffdc554f9e52510a14a887c99b80a 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -75,10 +75,11 @@ pub fn handleVerbose( ) error{OutOfMemory}!void { if (!graph.verbose) return; const arena = graph.arena; - const text = try std.zig.allocPrintCmd(arena, cwd, if (opt_env) |env| .{ - .child = env, - .parent = &graph.environ_map, - } else null, argv); + const text = try std.zig.allocPrintCmd(arena, argv, .{ + .cwd = cwd, + .parent_env = &graph.environ_map, + .child_env = opt_env, + }); defer arena.free(text); std.log.scoped(.verbose).info("{s}", .{text}); } diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 9d4257a2967ddb3ea44ceaf437b09cb87bc776b4..d67ee25ebc839cfb89c7fd30134a004ba301882e 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -70,6 +70,7 @@ pub const Extended = union(enum) { compile: Compile, config_header: Todo, fail: Todo, + find_program: Todo, fmt: Todo, install_artifact: InstallArtifact, install_dir: Todo, @@ -89,6 +90,7 @@ pub const Extended = union(enum) { .compile => .{ .compile = .{} }, .config_header => .{ .config_header = .{} }, .fail => .{ .fail = .{} }, + .find_program => .{ .find_program = .{} }, .fmt => .{ .fmt = .{} }, .install_artifact => .{ .install_artifact = .{} }, .install_dir => .{ .install_dir = .{} }, @@ -314,7 +316,7 @@ pub fn captureChildProcess( // If an error occurs, it's happened in this command: assert(s.result_failed_command == null); - s.result_failed_command = try std.zig.allocPrintCmd(gpa, .inherit, null, argv); + s.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{}); try handleChildProcUnsupported(s, maker); try graph.handleVerbose(.inherit, null, argv); @@ -382,7 +384,7 @@ pub fn evalZigProcess( // If an error occurs, it's happened in this command: assert(s.result_failed_command == null); - s.result_failed_command = try std.zig.allocPrintCmd(gpa, .inherit, null, argv); + s.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{}); if (s.getZigProcess()) |zp| update: { assert(watch); diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index c381c480cc1fc0a24101af657d6351ee0aad4918..30ffc13dc45d0e885c40129f4b11e41267606e74 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -2129,10 +2129,11 @@ fn spawnChildAndCollect( // If an error occurs, it's caused by this command: assert(step.result_failed_command == null); - step.result_failed_command = try std.zig.allocPrintCmd(arena, child_cwd, .{ - .child = environ_map, - .parent = &graph.environ_map, - }, argv); + step.result_failed_command = try std.zig.allocPrintCmd(arena, argv, .{ + .cwd = child_cwd, + .child_env = environ_map, + .parent_env = &graph.environ_map, + }); try step.handleChildProcUnsupported(maker); diff --git a/lib/compiler/Maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig index 9ad6580868017b134d71bfef58d034e287f495ab..de30ca30730b0d859dd599ce6165344d606e1281 100644 --- a/lib/compiler/Maker/WebServer.zig +++ b/lib/compiler/Maker/WebServer.zig @@ -714,7 +714,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim if (code != 0) { log.err( "the following command exited with error code {d}:\n{s}", - .{ code, try std.zig.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ code, try std.zig.allocPrintCmd(arena, argv.items, .{}) }, ); return error.WasmCompilationFailed; } @@ -722,21 +722,21 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim .signal => |sig| { log.err( "the following command terminated with signal {t}:\n{s}", - .{ sig, try std.zig.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ sig, try std.zig.allocPrintCmd(arena, argv.items, .{}) }, ); return error.WasmCompilationFailed; }, .stopped => |sig| { log.err( "the following command stopped unexpectedly with signal {t}:\n{s}", - .{ sig, try std.zig.allocPrintCmd(arena, .inherit, null, argv.items) }, + .{ sig, try std.zig.allocPrintCmd(arena, argv.items, .{}) }, ); return error.WasmCompilationFailed; }, .unknown => { log.err( "the following command terminated unexpectedly:\n{s}", - .{try std.zig.allocPrintCmd(arena, .inherit, null, argv.items)}, + .{try std.zig.allocPrintCmd(arena, argv.items, .{})}, ); return error.WasmCompilationFailed; }, @@ -746,14 +746,14 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim try result_error_bundle.renderToStderr(io, .{}, .auto); log.err("the following command failed with {d} compilation errors:\n{s}", .{ result_error_bundle.errorMessageCount(), - try std.zig.allocPrintCmd(arena, .inherit, null, argv.items), + try std.zig.allocPrintCmd(arena, argv.items, .{}), }); return error.WasmCompilationFailed; } const base_path = result orelse { log.err("child process failed to report result\n{s}", .{ - try std.zig.allocPrintCmd(arena, .inherit, null, argv.items), + try std.zig.allocPrintCmd(arena, argv.items, .{}), }); return error.WasmCompilationFailed; }; diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 6b034e7cabfb82240e726fd88cc48a153197cc99..96d1072c392550b21109cf6d93f969490bd1749b 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -62,11 +62,23 @@ pub fn main(init: process.Init.Minimal) !void { assert(try graph.wip_configuration.addString("") == .empty); assert(try graph.wip_configuration.addString("root") == .root); - const builder = try std.Build.create(&graph, dependencies.root_deps); - - var color: Color = .auto; var arg_i: usize = 1; // Skip own executable name. + const build_root_sub_path = expectArgOrFatal(args, &arg_i, "--build-root"); + + const cwd: Io.Dir = .cwd(); + + const build_root: std.Build.Cache.Path = .{ + .root_dir = .{ + .handle = try cwd.openDir(io, build_root_sub_path, .{}), + .path = build_root_sub_path, + }, + }; + + const builder = try std.Build.create(&graph, build_root, dependencies.root_deps); + + var color: Color = .auto; + while (nextArg(args, &arg_i)) |arg| { if (mem.cutPrefix(u8, arg, "-D")) |option_contents| { if (option_contents.len == 0) @@ -98,6 +110,8 @@ pub fn main(init: process.Init.Minimal) !void { // but it is handled by the parent process. The build runner // only sees this flag. graph.system_package_mode = true; + } else if (mem.eql(u8, arg, "--verbose")) { + graph.verbose = true; } else { fatalWithHint("unrecognized argument: {s}", .{arg}); } @@ -183,6 +197,12 @@ const Serialize = struct { .sub_path = sub_path, })); }, + .relative => |relative| i: { + break :i try wc.addExtra(@as(Configuration.LazyPath.Relative, .{ + .flags = .{ .base = relative.base }, + .sub_path = relative.sub_path, + })); + }, .dependency => |dependency| i: { const sub_path = try wc.addString(dependency.sub_path); break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ @@ -840,6 +860,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .install_dir => @panic("TODO"), .remove_dir => @panic("TODO"), .fail => @panic("TODO"), + .find_program => @panic("TODO"), .fmt => @panic("TODO"), .translate_c => @panic("TODO"), .write_file => @panic("TODO"), @@ -1038,6 +1059,13 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { return args[idx.*]; } +fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { + const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); + if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); + const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); + return arg; +} + const ErrorStyle = enum { verbose, minimal, diff --git a/lib/std/Build.zig b/lib/std/Build.zig index b6b3263bfa5a57e0fdc89a24f1569932b8e866f4..8bc4f6af407979e3f48feb69cd803520154065bd 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -34,6 +34,8 @@ available_options_map: std.array_hash_map.String(AvailableOption) = .empty, invalid_user_input: bool, default_step: *Step, top_level_steps: std.StringArrayHashMapUnmanaged(*Step.TopLevel), +/// Path to the directory containing build.zig. +root: Cache.Path, debug_log_scopes: []const []const u8 = &.{}, /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, /// in particular at `Step` creation. @@ -85,6 +87,7 @@ pub const Graph = struct { dependency_cache: InitializedDepMap = .empty, allow_so_scripts: ?bool = null, time_report: bool = false, + verbose: bool = false, /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also /// respects the '--color' flag. stderr_mode: ?Io.Terminal.Mode = null, @@ -117,6 +120,20 @@ pub const Graph = struct { for (array, strings) |*dest, source| dest.* = dupeString(graph, source); return array; } + + /// An absolute path or a path relative to the current working directory of + /// the build runner process. + /// + /// Use of this function indicates a dependency on the host system. + pub fn cwdRelativePath(graph: *Graph, sub_path: []const u8) LazyPath { + const wc = &graph.wip_configuration; + return .{ + .relative = .{ + .base = .cwd, + .sub_path = wc.addString(sub_path) catch @panic("OOM"), + }, + }; + } }; const AvailableDeps = []const struct { []const u8, []const u8 }; @@ -170,13 +187,6 @@ const InitializedDepContext = struct { } }; -pub const RunError = error{ - ReadFailure, - ExitCodeFailure, - ProcessTerminated, - ExecNotSupported, -} || std.process.SpawnError; - const UserInputOptionsMap = StringHashMap(UserInputOption); const AvailableOption = struct { @@ -204,6 +214,7 @@ const UserValue = union(enum) { pub fn create( graph: *Graph, + root: Cache.Path, available_deps: AvailableDeps, ) error{OutOfMemory}!*Build { const arena = graph.arena; @@ -211,6 +222,7 @@ pub fn create( const b = try arena.create(Build); b.* = .{ .graph = graph, + .root = root, .invalid_user_input = false, .allocator = arena, .user_input_options = UserInputOptionsMap.init(arena), @@ -247,6 +259,7 @@ pub fn create( fn createChild( parent: *Build, dep_name: []const u8, + root: Cache.Path, pkg_hash: []const u8, pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap, @@ -255,6 +268,7 @@ fn createChild( const child = try allocator.create(Build); child.* = .{ .graph = parent.graph, + .root = root, .allocator = allocator, .install_tls = .{ .step = .init(.{ @@ -1143,7 +1157,7 @@ pub fn step(b: *Build, name: []const u8, description: []const u8) *Step { .description = b.dupe(description), }; const gop = b.top_level_steps.getOrPut(b.allocator, name) catch @panic("OOM"); - if (gop.found_existing) std.debug.panic("A top-level step with name \"{s}\" already exists", .{name}); + if (gop.found_existing) panic("A top-level step with name \"{s}\" already exists", .{name}); gop.key_ptr.* = step_info.step.name; gop.value_ptr.* = step_info; @@ -1366,7 +1380,8 @@ pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8 } pub fn addUserInputFlag(b: *Build, name_raw: []const u8) error{OutOfMemory}!bool { - const name = b.dupe(name_raw); + const graph = b.graph; + const name = graph.dupeString(name_raw); const gop = try b.user_input_options.getOrPut(name); if (!gop.found_existing) { gop.value_ptr.* = .{ @@ -1388,7 +1403,7 @@ pub fn addUserInputFlag(b: *Build, name_raw: []const u8) error{OutOfMemory}!bool return true; }, .lazy_path => |lp| { - log.err("Flag '-D{s}' conflicts with option '-D{s}={s}'.", .{ name, name, lp.getDisplayName() }); + log.err("Flag '-D{s}' conflicts with option '-D{s}={f}'.", .{ name, name, lp.fmt(graph) }); return true; }, @@ -1538,7 +1553,7 @@ pub fn truncateFile(b: *Build, dest_path: []const u8) (Io.Dir.CreateDirError || /// References a file or directory relative to the source root. pub fn path(b: *Build, sub_path: []const u8) LazyPath { if (fs.path.isAbsolute(sub_path)) { - std.debug.panic("sub_path is expected to be relative to the build root, but was this absolute path: '{s}'. It is best avoid absolute paths, but if you must, it is supported by LazyPath.cwd_relative", .{ + panic("sub_path is expected to be relative to the build root, but was this absolute path: '{s}'. It is best avoid absolute paths, but if you must, it is supported by LazyPath.cwd_relative", .{ sub_path, }); } @@ -1560,117 +1575,164 @@ pub fn fmt(b: *Build, comptime format: []const u8, args: anytype) []u8 { return std.fmt.allocPrint(b.allocator, format, args) catch @panic("OOM"); } -fn supportedWindowsProgramExtension(ext: []const u8) bool { - inline for (@typeInfo(std.process.WindowsExtension).@"enum".fields) |field| { - if (std.ascii.eqlIgnoreCase(ext, "." ++ field.name)) return true; - } - return false; -} - -fn tryFindProgram(b: *Build, full_path: []const u8) ?[]const u8 { - const io = b.graph.io; - const arena = b.allocator; - - if (b.build_root.handle.realPathFileAlloc(io, full_path, arena)) |p| { - return p; - } else |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - else => {}, - } - - if (builtin.os.tag == .windows) { - if (b.graph.environ_map.get("PATHEXT")) |PATHEXT| { - var it = mem.tokenizeScalar(u8, PATHEXT, fs.path.delimiter); - - while (it.next()) |ext| { - if (!supportedWindowsProgramExtension(ext)) continue; - - return b.build_root.handle.realPathFileAlloc( - io, - b.fmt("{s}{s}", .{ full_path, ext }), - arena, - ) catch |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - else => continue, - }; - } - } - } - - return null; -} - -pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) LazyPath { - _ = b; - _ = names; - _ = paths; - @panic("TODO rework findProgram to be based on LazyPath"); +/// Creates an anonymous `Step` that searches for an executable on the host that +/// has more than one possible name. +/// +/// Names are searched in order, observing search prefixes first and then PATH +/// environment variable. +/// +/// Returns the `LazyPath` of the found executable. The search only takes place +/// if the `LazyPath` will be used by a depending `Step`. +pub fn findProgram(b: *Build, names: []const []const u8) LazyPath { + const graph = b.graph; + const wc = &graph.wip_configuration; + const string_list = wc.addStringList(names) catch @panic("OOM"); + _ = string_list; + @panic("TODO"); } +/// Deprecated; use `runFallible`. pub fn runAllowFail( b: *Build, argv: []const []const u8, - out_code: *u8, - stderr_behavior: std.process.SpawnOptions.StdIo, -) RunError![]u8 { + exit_code: *u8, + stderr_behavior: process.SpawnOptions.StdIo, +) anyerror![]u8 { + if (!process.can_spawn) return error.ExecNotSupported; + switch (runFallible(b, argv, .{ + .stderr_behavior = stderr_behavior, + })) { + .success => |stdout| return stdout, + .spawn_failed => |err| return err, + .bad_exit_code => |code| { + exit_code.* = code; + return error.ExitCodeFailure; + }, + .crashed => { + exit_code.* = 255; + return error.ProcessTerminated; + }, + } +} + +pub const RunOptions = struct { + stderr_behavior: process.SpawnOptions.StdIo = .inherit, + /// Fail the configuration if stdout is larger than this. + stdout_limit: Io.Limit = .limited(1_000_000), + /// Set to change the current working directory when spawning the child + /// process. + cwd: process.Child.Cwd = .inherit, + /// Replaces the child environment when provided. The PATH value from here + /// is not used to resolve `argv[0]`; that resolution always uses parent + /// environment. + environ_map: ?*const process.Environ.Map = null, + expand_arg0: process.ArgExpansion = .no_expand, +}; + +pub const RunResult = union(enum) { + /// Thild process exited with code 0, writing this stdout. + success: []u8, + /// The child process could not be created. + spawn_failed: process.SpawnError, + /// The child process indicated failure. + bad_exit_code: u8, + /// The child process terminated abnormally. + crashed, +}; + +/// Executes the provided command immediately, allowing failure. +/// +/// If the program exits successfully, stdout is returned. Otherwise, returns +/// an indication of failure. +/// +/// See also: +/// * `run`. +pub fn runFallible(b: *Build, argv: []const []const u8, options: RunOptions) RunResult { assert(argv.len != 0); - if (!process.can_spawn) - return error.ExecNotSupported; - const graph = b.graph; const io = graph.io; const arena = graph.arena; - const max_output_size = 400 * 1024; + const print_opts: std.zig.AllocPrintCmdOptions = .{ + .cwd = options.cwd, + .child_env = options.environ_map, + .parent_env = &graph.environ_map, + }; + if (graph.verbose) { - const text = std.zig.allocPrintCmd(arena, .inherit, null, argv); + const text = std.zig.allocPrintCmd(arena, argv, print_opts) catch @panic("OOM"); std.log.scoped(.verbose).info("{s}", .{text}); } - var child = try std.process.spawn(io, .{ + var child = process.spawn(io, .{ .argv = argv, - .environ_map = &graph.environ_map, .stdin = .ignore, .stdout = .pipe, - .stderr = stderr_behavior, - }); + .stderr = options.stderr_behavior, + .cwd = options.cwd, + .environ_map = &graph.environ_map, + .expand_arg0 = options.expand_arg0, + }) catch |err| return .{ .spawn_failed = err }; var stdout_reader = child.stdout.?.readerStreaming(io, &.{}); - const stdout = stdout_reader.interface.allocRemaining(arena, .limited(max_output_size)) catch { - return error.ReadFailure; + const stdout = stdout_reader.interface.allocRemaining(arena, options.stdout_limit) catch |err| switch (err) { + error.ReadFailed => panic("failed to read from child: {t}", .{stdout_reader.err.?}), + else => |e| panic("failed to read from child: {t}", .{e}), }; - errdefer arena.free(stdout); - const term = try child.wait(io); - switch (term) { - .exited => |code| { - if (code != 0) { - out_code.* = @as(u8, @truncate(code)); - return error.ExitCodeFailure; - } - return stdout; - }, - .signal, .stopped => |sig| { - out_code.* = @as(u8, @truncate(@intFromEnum(sig))); - return error.ProcessTerminated; - }, - .unknown => |code| { - out_code.* = @as(u8, @truncate(code)); - return error.ProcessTerminated; + const term = child.wait(io) catch @panic("unexpected"); + + return switch (term) { + .exited => |code| switch (code) { + 0 => .{ .success = stdout }, + else => .{ .bad_exit_code = code }, }, - } + .signal, .stopped, .unknown => .crashed, + }; } -/// This is a helper function to be called from build.zig scripts, *not* from -/// inside step make() functions. If any errors occur, it fails the build with -/// a helpful message. +/// Executes the provided command immediately. +/// +/// If the program exits successfully, stdout is returned. Otherwise, fails the +/// build with a helpful message. +/// +/// See also: +/// * `runFallible`. pub fn run(b: *Build, argv: []const []const u8) []u8 { - var code: u8 = undefined; - return b.runAllowFail(argv, &code, .inherit) catch |err| process.fatal( - "the following command failed with {t}:\n{s}", - .{ err, Step.allocPrintCmd(b.allocator, .inherit, null, argv) catch @panic("OOM") }, - ); + const graph = b.graph; + const arena = graph.arena; + switch (b.runFallible(argv, .{ + .stderr_behavior = .inherit, + })) { + .success => |stdout| return stdout, + .spawn_failed => |err| process.fatal("the following command failed with {t}:\n{s}", .{ + err, std.zig.allocPrintCmd(arena, argv, .{}) catch @panic("OOM"), + }), + .bad_exit_code => |code| process.fatal("the following command exited with code {d}:\n{s}", .{ + code, std.zig.allocPrintCmd(arena, argv, .{}) catch @panic("OOM"), + }), + .crashed => process.fatal("the following command crashed:\n{s}", .{ + std.zig.allocPrintCmd(arena, argv, .{}) catch @panic("OOM"), + }), + } +} + +/// Adds additional paths, equivalent to the `--search-prefix` arguments +/// provided by the user. Paths added with this function have lower precedence +/// than the ones specified by the user on the command line. +/// +/// It is generally best practice to avoid calling this function, instead +/// relying on the user to provide these paths via the standard build system +/// interface. However, when integrating with other build systems, the user may +/// have already provided the information to the other build system, and thus +/// it is desirable to use that same information without requiring the user to +/// provide it again. +pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { + _ = b; + _ = search_prefix; + @panic("TODO"); + //b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); } pub const Dependency = struct { @@ -1727,8 +1789,8 @@ fn findPkgHashOrFatal(b: *Build, name: []const u8) []const u8 { if (mem.eql(u8, dep[0], name)) return dep[1]; } std.log.info("all dependencies used by build.zig must be declared in corresponding build.zig.zon", .{}); - if (b.pkg_hash.len == 0) std.debug.panic("no dependency named {s}", .{name}); - std.debug.panic("no dependency named {s} in {s} ({s})", .{ name, b.dep_prefix, b.pkg_hash }); + if (b.pkg_hash.len == 0) panic("no dependency named {s}", .{name}); + panic("no dependency named {s} in {s} ({s})", .{ name, b.dep_prefix, b.pkg_hash }); } inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, comptime dep_name: []const u8) []const u8 { @@ -1741,7 +1803,7 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c if (@hasDecl(pkg, "build_zig") and pkg.build_zig == asking_build_zig) break .{ pkg_hash, pkg.deps }; } else .{ "", deps.root_deps }; if (!std.mem.eql(u8, b_pkg_hash, b.pkg_hash)) { - std.debug.panic("'{}' is not the struct that corresponds to '{s}'", .{ + panic("'{}' is not the struct that corresponds to '{s}'", .{ asking_build_zig, b.pathFromRoot("build.zig"), }); } @@ -1750,7 +1812,7 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c }; const full_path = b.pathFromRoot("build.zig.zon"); - std.debug.panic("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file", .{ dep_name, full_path }); + panic("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file", .{ dep_name, full_path }); } fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void { @@ -1799,7 +1861,7 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency { if (mem.eql(u8, decl.name, pkg_hash)) { const pkg = @field(deps.packages, decl.name); if (@hasDecl(pkg, "available")) { - std.debug.panic("dependency '{s}{s}' is marked as lazy in build.zig.zon which means it must use the lazyDependency function instead", .{ b.dep_prefix, name }); + panic("dependency '{s}{s}' is marked as lazy in build.zig.zon which means it must use the lazyDependency function instead", .{ b.dep_prefix, name }); } return dependencyInner(b, name, pkg.build_root, if (@hasDecl(pkg, "build_zig")) pkg.build_zig else null, pkg_hash, pkg.deps, args); } @@ -1866,7 +1928,7 @@ pub fn dependencyFromBuildZig( } const full_path = b.pathFromRoot("build.zig.zon"); - std.debug.panic("'{}' is not a build.zig struct of a dependency in '{s}'", .{ build_zig, full_path }); + panic("'{}' is not a build.zig struct of a dependency in '{s}'", .{ build_zig, full_path }); } fn userValuesAreSame(lhs: UserValue, rhs: UserValue) bool { @@ -1960,14 +2022,23 @@ fn dependencyInner( pkg_deps: AvailableDeps, args: anytype, ) *Dependency { - const user_input_options = userInputOptionsFromArgs(b.allocator, args); + const io = b.graph.io; + const arena = b.graph.arena; + const user_input_options = userInputOptionsFromArgs(arena, args); if (b.graph.dependency_cache.getContext(.{ .build_root_string = build_root_string, .user_input_options = user_input_options, - }, .{ .allocator = b.graph.arena })) |dep| - return dep; + }, .{ .allocator = arena })) |dep| return dep; - const sub_builder = b.createChild(name, pkg_hash, pkg_deps, user_input_options) catch + const dep_root: Cache.Path = .{ + .root_dir = .{ + .path = build_root_string, + .handle = Io.Dir.cwd().openDir(io, build_root_string, .{}) catch |err| + process.fatal("unable to open {s}: {t}", .{ build_root_string, err }), + }, + }; + + const sub_builder = b.createChild(name, dep_root, pkg_hash, pkg_deps, user_input_options) catch @panic("unhandled error"); if (build_zig) |bz| { sub_builder.runBuild(bz) catch @panic("unhandled error"); @@ -1977,13 +2048,13 @@ fn dependencyInner( } } - const dep = b.allocator.create(Dependency) catch @panic("OOM"); + const dep = arena.create(Dependency) catch @panic("OOM"); dep.* = .{ .builder = sub_builder }; b.graph.dependency_cache.putContext(b.graph.arena, .{ .build_root_string = build_root_string, .user_input_options = user_input_options, - }, dep, .{ .allocator = b.graph.arena }) catch @panic("OOM"); + }, dep, .{ .allocator = arena }) catch @panic("OOM"); return dep; } @@ -2046,14 +2117,7 @@ pub const LazyPath = union(enum) { sub_path: []const u8 = "", }, - /// An absolute path or a path relative to the current working directory of - /// the build runner process. - /// - /// This is uncommon but used for system environment paths such as `--zig-lib-dir` which - /// ignore the file system path of build.zig and instead are relative to the directory from - /// which `zig build` was invoked. - /// - /// Use of this tag indicates a dependency on the host system. + /// Deprecated; call `Graph.cwdRelativePath` instead. cwd_relative: []const u8, dependency: struct { @@ -2061,6 +2125,19 @@ pub const LazyPath = union(enum) { sub_path: []const u8, }, + relative: struct { + base: Configuration.Path.Base, + sub_path: Configuration.String = .empty, + }, + + /// Path to the Zig executable being used to execute "zig build". + pub const zig_exe: LazyPath = .{ .relative = .{ .base = .zig_exe } }; + /// Path to the "lib/" directory from the Zig installation being used to + /// execute "zig build". + pub const zig_lib: LazyPath = .{ .relative = .{ .base = .zig_lib } }; + /// Path to the project's local cache directory (usually called ".zig-cache"). + pub const cache_root: LazyPath = .{ .relative = .{ .base = .local_cache } }; + /// Returns a lazy path referring to the directory containing this path. /// /// The dirname is not allowed to escape the logical root for underlying path. @@ -2147,21 +2224,33 @@ pub const LazyPath = union(enum) { }; } - /// Returns a string that can be shown to represent the file source. - /// Either returns the path, `"generated"`, or `"dependency"`. - pub fn getDisplayName(lazy_path: LazyPath) []const u8 { - return switch (lazy_path) { - .src_path => |sp| sp.sub_path, - .cwd_relative => |p| p, - .generated => "generated", - .dependency => "dependency", - }; + pub const Format = struct { + graph: *const Graph, + lazy_path: *const LazyPath, + + pub fn format(f: Format, w: *Io.Writer) Io.Writer.Error!void { + switch (f.lazy_path.*) { + .src_path => |sp| try w.writeAll(sp.sub_path), + .cwd_relative => |p| try w.writeAll(p), + .generated => try w.writeAll("generated"), + .dependency => try w.writeAll("dependency"), + .relative => |r| { + const wc = &f.graph.wip_configuration; + try w.writeAll(@tagName(r.base)); + try w.writeAll(wc.stringSlice(r.sub_path)); + }, + } + } + }; + + pub fn fmt(lp: *const LazyPath, graph: *const Graph) Format { + return .{ .graph = graph, .lazy_path = lp }; } /// Adds dependencies this file source implies to the given step. pub fn addStepDependencies(lazy_path: LazyPath, other_step: *Step) void { switch (lazy_path) { - .src_path, .cwd_relative, .dependency => {}, + .src_path, .cwd_relative, .relative, .dependency => {}, .generated => |gen| { const graph = other_step.owner.graph; const generated_owner_step = graph.generated_files.items[@intFromEnum(gen.index)]; @@ -2190,6 +2279,7 @@ pub const LazyPath = union(enum) { return switch (lazy_path) { .src_path => |sp| .{ .src_path = .{ .owner = sp.owner, .sub_path = sp.owner.dupePath(sp.sub_path) } }, .cwd_relative => |p| .{ .cwd_relative = graph.dupePath(p) }, + .relative => |r| .{ .relative = r }, .generated => |gen| .{ .generated = .{ .index = gen.index, .up = gen.up, diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index ca1c82074aec6ffd2ed75b819f12799922690287..2e81421313f2300092d7deaae15601a40807849f 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -402,6 +402,12 @@ pub const Wip = struct { defer wip.next_generated_file_index += 1; return @enumFromInt(wip.next_generated_file_index); } + + /// Returned slice expires upon next append to the configuration. + pub fn stringSlice(wip: *const Wip, s: String) [:0]const u8 { + const start_slice = wip.string_bytes.items[@intFromEnum(s)..]; + return start_slice[0..std.mem.indexOfScalar(u8, start_slice, 0).? :0]; + } }; pub const SystemIntegration = extern struct { @@ -445,6 +451,7 @@ pub const Step = extern struct { compile: Compile, config_header: ConfigHeader, fail: Fail, + find_program: FindProgram, fmt: Fmt, install_artifact: InstallArtifact, install_dir: InstallDir, @@ -479,6 +486,7 @@ pub const Step = extern struct { compile, config_header, fail, + find_program, fmt, install_artifact, install_dir, @@ -1037,6 +1045,17 @@ pub const Step = extern struct { }; }; + pub const FindProgram = struct { + flags: @This().Flags, + names: StringList, + generated_file: GeneratedFileIndex, + + pub const Flags = packed struct(u32) { + tag: Tag = .find_program, + _: u27 = 0, + }; + }; + pub const InstallDir = struct { flags: @This().Flags, source_dir: LazyPath.Index, @@ -1516,6 +1535,8 @@ pub const Path = extern struct { local_cache, global_cache, build_root, + zig_exe, + zig_lib, }; pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path { diff --git a/lib/std/Build/Module.zig b/lib/std/Build/Module.zig index 1c271023efad2c4458f5185d88620a692b7581ae..cdc0c134a56f9d8810946563fbaf4e41a31673bc 100644 --- a/lib/std/Build/Module.zig +++ b/lib/std/Build/Module.zig @@ -118,10 +118,10 @@ pub const CSourceFile = struct { /// By default, determines language of each file individually based on its file extension language: ?CSourceLanguage = null, - pub fn dupe(file: CSourceFile, b: *std.Build) CSourceFile { + pub fn dupe(file: CSourceFile, graph: *const std.Build.Graph) CSourceFile { return .{ - .file = file.file.dupe(b), - .flags = b.dupeStrings(file.flags), + .file = file.file.dupe(graph), + .flags = graph.dupeStrings(file.flags), .language = file.language, }; } @@ -146,10 +146,12 @@ pub const RcSourceFile = struct { include_paths: []const LazyPath = &.{}, pub fn dupe(file: RcSourceFile, b: *std.Build) RcSourceFile { - const include_paths = b.allocator.alloc(LazyPath, file.include_paths.len) catch @panic("OOM"); - for (include_paths, file.include_paths) |*dest, lazy_path| dest.* = lazy_path.dupe(b); + const graph = b.owner.graph; + const arena = graph.arena; + const include_paths = arena.alloc(LazyPath, file.include_paths.len) catch @panic("OOM"); + for (include_paths, file.include_paths) |*dest, lazy_path| dest.* = lazy_path.dupe(graph); return .{ - .file = file.file.dupe(b), + .file = file.file.dupe(graph), .flags = b.dupeStrings(file.flags), .include_paths = include_paths, }; @@ -290,15 +292,18 @@ pub fn init( } pub fn create(owner: *std.Build, options: CreateOptions) *Module { - const m = owner.allocator.create(Module) catch @panic("OOM"); + const graph = owner.graph; + const arena = graph.arena; + const m = arena.create(Module) catch @panic("OOM"); m.init(owner, .{ .options = options }); return m; } /// Adds an existing module to be used with `@import`. pub fn addImport(m: *Module, name: []const u8, module: *Module) void { - const b = m.owner; - m.import_table.put(b.allocator, b.dupe(name), module) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.import_table.put(arena, graph.dupeString(name), module) catch @panic("OOM"); } /// Creates a new module and adds it to be used with `@import`. @@ -338,7 +343,8 @@ pub fn linkSystemLibrary( name: []const u8, options: LinkSystemLibraryOptions, ) void { - const b = m.owner; + const graph = m.owner.graph; + const arena = graph.arena; const target = m.requireKnownTarget(); if (std.zig.target.isLibCLibName(target, name)) { @@ -350,9 +356,9 @@ pub fn linkSystemLibrary( return; } - m.link_objects.append(b.allocator, .{ + m.link_objects.append(arena, .{ .system_lib = .{ - .name = b.dupe(name), + .name = graph.dupeString(name), .needed = options.needed, .weak = options.weak, .use_pkg_config = options.use_pkg_config, @@ -363,8 +369,9 @@ pub fn linkSystemLibrary( } pub fn linkFramework(m: *Module, name: []const u8, options: LinkFrameworkOptions) void { - const b = m.owner; - m.frameworks.put(b.allocator, b.dupe(name), options) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.frameworks.put(arena, graph.dupeString(name), options) catch @panic("OOM"); } pub const AddCSourceFilesOptions = struct { @@ -380,7 +387,8 @@ pub const AddCSourceFilesOptions = struct { /// Handy when you have many non-Zig source files and want them all to have the same flags. pub fn addCSourceFiles(m: *Module, options: AddCSourceFilesOptions) void { const b = m.owner; - const allocator = b.allocator; + const graph = m.owner.graph; + const arena = graph.arena; for (options.files) |path| { if (std.fs.path.isAbsolute(path)) { @@ -391,48 +399,50 @@ pub fn addCSourceFiles(m: *Module, options: AddCSourceFilesOptions) void { } } - const c_source_files = allocator.create(CSourceFiles) catch @panic("OOM"); + const c_source_files = arena.create(CSourceFiles) catch @panic("OOM"); c_source_files.* = .{ .root = options.root orelse b.path(""), .files = b.dupeStrings(options.files), .flags = b.dupeStrings(options.flags), .language = options.language, }; - m.link_objects.append(allocator, .{ .c_source_files = c_source_files }) catch @panic("OOM"); + m.link_objects.append(arena, .{ .c_source_files = c_source_files }) catch @panic("OOM"); } pub fn addCSourceFile(m: *Module, source: CSourceFile) void { - const b = m.owner; - const allocator = b.allocator; - const c_source_file = allocator.create(CSourceFile) catch @panic("OOM"); - c_source_file.* = source.dupe(b); - m.link_objects.append(allocator, .{ .c_source_file = c_source_file }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + const c_source_file = arena.create(CSourceFile) catch @panic("OOM"); + c_source_file.* = source.dupe(graph); + m.link_objects.append(arena, .{ .c_source_file = c_source_file }) catch @panic("OOM"); } /// Resource files must have the extension `.rc`. /// Can be called regardless of target. The .rc file will be ignored /// if the target object format does not support embedded resources. pub fn addWin32ResourceFile(m: *Module, source: RcSourceFile) void { - const b = m.owner; - const allocator = b.allocator; + const graph = m.owner.graph; + const arena = graph.arena; const target = m.requireKnownTarget(); // Only the PE/COFF format has a Resource Table, so for any other target // the resource file is ignored. if (target.ofmt != .coff) return; - const rc_source_file = allocator.create(RcSourceFile) catch @panic("OOM"); - rc_source_file.* = source.dupe(b); - m.link_objects.append(allocator, .{ .win32_resource_file = rc_source_file }) catch @panic("OOM"); + const rc_source_file = arena.create(RcSourceFile) catch @panic("OOM"); + rc_source_file.* = source.dupe(graph); + m.link_objects.append(arena, .{ .win32_resource_file = rc_source_file }) catch @panic("OOM"); } pub fn addAssemblyFile(m: *Module, source: LazyPath) void { - const b = m.owner; - m.link_objects.append(b.allocator, .{ .assembly_file = source.dupe(b) }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.link_objects.append(arena, .{ .assembly_file = source.dupe(graph) }) catch @panic("OOM"); } pub fn addObjectFile(m: *Module, object: LazyPath) void { - const b = m.owner; - m.link_objects.append(b.allocator, .{ .static_path = object.dupe(b) }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.link_objects.append(arena, .{ .static_path = object.dupe(graph) }) catch @panic("OOM"); } pub fn addObject(m: *Module, object: *Step.Compile) void { @@ -446,55 +456,63 @@ pub fn linkLibrary(m: *Module, library: *Step.Compile) void { } pub fn addAfterIncludePath(m: *Module, lazy_path: LazyPath) void { - const b = m.owner; - m.include_dirs.append(b.allocator, .{ .path_after = lazy_path.dupe(b) }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.include_dirs.append(arena, .{ .path_after = lazy_path.dupe(graph) }) catch @panic("OOM"); } pub fn addSystemIncludePath(m: *Module, lazy_path: LazyPath) void { - const b = m.owner; - m.include_dirs.append(b.allocator, .{ .path_system = lazy_path.dupe(b) }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.include_dirs.append(arena, .{ .path_system = lazy_path.dupe(graph) }) catch @panic("OOM"); } pub fn addIncludePath(m: *Module, lazy_path: LazyPath) void { - const b = m.owner; - m.include_dirs.append(b.allocator, .{ .path = lazy_path.dupe(b) }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.include_dirs.append(arena, .{ .path = lazy_path.dupe(graph) }) catch @panic("OOM"); } pub fn addConfigHeader(m: *Module, config_header: *Step.ConfigHeader) void { - const allocator = m.owner.allocator; - m.include_dirs.append(allocator, .{ .config_header_step = config_header }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.include_dirs.append(arena, .{ .config_header_step = config_header }) catch @panic("OOM"); } pub fn addSystemFrameworkPath(m: *Module, directory_path: LazyPath) void { - const b = m.owner; - m.include_dirs.append(b.allocator, .{ .framework_path_system = directory_path.dupe(b) }) catch - @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.include_dirs.append(arena, .{ .framework_path_system = directory_path.dupe(graph) }) catch @panic("OOM"); } pub fn addFrameworkPath(m: *Module, directory_path: LazyPath) void { - const b = m.owner; - m.include_dirs.append(b.allocator, .{ .framework_path = directory_path.dupe(b) }) catch - @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.include_dirs.append(arena, .{ .framework_path = directory_path.dupe(graph) }) catch @panic("OOM"); } pub fn addEmbedPath(m: *Module, lazy_path: LazyPath) void { - const b = m.owner; - m.include_dirs.append(b.allocator, .{ .embed_path = lazy_path.dupe(b) }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.include_dirs.append(arena, .{ .embed_path = lazy_path.dupe(graph) }) catch @panic("OOM"); } pub fn addLibraryPath(m: *Module, directory_path: LazyPath) void { - const b = m.owner; - m.lib_paths.append(b.allocator, directory_path.dupe(b)) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.lib_paths.append(arena, directory_path.dupe(graph)) catch @panic("OOM"); } pub fn addRPath(m: *Module, directory_path: LazyPath) void { - const b = m.owner; - m.rpaths.append(b.allocator, .{ .lazy_path = directory_path.dupe(b) }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.rpaths.append(arena, .{ .lazy_path = directory_path.dupe(graph) }) catch @panic("OOM"); } pub fn addRPathSpecial(m: *Module, bytes: []const u8) void { - const b = m.owner; - m.rpaths.append(b.allocator, .{ .special = b.dupe(bytes) }) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.rpaths.append(arena, .{ .special = graph.dupeString(bytes) }) catch @panic("OOM"); } /// Equvialent to the following C code, applied to all C source files owned by @@ -505,19 +523,23 @@ pub fn addRPathSpecial(m: *Module, bytes: []const u8) void { /// `name` and `value` need not live longer than the function call. pub fn addCMacro(m: *Module, name: []const u8, value: []const u8) void { const b = m.owner; - m.c_macros.append(b.allocator, b.fmt("-D{s}={s}", .{ name, value })) catch @panic("OOM"); + const graph = m.owner.graph; + const arena = graph.arena; + m.c_macros.append(arena, b.fmt("-D{s}={s}", .{ name, value })) catch @panic("OOM"); } fn linkLibraryOrObject(m: *Module, other: *Step.Compile) void { - const allocator = m.owner.allocator; + const graph = m.owner.graph; + const arena = graph.arena; + _ = other.getEmittedBin(); // Indicate there is a dependency on the outputted binary. if (other.rootModuleTarget().os.tag == .windows and other.isDynamicLibrary()) { _ = other.getEmittedImplib(); // Indicate dependency on the outputted implib. } - m.link_objects.append(allocator, .{ .other_step = other }) catch @panic("OOM"); - m.include_dirs.append(allocator, .{ .other_step = other }) catch @panic("OOM"); + m.link_objects.append(arena, .{ .other_step = other }) catch @panic("OOM"); + m.include_dirs.append(arena, .{ .other_step = other }) catch @panic("OOM"); } fn requireKnownTarget(m: *Module) *const std.Target { diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index a1f613e1cb4471f39dfd7311db6d8b14cf52d433..8e76d39fa155de6d614c41b55ebd016e41063bc0 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -83,7 +83,7 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { } const name = if (options.style.getPath()) |s| - owner.fmt("configure {t} header {s} to {s}", .{ options.style, s.getDisplayName(), include_path }) + owner.fmt("configure {t} header {f} to {s}", .{ options.style, s.fmt(graph), include_path }) else owner.fmt("configure {t} header to {s}", .{ options.style, include_path }); diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index f1a904cba5d150a5c21b84bd4a69c1eb52721d63..02a436e3a752b965ef4fa6eb41ae64a809a68d87 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -47,7 +47,7 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir { install_dir.* = .{ .step = Step.init(.{ .tag = base_tag, - .name = owner.fmt("install {s}/", .{options.source_dir.getDisplayName()}), + .name = owner.fmt("install {f}/", .{options.source_dir.fmt(graph)}), .owner = owner, }), .options = options.dupe(graph), diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index f143e43e4e0b30c1c088309adb43d7f1ef127c1c..7d1d11cf5ad4b1ca8d33cbdf6fece43075e62f6d 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -26,7 +26,7 @@ pub fn create( install_file.* = .{ .step = Step.init(.{ .tag = base_tag, - .name = owner.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), + .name = owner.fmt("install {f} to {s}", .{ source.fmt(graph), dest_rel_path }), .owner = owner, }), .source = source.dupe(graph), diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index d7de00c0ef28400009daf1f521fec43ba90403dc..94e6a405e84ee6d1ff26ba2a4f330c7f40f01ad3 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -116,12 +116,12 @@ pub fn create( objcopy.* = ObjCopy{ .step = Step.init(.{ .tag = base_tag, - .name = owner.fmt("objcopy {s}", .{input_file.getDisplayName()}), + .name = owner.fmt("objcopy {f}", .{input_file.fmt(graph)}), .owner = owner, .makeFn = make, }), .input_file = input_file, - .basename = options.basename orelse input_file.getDisplayName(), + .basename = options.basename orelse std.fmt.allocPrint("{f}", .{input_file.fmt(graph)}) catch @panic("OOM"), .output_file = graph.addGeneratedFile(&objcopy.step), .output_file_debug = if (options.strip != .none and options.extract_to_separate_file) .init(graph.addGeneratedFile(&objcopy.step)) diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 700978a94d51c0e4071c24156c9869d347e6fac4..18d5db1c1d9853d8aee522e24440fbcd2dd0fa22 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1165,15 +1165,13 @@ pub const ClangCliParam = struct { } }; -pub fn allocPrintCmd( - gpa: Allocator, - cwd: std.process.Child.Cwd, - opt_env: ?struct { - child: *const std.process.Environ.Map, - parent: *const std.process.Environ.Map, - }, - argv: []const []const u8, -) Allocator.Error![]u8 { +pub const AllocPrintCmdOptions = struct { + cwd: std.process.Child.Cwd = .inherit, + parent_env: ?*const std.process.Environ.Map = null, + child_env: ?*const std.process.Environ.Map = null, +}; + +pub fn allocPrintCmd(gpa: Allocator, argv: []const []const u8, options: AllocPrintCmdOptions) Allocator.Error![]u8 { const shell = struct { fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { for (string) |c| { @@ -1212,18 +1210,17 @@ pub fn allocPrintCmd( var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); const writer = &aw.writer; - switch (cwd) { + switch (options.cwd) { .inherit => {}, .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, .dir => @panic("TODO"), } - if (opt_env) |env| { - var it = env.child.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; - if (env.parent.get(key)) |process_value| { - if (std.mem.eql(u8, value, process_value)) continue; + if (options.child_env) |child_env| { + for (child_env.keys(), child_env.values()) |key, value| { + if (options.parent_env) |parent_env| { + if (parent_env.get(key)) |process_value| { + if (std.mem.eql(u8, value, process_value)) continue; + } } writer.print("{s}=", .{key}) catch return error.OutOfMemory; shell.escape(writer, value, false) catch return error.OutOfMemory; diff --git a/src/main.zig b/src/main.zig index 0f048adaffefe7f6ff01cbfbc758f58cb918f893..9f4c8507459f7ba2592422689714ea84ef748cb9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4987,7 +4987,7 @@ fn cmdBuild( const argv_index_zig_lib_dir = make_argv.items.len - 1; make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; - const argv_index_build_file = make_argv.items.len - 1; + const make_argv_index_build_root = make_argv.items.len - 1; make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--local-cache", undefined }; const argv_index_cache_dir = make_argv.items.len - 1; @@ -5001,6 +5001,9 @@ fn cmdBuild( make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--seed", default_seed }; const argv_index_seed = make_argv.items.len - 1; + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; + const conf_argv_index_build_root = configure_argv.items.len - 1; + var color: Color = .auto; var n_jobs: ?u32 = null; @@ -5065,6 +5068,10 @@ fn cmdBuild( i += 1; override_global_cache_dir = args[i]; continue; + } else if (mem.eql(u8, arg, "--verbose")) { + // Intentionally is added both to make and configure but + // does not go into the cache hash. + configure_argv.appendAssumeCapacity(arg); } else if (mem.eql(u8, arg, "-freference-trace")) { reference_trace = 256; } else if (mem.eql(u8, arg, "--fetch")) { @@ -5283,10 +5290,12 @@ fn cmdBuild( defer _ = make_runner_task.cancel(io) catch {}; make_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; - make_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; + make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path; make_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; make_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; + configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; + // Dummy http client that is not actually used when fetch_command is unsupported. // Prevents bootstrap from depending on a bunch of unnecessary stuff. var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { -- 2.54.0 From 3d785897658fd8b61660649b24d8943bda545982 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 4 May 2026 15:37:01 -0700 Subject: [PATCH 080/179] std.Build: port Fmt step to new system and integrate properly with LazyPath --- BRANCH_TODO | 16 +++++- build.zig | 4 +- lib/compiler/Maker/Step.zig | 2 +- lib/compiler/Maker/Step/Fmt.zig | 57 +++++++++++++++++++ lib/compiler/Maker/Step/InstallDir.zig | 2 +- lib/compiler/Maker/Step/Options.zig | 4 +- lib/std/Build.zig | 50 ++++++++++++----- lib/std/Build/Configuration.zig | 7 ++- lib/std/Build/Step/Fmt.zig | 78 ++++++++------------------ 9 files changed, 143 insertions(+), 77 deletions(-) create mode 100644 lib/compiler/Maker/Step/Fmt.zig diff --git a/BRANCH_TODO b/BRANCH_TODO index c51f87fd6916014f8f61687a37e4b21109fb5805..8f95fb7ec6a5b6cd97a3478d534188e8d8216944 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -27,10 +27,13 @@ - but artifact install steps also add paths for dyn libs on windows * no more "artifact arg" to run step. if you want to run the post-install binary, get the lazy path from the install step. - +* build system fmt step with check=false does not acquire a write lock on source files #35204 +* fmt step: import zig fmt code directly rather than child proc ## Release Notes +### Run Step: Passthru Args + In the Run step, passthru args are all together now, not observable in configure phase whether run args are provided. @@ -51,3 +54,14 @@ those arguments. In exchange, it means that when changing those arguments, build scripts no longer must be rebuilt from source. closes #31397 + +### Fmt Step: Options + +`paths` and `exclude_paths` are now LazyPath lists. There is a convenience method to create them: `b.pathList`. + +```diff +- const fmt_include_paths = &.{ "lib", "src", "test", "tools", "build.zig", "build.zig.zon" }; +- const fmt_exclude_paths = &.{ "test/cases", "test/behavior/zon" }; ++ const fmt_include_paths = b.pathList(&.{ "lib", "src", "test", "tools", "build.zig", "build.zig.zon" }); ++ const fmt_exclude_paths = b.pathList(&.{ "test/cases", "test/behavior/zon" }); +``` diff --git a/build.zig b/build.zig index 5400169113ebc1ffac82a3d54df215ea1238dc03..f539f8e0d9fa0b75b6e53e1664e992bb448784bc 100644 --- a/build.zig +++ b/build.zig @@ -427,8 +427,8 @@ pub fn build(b: *std.Build) !void { else null; - const fmt_include_paths = &.{ "lib", "src", "test", "tools", "build.zig", "build.zig.zon" }; - const fmt_exclude_paths = &.{ "test/cases", "test/behavior/zon" }; + const fmt_include_paths = b.pathList(&.{ "lib", "src", "test", "tools", "build.zig", "build.zig.zon" }); + const fmt_exclude_paths = b.pathList(&.{ "test/cases", "test/behavior/zon" }); const do_fmt = b.addFmt(.{ .paths = fmt_include_paths, .exclude_paths = fmt_exclude_paths, diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index d67ee25ebc839cfb89c7fd30134a004ba301882e..261310f726d9f6f0776c7e2b5b7def4689ae1d47 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -311,7 +311,7 @@ pub fn captureChildProcess( ) !std.process.RunResult { const gpa = maker.gpa; const graph = maker.graph; - const arena = graph.arena; + const arena = graph.arena; // TODO stop leaking into process arena const io = graph.io; // If an error occurs, it's happened in this command: diff --git a/lib/compiler/Maker/Step/Fmt.zig b/lib/compiler/Maker/Step/Fmt.zig new file mode 100644 index 0000000000000000000000000000000000000000..292a172ac1a4bf4c38da868a3f7467fbb40e6242 --- /dev/null +++ b/lib/compiler/Maker/Step/Fmt.zig @@ -0,0 +1,57 @@ +const Fmt = @This(); + +const std = @import("std"); +const Configuration = std.Build.Configuration; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +/// Persisted to reuse memory on subsequent calls to `make`. +argv: std.ArrayList([]const u8) = .empty, + +pub fn make( + fmt: *Fmt, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + const graph = maker.graph; + const step = maker.stepByIndex(step_index); + const gpa = maker.gpa; + const arena = graph.arena; // TODO don't leak into the process arena + const argv = &fmt.argv; + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_fmt = conf_step.extended.get(conf.extra).fmt; + const paths = conf_fmt.paths.slice; + const exclude_paths = conf_fmt.paths.exclude_paths; + + argv.clearRetainingCapacity(); + try argv.ensureUnusedCapacity(gpa, 2 + 1 + paths.len + 2 * exclude_paths.len); + + argv.appendAssumeCapacity(graph.zig_exe); + argv.appendAssumeCapacity("fmt"); + + if (fmt.check) + argv.appendAssumeCapacity("--check"); + + for (fmt.paths) |lp| + argv.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, step_index)); + + for (fmt.exclude_paths) |lp| { + argv.appendAssumeCapacity("--exclude"); + argv.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, step_index)); + } + + const run_result = try step.captureChildProcess(maker, progress_node, argv.items); + if (fmt.check) switch (run_result.term) { + .exited => |code| if (code != 0 and run_result.stdout.len != 0) { + var it = std.mem.tokenizeScalar(u8, run_result.stdout, '\n'); + while (it.next()) |bad_file_name| { + try step.addError("{s}: non-conforming formatting", .{bad_file_name}); + } + }, + else => {}, + }; + try step.handleChildProcessTerm(maker, run_result.term); +} diff --git a/lib/compiler/Maker/Step/InstallDir.zig b/lib/compiler/Maker/Step/InstallDir.zig index fb5f288c68404ad3307c2293bf5dc8942efcfa6e..7d079380dd9158a79d4e194490f168a5512e7621 100644 --- a/lib/compiler/Maker/Step/InstallDir.zig +++ b/lib/compiler/Maker/Step/InstallDir.zig @@ -11,7 +11,7 @@ pub fn make( step_index: Configuration.Step.Index, maker: *Maker, progress_node: std.Progress.Node, -) !void { +) Step.ExtendedMakeError!void { const graph = maker.graph; const arena = maker.graph.arena; // TODO don't leak into process arena const io = graph.io; diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index dcdc49a4a0cc051cb67a64afd3fa20a88b098557..f6f3d532c2769be522dba1e0a456400e1f3c7d2a 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -7,12 +7,12 @@ const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); -fn make( +pub fn make( options: *Options, step_index: Configuration.Step.Index, maker: *Maker, progress_node: std.Progress.Node, -) !void { +) Step.ExtendedMakeError!void { // This step completes so quickly that no progress reporting is necessary. _ = progress_node; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 8bc4f6af407979e3f48feb69cd803520154065bd..be9d9940fa255e795401cd6de22490a5b87ffb7d 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -115,8 +115,7 @@ pub const Graph = struct { } pub fn dupeStrings(graph: *const Graph, strings: []const []const u8) []const []const u8 { - const arena = graph.arena; - const array = arena.alloc([]const u8, strings.len) catch @panic("OOM"); + const array = graph.alloc([]const u8, strings.len); for (array, strings) |*dest, source| dest.* = dupeString(graph, source); return array; } @@ -134,6 +133,21 @@ pub const Graph = struct { }, }; } + + /// Allocates using the global process arena, failing the build on + /// allocation failure. + pub fn alloc(graph: *const Graph, comptime T: type, n: usize) []T { + return graph.arena.allocAdvancedWithRetAddr(T, null, n, @returnAddress()) catch @panic("OOM"); + } + + /// Allocates using the global process arena, failing the build on + /// allocation failure. + pub fn create(graph: *const Graph, comptime T: type) *T { + return if (@sizeOf(T) == 0) + comptime @ptrFromInt(mem.alignBackward(usize, std.math.maxInt(usize), @alignOf(T))) + else + @ptrCast(graph.arena.allocBytesWithAlignment(.of(T), @sizeOf(T), @returnAddress()) catch @panic("OOM")); + } }; const AvailableDeps = []const struct { []const u8, []const u8 }; @@ -953,9 +967,10 @@ pub fn getUninstallStep(b: *Build) *Step { /// these options when calling the dependency's build.zig script as a function. /// `null` is returned when an option is left to default. pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw: []const u8) ?T { - const arena = b.allocator; - const name = b.dupe(name_raw); - const description = b.dupe(description_raw); + const graph = b.graph; + const arena = graph.arena; + const name = graph.dupeString(name_raw); + const description = graph.dupeString(description_raw); const type_id = comptime typeToEnum(T); const enum_options = if (type_id == .@"enum" or type_id == .enum_list) blk: { const EnumType = if (type_id == .enum_list) @typeInfo(T).pointer.child else T; @@ -1105,7 +1120,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw }, .list => |lst| { const Child = @typeInfo(T).pointer.child; - const new_list = arena.alloc(Child, lst.items.len) catch @panic("OOM"); + const new_list = graph.alloc(Child, lst.items.len); for (new_list, lst.items) |*new_item, str| { new_item.* = std.meta.stringToEnum(Child, str) orelse { log.err("expected -D{s} to be of type {s}", .{ name, @typeName(Child) }); @@ -1130,7 +1145,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw .scalar => |s| return arena.dupe(LazyPath, &[_]LazyPath{.{ .cwd_relative = s }}) catch @panic("OOM"), .lazy_path => |lp| return arena.dupe(LazyPath, &[_]LazyPath{lp}) catch @panic("OOM"), .list => |lst| { - const new_list = arena.alloc(LazyPath, lst.items.len) catch @panic("OOM"); + const new_list = graph.alloc(LazyPath, lst.items.len); for (new_list, lst.items) |*new_item, str| { new_item.* = .{ .cwd_relative = str }; } @@ -1553,7 +1568,7 @@ pub fn truncateFile(b: *Build, dest_path: []const u8) (Io.Dir.CreateDirError || /// References a file or directory relative to the source root. pub fn path(b: *Build, sub_path: []const u8) LazyPath { if (fs.path.isAbsolute(sub_path)) { - panic("sub_path is expected to be relative to the build root, but was this absolute path: '{s}'. It is best avoid absolute paths, but if you must, it is supported by LazyPath.cwd_relative", .{ + panic("sub_path is expected to be relative to the build root, but was this absolute path: '{s}'. Absolute paths can cause problems but can be created via Graph.cwdRelativePath", .{ sub_path, }); } @@ -1563,6 +1578,14 @@ pub fn path(b: *Build, sub_path: []const u8) LazyPath { } }; } +/// Creates a list of files and/or directories relative to the source root. +pub fn pathList(b: *Build, sub_paths: []const []const u8) []const LazyPath { + const graph = b.graph; + const result = graph.alloc(LazyPath, sub_paths.len); + for (result, sub_paths) |*d, s| d.* = path(b, s); + return result; +} + pub fn pathJoin(b: *Build, paths: []const []const u8) []u8 { return fs.path.join(b.allocator, paths) catch @panic("OOM"); } @@ -2022,10 +2045,11 @@ fn dependencyInner( pkg_deps: AvailableDeps, args: anytype, ) *Dependency { - const io = b.graph.io; - const arena = b.graph.arena; + const graph = b.graph; + const io = graph.io; + const arena = graph.arena; const user_input_options = userInputOptionsFromArgs(arena, args); - if (b.graph.dependency_cache.getContext(.{ + if (graph.dependency_cache.getContext(.{ .build_root_string = build_root_string, .user_input_options = user_input_options, }, .{ .allocator = arena })) |dep| return dep; @@ -2048,10 +2072,10 @@ fn dependencyInner( } } - const dep = arena.create(Dependency) catch @panic("OOM"); + const dep = graph.create(Dependency); dep.* = .{ .builder = sub_builder }; - b.graph.dependency_cache.putContext(b.graph.arena, .{ + graph.dependency_cache.putContext(arena, .{ .build_root_string = build_root_string, .user_input_options = user_input_options, }, dep, .{ .allocator = arena }) catch @panic("OOM"); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 2e81421313f2300092d7deaae15601a40807849f..fc172609a42be5bb6d65940d6a2007b8d3aeb886 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1038,10 +1038,15 @@ pub const Step = extern struct { pub const Fmt = struct { flags: @This().Flags, + paths: Storage.FlagLengthPrefixedList(.flags, .paths, LazyPath.Index), + exclude_paths: Storage.FlagLengthPrefixedList(.flags, .exclude_paths, LazyPath.Index), pub const Flags = packed struct(u32) { tag: Tag = .fmt, - _: u27 = 0, + paths: bool, + exclude_paths: bool, + check: bool, + _: u24 = 0, }; }; diff --git a/lib/std/Build/Step/Fmt.zig b/lib/std/Build/Step/Fmt.zig index bca5385a541ca992e4429e2e64b945ddcf8dfc4e..68f31e36d7fed6c6d39d7053bdcc22feb56d1636 100644 --- a/lib/std/Build/Step/Fmt.zig +++ b/lib/std/Build/Step/Fmt.zig @@ -1,81 +1,47 @@ //! This step has two modes: //! * Modify mode: directly modify source files, formatting them in place. //! * Check mode: fail the step if a non-conforming file is found. +const Fmt = @This(); + const std = @import("std"); const Step = std.Build.Step; -const Fmt = @This(); +const LazyPath = std.Build.LazyPath; +const Configuration = std.Build.Configuration; step: Step, -paths: []const []const u8, -exclude_paths: []const []const u8, +/// Intended to be read-only after the `Fmt` step is created. +paths: []const LazyPath, +/// Intended to be read-only after the `Fmt` step is created. +exclude_paths: []const LazyPath, check: bool, pub const base_tag: Step.Tag = .fmt; pub const Options = struct { - paths: []const []const u8 = &.{}, - exclude_paths: []const []const u8 = &.{}, + paths: []const LazyPath = &.{}, + exclude_paths: []const LazyPath = &.{}, /// If true, fails the build step when any non-conforming files are encountered. check: bool = false, }; pub fn create(owner: *std.Build, options: Options) *Fmt { - const fmt = owner.allocator.create(Fmt) catch @panic("OOM"); - const name = if (options.check) "zig fmt --check" else "zig fmt"; + const graph = owner.graph; + const arena = graph.arena; + const fmt = arena.create(Fmt) catch @panic("OOM"); + fmt.* = .{ - .step = Step.init(.{ + .step = .init(.{ .tag = base_tag, - .name = name, + .name = if (options.check) "zig fmt --check" else "zig fmt", .owner = owner, - .makeFn = make, }), - .paths = owner.dupeStrings(options.paths), - .exclude_paths = owner.dupeStrings(options.exclude_paths), + .paths = options.paths, + .exclude_paths = options.exclude_paths, .check = options.check, }; + + for (options.paths) |lp| lp.addStepDependencies(&fmt.step); + for (options.exclude_paths) |lp| lp.addStepDependencies(&fmt.step); + return fmt; } - -fn make(step: *Step, options: Step.MakeOptions) !void { - const prog_node = options.progress_node; - - // TODO: if check=false, this means we are modifying source files in place, which - // is an operation that could race against other operations also modifying source files - // in place. In this case, this step should obtain a write lock while making those - // modifications. - - const b = step.owner; - const arena = b.allocator; - const fmt: *Fmt = @fieldParentPtr("step", step); - - var argv: std.ArrayList([]const u8) = .empty; - try argv.ensureUnusedCapacity(arena, 2 + 1 + fmt.paths.len + 2 * fmt.exclude_paths.len); - - argv.appendAssumeCapacity(b.graph.zig_exe); - argv.appendAssumeCapacity("fmt"); - - if (fmt.check) { - argv.appendAssumeCapacity("--check"); - } - - for (fmt.paths) |p| { - argv.appendAssumeCapacity(b.pathFromRoot(p)); - } - - for (fmt.exclude_paths) |p| { - argv.appendAssumeCapacity("--exclude"); - argv.appendAssumeCapacity(b.pathFromRoot(p)); - } - - const run_result = try step.captureChildProcess(options.gpa, prog_node, argv.items); - if (fmt.check) switch (run_result.term) { - .exited => |code| if (code != 0 and run_result.stdout.len != 0) { - var it = std.mem.tokenizeScalar(u8, run_result.stdout, '\n'); - while (it.next()) |bad_file_name| { - try step.addError("{s}: non-conforming formatting", .{bad_file_name}); - } - }, - else => {}, - }; - try step.handleChildProcessTerm(run_result.term); -} -- 2.54.0 From 1a83b4d8fa4b631b6cabf06af8321a8d98eccb94 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 4 May 2026 15:57:18 -0700 Subject: [PATCH 081/179] zig build: add zig_exe back to argv trying to eliminate this can be a followup --- BRANCH_TODO | 4 ++++ lib/compiler/configurer.zig | 10 ++++++---- lib/std/Build.zig | 6 ++---- lib/std/mem/Allocator.zig | 6 +++--- src/main.zig | 1 + test/tests.zig | 4 ++-- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 8f95fb7ec6a5b6cd97a3478d534188e8d8216944..762157b6c35e9f86d01e1865a1a2e8e40023ea58 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -65,3 +65,7 @@ closes #31397 + const fmt_include_paths = b.pathList(&.{ "lib", "src", "test", "tools", "build.zig", "build.zig.zon" }); + const fmt_exclude_paths = b.pathList(&.{ "test/cases", "test/behavior/zon" }); ``` + +### std.Build API + +* `b.build_root` (Directory) -> `b.root` (Path) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 96d1072c392550b21109cf6d93f969490bd1749b..b5e874af0c869c88af49f7fa9aac30152611e68f 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -39,6 +39,11 @@ pub fn main(init: process.Init.Minimal) !void { const args = try init.args.toSlice(arena); + var arg_i: usize = 1; // Skip own executable name. + + const zig_exe = expectArgOrFatal(args, &arg_i, "--zig"); + const build_root_sub_path = expectArgOrFatal(args, &arg_i, "--build-root"); + var graph: std.Build.Graph = .{ .io = io, .arena = arena, @@ -49,6 +54,7 @@ pub fn main(init: process.Init.Minimal) !void { .result = try std.zig.system.resolveTargetQuery(io, .{}), }, .generated_files = .empty, + .zig_exe = zig_exe, // Created before running the user's configure script so that some things // can be added during script execution such as strings. @@ -62,10 +68,6 @@ pub fn main(init: process.Init.Minimal) !void { assert(try graph.wip_configuration.addString("") == .empty); assert(try graph.wip_configuration.addString("root") == .root); - var arg_i: usize = 1; // Skip own executable name. - - const build_root_sub_path = expectArgOrFatal(args, &arg_i, "--build-root"); - const cwd: Io.Dir = .cwd(); const build_root: std.Build.Cache.Path = .{ diff --git a/lib/std/Build.zig b/lib/std/Build.zig index be9d9940fa255e795401cd6de22490a5b87ffb7d..69212eee8694156edec6ce0fe5281598a7bf0eb6 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -80,6 +80,7 @@ pub const Graph = struct { arena: Allocator, system_integration_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .empty, system_package_mode: bool = false, + zig_exe: []const u8, environ_map: process.Environ.Map, needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .empty, /// Information about the native target. Computed before build() is invoked. @@ -143,10 +144,7 @@ pub const Graph = struct { /// Allocates using the global process arena, failing the build on /// allocation failure. pub fn create(graph: *const Graph, comptime T: type) *T { - return if (@sizeOf(T) == 0) - comptime @ptrFromInt(mem.alignBackward(usize, std.math.maxInt(usize), @alignOf(T))) - else - @ptrCast(graph.arena.allocBytesWithAlignment(.of(T), @sizeOf(T), @returnAddress()) catch @panic("OOM")); + return @ptrCast(graph.arena.allocBytesAligned(.of(T), @sizeOf(T), @returnAddress()) catch @panic("OOM")); } }; diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index e348e02921c1e345f1a2451072e9c25eea96cc05..4ca16f21258418d602ebf2c67b8a6396f664b728 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -169,7 +169,7 @@ pub fn create(a: Allocator, comptime T: type) Error!*T { const ptr = comptime std.mem.alignBackward(usize, math.maxInt(usize), @alignOf(T)); return @ptrFromInt(ptr); } - const ptr: *T = @ptrCast(try a.allocBytesWithAlignment(.of(T), @sizeOf(T), @returnAddress())); + const ptr: *T = @ptrCast(try a.allocBytesAligned(.of(T), @sizeOf(T), @returnAddress())); return ptr; } @@ -285,10 +285,10 @@ fn allocWithSizeAndAlignment( return_address: usize, ) Error![*]align(alignment.toByteUnits()) u8 { const byte_count = math.mul(usize, size, n) catch return error.OutOfMemory; - return self.allocBytesWithAlignment(alignment, byte_count, return_address); + return self.allocBytesAligned(alignment, byte_count, return_address); } -fn allocBytesWithAlignment( +pub fn allocBytesAligned( self: Allocator, comptime alignment: Alignment, byte_count: usize, diff --git a/src/main.zig b/src/main.zig index 9f4c8507459f7ba2592422689714ea84ef748cb9..cd2ec3c28e53923c20f0297aed6b72cd14284ba0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4982,6 +4982,7 @@ fn cmdBuild( _ = make_argv.addOneAssumeCapacity(); // maker executable make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig-lib-dir", undefined }; const argv_index_zig_lib_dir = make_argv.items.len - 1; diff --git a/test/tests.zig b/test/tests.zig index 5ee5c3f3e2b4b7b2a37363d551385ceba8a12ff0..8591de270ed2773be1ae6c907eccfdc2ec7097d1 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2890,7 +2890,7 @@ pub fn addCases( var cases = @import("src/Cases.zig").init(gpa, arena, io); - var dir = try b.build_root.handle.openDir(io, "test/cases", .{ .iterate = true }); + var dir = try b.root.openDir(io, "test/cases", .{ .iterate = true }); defer dir.close(io); cases.addFromDir(dir, b); @@ -2948,7 +2948,7 @@ pub fn addIncrementalTests(b: *std.Build, test_step: *Step, test_filters: []cons }), }); - var dir = try b.build_root.handle.openDir(io, "test/incremental", .{ .iterate = true }); + var dir = try b.root.openDir(io, "test/incremental", .{ .iterate = true }); defer dir.close(io); var it = try dir.walk(b.graph.arena); -- 2.54.0 From f9ebe0daa2aacb582ca94c1970ec02cc8c39fd75 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 4 May 2026 16:19:25 -0700 Subject: [PATCH 082/179] remove run_output_caching standalone test it relied on custom build steps --- test/standalone/build.zig.zon | 3 - test/standalone/run_output_caching/build.zig | 140 ------------------- test/standalone/run_output_caching/main.zig | 11 -- 3 files changed, 154 deletions(-) delete mode 100644 test/standalone/run_output_caching/build.zig delete mode 100644 test/standalone/run_output_caching/main.zig diff --git a/test/standalone/build.zig.zon b/test/standalone/build.zig.zon index e60c23ee8770dfbea57e9b39b0ffb44208349af5..072bcd0309cf4decce854badf096994606a1e550 100644 --- a/test/standalone/build.zig.zon +++ b/test/standalone/build.zig.zon @@ -171,9 +171,6 @@ .run_output_paths = .{ .path = "run_output_paths", }, - .run_output_caching = .{ - .path = "run_output_caching", - }, .empty_global_error_set = .{ .path = "empty_global_error_set", }, diff --git a/test/standalone/run_output_caching/build.zig b/test/standalone/run_output_caching/build.zig deleted file mode 100644 index 90623e81e6467450f17085add7123393abae868b..0000000000000000000000000000000000000000 --- a/test/standalone/run_output_caching/build.zig +++ /dev/null @@ -1,140 +0,0 @@ -const builtin = @import("builtin"); -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const test_step = b.step("test", "Test it"); - b.default_step = test_step; - - if (builtin.os.tag == .windows) return; // https://codeberg.org/ziglang/zig/issues/31564 - - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - const exe = b.addExecutable(.{ - .name = "create-file", - .root_module = b.createModule(.{ - .root_source_file = b.path("main.zig"), - .target = target, - .optimize = optimize, - }), - }); - - { - const run_random_with_sideeffects_first = b.addRunArtifact(exe); - run_random_with_sideeffects_first.setName("run with side-effects (first)"); - run_random_with_sideeffects_first.has_side_effects = true; - - const run_random_with_sideeffects_second = b.addRunArtifact(exe); - run_random_with_sideeffects_second.setName("run with side-effects (second)"); - run_random_with_sideeffects_second.has_side_effects = true; - - // ensure that "second" runs after "first" - run_random_with_sideeffects_second.step.dependOn(&run_random_with_sideeffects_first.step); - - const first_output = run_random_with_sideeffects_first.addOutputFileArg("a.txt"); - const second_output = run_random_with_sideeffects_second.addOutputFileArg("a.txt"); - - const expect_uncached_dependencies = CheckOutputCaching.init(b, false, &.{ first_output, second_output }); - test_step.dependOn(&expect_uncached_dependencies.step); - - const expect_unequal_output = CheckPathEquality.init(b, true, &.{ first_output, second_output }); - test_step.dependOn(&expect_unequal_output.step); - - const check_first_output = b.addCheckFile(first_output, .{ .expected_matches = &.{"a.txt"} }); - test_step.dependOn(&check_first_output.step); - const check_second_output = b.addCheckFile(second_output, .{ .expected_matches = &.{"a.txt"} }); - test_step.dependOn(&check_second_output.step); - } - - { - const run_random_without_sideeffects_1 = b.addRunArtifact(exe); - run_random_without_sideeffects_1.setName("run without side-effects (A)"); - - const run_random_without_sideeffects_2 = b.addRunArtifact(exe); - run_random_without_sideeffects_2.setName("run without side-effects (B)"); - - run_random_without_sideeffects_2.step.dependOn(&run_random_without_sideeffects_1.step); - - const first_output = run_random_without_sideeffects_1.addOutputFileArg("a.txt"); - const second_output = run_random_without_sideeffects_2.addOutputFileArg("a.txt"); - - const expect_cached_dependencies = CheckOutputCaching.init(b, true, &.{second_output}); - test_step.dependOn(&expect_cached_dependencies.step); - - const expect_equal_output = CheckPathEquality.init(b, true, &.{ first_output, second_output }); - test_step.dependOn(&expect_equal_output.step); - - const check_first_output = b.addCheckFile(first_output, .{ .expected_matches = &.{"a.txt"} }); - test_step.dependOn(&check_first_output.step); - const check_second_output = b.addCheckFile(second_output, .{ .expected_matches = &.{"a.txt"} }); - test_step.dependOn(&check_second_output.step); - } -} - -const CheckOutputCaching = struct { - step: std.Build.Step, - expect_caching: bool, - - pub fn init(owner: *std.Build, expect_caching: bool, output_paths: []const std.Build.LazyPath) *CheckOutputCaching { - const check = owner.allocator.create(CheckOutputCaching) catch @panic("OOM"); - check.* = .{ - .step = std.Build.Step.init(.{ - .id = .custom, - .name = "check output caching", - .owner = owner, - .makeFn = make, - }), - .expect_caching = expect_caching, - }; - for (output_paths) |output_path| { - output_path.addStepDependencies(&check.step); - } - return check; - } - - fn make(step: *std.Build.Step, _: std.Build.Step.MakeOptions) !void { - const check: *CheckOutputCaching = @fieldParentPtr("step", step); - - for (step.dependencies.items) |dependency| { - if (check.expect_caching) { - if (dependency.result_cached) continue; - return step.fail("expected '{s}' step to be cached, but it was not", .{dependency.name}); - } else { - if (!dependency.result_cached) continue; - return step.fail("expected '{s}' step to not be cached, but it was", .{dependency.name}); - } - } - } -}; - -const CheckPathEquality = struct { - step: std.Build.Step, - expected_equality: bool, - output_paths: []const std.Build.LazyPath, - - pub fn init(owner: *std.Build, expected_equality: bool, output_paths: []const std.Build.LazyPath) *CheckPathEquality { - const check = owner.allocator.create(CheckPathEquality) catch @panic("OOM"); - check.* = .{ - .step = std.Build.Step.init(.{ - .id = .custom, - .name = "check output path equality", - .owner = owner, - .makeFn = make, - }), - .expected_equality = expected_equality, - .output_paths = owner.allocator.dupe(std.Build.LazyPath, output_paths) catch @panic("OOM"), - }; - for (output_paths) |output_path| { - output_path.addStepDependencies(&check.step); - } - return check; - } - - fn make(step: *std.Build.Step, _: std.Build.Step.MakeOptions) !void { - const check: *CheckPathEquality = @fieldParentPtr("step", step); - std.debug.assert(check.output_paths.len != 0); - for (check.output_paths[0 .. check.output_paths.len - 1], check.output_paths[1..]) |a, b| { - try std.testing.expectEqual(check.expected_equality, std.mem.eql(u8, a.getPath(step.owner), b.getPath(step.owner))); - } - } -}; diff --git a/test/standalone/run_output_caching/main.zig b/test/standalone/run_output_caching/main.zig deleted file mode 100644 index 8a9855f5ecef4709e02b20a6b302370562895224..0000000000000000000000000000000000000000 --- a/test/standalone/run_output_caching/main.zig +++ /dev/null @@ -1,11 +0,0 @@ -const std = @import("std"); - -pub fn main(init: std.process.Init) !void { - const io = init.io; - var args = try init.minimal.args.iterateAllocator(init.arena.allocator()); - _ = args.skip(); - const filename = args.next().?; - const file = try std.Io.Dir.cwd().createFile(io, filename, .{}); - defer file.close(io); - try file.writeStreamingAll(io, filename); -} -- 2.54.0 From beef9fdf42726d8cfe3017ff3017767fb615ef08 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 4 May 2026 16:28:46 -0700 Subject: [PATCH 083/179] remove ios standalone test relied on setting sysroot in configure phase let's examine this use case more carefully next time before reinstating this test. --- test/standalone/build.zig.zon | 3 --- test/standalone/ios/build.zig | 40 ----------------------------------- test/standalone/ios/main.m | 34 ----------------------------- 3 files changed, 77 deletions(-) delete mode 100644 test/standalone/ios/build.zig delete mode 100644 test/standalone/ios/main.m diff --git a/test/standalone/build.zig.zon b/test/standalone/build.zig.zon index 072bcd0309cf4decce854badf096994606a1e550..175a106f3e544374e88a8aa1ab76d3a2abfc4895 100644 --- a/test/standalone/build.zig.zon +++ b/test/standalone/build.zig.zon @@ -153,9 +153,6 @@ .compiler_rt_panic = .{ .path = "compiler_rt_panic", }, - .ios = .{ - .path = "ios", - }, .depend_on_main_mod = .{ .path = "depend_on_main_mod", }, diff --git a/test/standalone/ios/build.zig b/test/standalone/ios/build.zig deleted file mode 100644 index c8d4eb6dfb2ce45945a89938d091d5ca9fd44cba..0000000000000000000000000000000000000000 --- a/test/standalone/ios/build.zig +++ /dev/null @@ -1,40 +0,0 @@ -const std = @import("std"); - -pub const requires_symlinks = true; -pub const requires_ios_sdk = true; - -pub fn build(b: *std.Build) void { - const test_step = b.step("test", "Test it"); - b.default_step = test_step; - - const optimize: std.builtin.OptimizeMode = .Debug; - const target = b.resolveTargetQuery(.{ - .cpu_arch = .aarch64, - .os_tag = .ios, - }); - - const exe = b.addExecutable(.{ - .name = "main", - .root_module = b.createModule(.{ - .root_source_file = null, - .optimize = optimize, - .target = target, - .link_libc = true, - }), - }); - - const io = b.graph.io; - - if (std.zig.system.darwin.getSdk(b.allocator, io, &target.result)) |sdk| { - b.sysroot = sdk; - exe.root_module.addSystemIncludePath(.{ .cwd_relative = b.pathJoin(&.{ sdk, "/usr/include" }) }); - exe.root_module.addSystemFrameworkPath(.{ .cwd_relative = b.pathJoin(&.{ sdk, "/System/Library/Frameworks" }) }); - exe.root_module.addLibraryPath(.{ .cwd_relative = b.pathJoin(&.{ sdk, "/usr/lib" }) }); - } else { - exe.step.dependOn(&b.addFail("no iOS SDK found").step); - } - - exe.root_module.addCSourceFile(.{ .file = b.path("main.m"), .flags = &.{} }); - exe.root_module.linkFramework("Foundation", .{}); - exe.root_module.linkFramework("UIKit", .{}); -} diff --git a/test/standalone/ios/main.m b/test/standalone/ios/main.m deleted file mode 100644 index 35465210e365e6f20d7a2d72f369a04e110d5317..0000000000000000000000000000000000000000 --- a/test/standalone/ios/main.m +++ /dev/null @@ -1,34 +0,0 @@ -#import - -@interface AppDelegate : UIResponder -@property (strong, nonatomic) UIWindow *window; -@end - -int main() { - @autoreleasepool { - return UIApplicationMain(0, nil, nil, NSStringFromClass([AppDelegate class])); - } -} - -@implementation AppDelegate - -- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(id)options { - CGRect mainScreenBounds = [[UIScreen mainScreen] bounds]; - self.window = [[UIWindow alloc] initWithFrame:mainScreenBounds]; - UIViewController *viewController = [[UIViewController alloc] init]; - viewController.view.frame = mainScreenBounds; - - NSString* msg = @"Hello world"; - - UILabel *label = [[UILabel alloc] initWithFrame:mainScreenBounds]; - [label setText:msg]; - [viewController.view addSubview: label]; - - self.window.rootViewController = viewController; - - [self.window makeKeyAndVisible]; - - return YES; -} - -@end -- 2.54.0 From d3ec255a1f7cd38164de816462c29de4a2981490 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 4 May 2026 17:28:48 -0700 Subject: [PATCH 084/179] more progress towards zig's build.zig compiling --- BRANCH_TODO | 5 +- lib/compiler/Maker/Step.zig | 23 +- lib/compiler/Maker/Step/CheckFile.zig | 60 ++ lib/compiler/Maker/Step/ConfigHeader.zig | 903 +++++++++++++++++++ lib/compiler/configurer.zig | 7 +- lib/std/Build.zig | 56 +- lib/std/Build/Configuration.zig | 12 +- lib/std/Build/Module.zig | 5 +- lib/std/Build/Step/CheckFile.zig | 76 +- lib/std/Build/Step/Compile.zig | 18 +- lib/std/Build/Step/ConfigHeader.zig | 937 +------------------- lib/std/Build/Step/Fail.zig | 26 +- test/standalone/libfuzzer/build.zig | 2 +- test/standalone/windows_resources/build.zig | 2 +- test/tests.zig | 13 +- 15 files changed, 1117 insertions(+), 1028 deletions(-) create mode 100644 lib/compiler/Maker/Step/CheckFile.zig create mode 100644 lib/compiler/Maker/Step/ConfigHeader.zig diff --git a/BRANCH_TODO b/BRANCH_TODO index 762157b6c35e9f86d01e1865a1a2e8e40023ea58..846ce64fe6aa5416b2f854a9882ce65aa5dfdc46 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -27,9 +27,12 @@ - but artifact install steps also add paths for dyn libs on windows * no more "artifact arg" to run step. if you want to run the post-install binary, get the lazy path from the install step. -* build system fmt step with check=false does not acquire a write lock on source files #35204 * fmt step: import zig fmt code directly rather than child proc +## Already Filed Followup Issues +* build system fmt step with check=false does not acquire a write lock on source files #35204 +* enhance CheckFile step output when there is not a match #35208 + ## Release Notes ### Run Step: Passthru Args diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 261310f726d9f6f0776c7e2b5b7def4689ae1d47..edc7dd7edfff9d21b1109c3dc90ef65423dc987c 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -69,7 +69,7 @@ pub const Extended = union(enum) { check_file: Todo, compile: Compile, config_header: Todo, - fail: Todo, + fail: Fail, find_program: Todo, fmt: Todo, install_artifact: InstallArtifact, @@ -133,6 +133,27 @@ pub const Extended = union(enum) { _ = progress_node; } }; + + pub const Fail = struct { + pub fn make( + this: *@This(), + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, + ) Step.ExtendedMakeError!void { + _ = this; + _ = progress_node; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + const conf = &maker.scanned_config.configuration; + const step = maker.stepByIndex(step_index); + const conf_step = step_index.ptr(conf); + const conf_fail = conf_step.extended.get(conf.extra).fail; + + try step.result_error_msgs.append(arena, conf_fail.msg.slice(conf)); + return error.MakeFailed; + } + }; }; pub const State = enum { diff --git a/lib/compiler/Maker/Step/CheckFile.zig b/lib/compiler/Maker/Step/CheckFile.zig new file mode 100644 index 0000000000000000000000000000000000000000..0ed23c05e01952091d0c02ff26a642446e9ae1a0 --- /dev/null +++ b/lib/compiler/Maker/Step/CheckFile.zig @@ -0,0 +1,60 @@ +const CheckFile = @This(); + +const std = @import("std"); +const Io = std.Io; +const Configuration = std.Build.Configuration; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + check_file: *CheckFile, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + _ = progress_node; + const graph = maker.graph; + const arena = maker.graph.arena; // TODO don't leak into process arena + const io = graph.io; + const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_cf = conf_step.extended.get(conf.extra).install_file; + const lazy_path = conf_cf.file.get(conf); + + try step.singleUnchangingWatchInput(maker, arena, lazy_path); + + const src_path = try maker.resolveLazyPath(arena, lazy_path, step_index); + const limit: Io.Limit = if (conf_cf.max_bytes.value) |x| .limited(x) else .unlimited; + + const contents = src_path.root_dir.handle.readFileAlloc(io, src_path.sub_path, arena, limit) catch |err| + return step.fail("failed to read {f}: {t}", .{ src_path, err }); + + for (check_file.expected_matches) |expected_match| { + if (std.mem.find(u8, contents, expected_match) == null) { + return step.fail( + \\ + \\========= expected to find: =================== + \\{s} + \\========= but file does not contain it: ======= + \\{s} + \\=============================================== + , .{ expected_match, contents }); + } + } + + if (check_file.expected_exact) |expected_exact| { + if (!std.mem.eql(u8, expected_exact, contents)) { + return step.fail( + \\ + \\========= expected: ===================== + \\{s} + \\========= but found: ==================== + \\{s} + \\========= from the following file: ====== + \\{s} + , .{ expected_exact, contents, src_path }); + } + } +} diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig new file mode 100644 index 0000000000000000000000000000000000000000..89c8134400d280875b4e0148f847356f97da3b4a --- /dev/null +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -0,0 +1,903 @@ +const ConfigHeader = @This(); + +const std = @import("std"); +const Io = std.Io; +const Configuration = std.Build.Configuration; +const Writer = std.Io.Writer; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + config_header: *ConfigHeader, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + const graph = maker.graph; + const arena = maker.graph.arena; // TODO don't leak into process arena + const step = maker.stepByIndex(step_index); + const io = graph.io; + + if (config_header.style.getPath()) |lp| + try step.singleUnchangingWatchInput(maker, arena, lp); + + var man = graph.cache.obtain(); + defer man.deinit(); + + // Random bytes to make ConfigHeader unique. Refresh this with new + // random bytes when ConfigHeader implementation is modified in a + // non-backwards-compatible way. + man.hash.add(@as(u32, 0xdef08d23)); + man.hash.addBytes(config_header.include_path); + man.hash.addOptionalBytes(config_header.include_guard_override); + + var aw: Writer.Allocating = .init(arena); + defer aw.deinit(); + const bw = &aw.writer; + + const header_text = "This file was generated by ConfigHeader using the Zig Build System."; + const c_generated_line = "/* " ++ header_text ++ " */\n"; + const asm_generated_line = "; " ++ header_text ++ "\n"; + + switch (config_header.style) { + .autoconf_undef, .autoconf_at => |file_source| { + try bw.writeAll(c_generated_line); + const src_path = file_source.getPath2(b, step); + const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(config_header.max_bytes)) catch |err| { + return step.fail("unable to read autoconf input file {s}: {t}", .{ src_path, err }); + }; + switch (config_header.style) { + .autoconf_undef => try render_autoconf_undef(step, contents, bw, &config_header.values, src_path), + .autoconf_at => try render_autoconf_at(step, contents, &aw, &config_header.values, src_path), + else => unreachable, + } + }, + .cmake => |file_source| { + try bw.writeAll(c_generated_line); + const src_path = file_source.getPath2(b, step); + const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(config_header.max_bytes)) catch |err| { + return step.fail("unable to read cmake input file {s}: {t}", .{ src_path, err }); + }; + try render_cmake(step, contents, bw, config_header.values, src_path); + }, + .blank => { + try bw.writeAll(c_generated_line); + try render_blank(gpa, bw, config_header.values, config_header.include_path, config_header.include_guard_override); + }, + .nasm => { + try bw.writeAll(asm_generated_line); + try render_nasm(bw, config_header.values); + }, + } + + const output = aw.written(); + man.hash.addBytes(output); + + if (try step.cacheHit(&man)) { + const digest = man.final(); + config_header.generated_dir.path = try b.cache_root.join(arena, &.{ "o", &digest }); + return; + } + + const digest = man.final(); + + // If output_path has directory parts, deal with them. Example: + // output_dir is zig-cache/o/HASH + // output_path is libavutil/avconfig.h + // We want to open directory zig-cache/o/HASH/libavutil/ + // but keep output_dir as zig-cache/o/HASH for -I include + const sub_path = b.pathJoin(&.{ "o", &digest, config_header.include_path }); + const sub_path_dirname = std.fs.path.dirname(sub_path).?; + + b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, sub_path_dirname, @errorName(err), + }); + }; + + b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = output }) catch |err| { + return step.fail("unable to write file '{f}{s}': {s}", .{ + b.cache_root, sub_path, @errorName(err), + }); + }; + + config_header.generated_dir.path = try b.cache_root.join(arena, &.{ "o", &digest }); + try man.writeManifest(); +} + + +fn render_autoconf_undef( + step: *Step, + contents: []const u8, + bw: *Writer, + values: *const std.array_hash_map.String(Value), + src_path: []const u8, +) !void { + const build = step.owner; + const allocator = build.allocator; + + var is_used: std.bit_set.Dynamic = try .initEmpty(allocator, values.count()); + defer is_used.deinit(allocator); + + var any_errors = false; + var line_index: u32 = 0; + var line_it = std.mem.splitScalar(u8, contents, '\n'); + while (line_it.next()) |line| : (line_index += 1) { + if (!std.mem.startsWith(u8, line, "#")) { + try bw.writeAll(line); + try bw.writeByte('\n'); + continue; + } + var it = std.mem.tokenizeAny(u8, line[1..], " \t\r"); + const undef = it.next().?; + if (!std.mem.eql(u8, undef, "undef")) { + try bw.writeAll(line); + try bw.writeByte('\n'); + continue; + } + const name = it.next().?; + const index = values.getIndex(name) orelse { + try step.addError("{s}:{d}: error: unspecified config header value: '{s}'", .{ + src_path, line_index + 1, name, + }); + any_errors = true; + continue; + }; + is_used.set(index); + try renderValueC(bw, name, values.values()[index]); + } + + var unused_value_it = is_used.iterator(.{ .kind = .unset }); + while (unused_value_it.next()) |index| { + try step.addError("{s}: error: config header value unused: '{s}'", .{ src_path, values.keys()[index] }); + any_errors = true; + } + + if (any_errors) { + return error.MakeFailed; + } +} + +fn render_autoconf_at( + step: *Step, + contents: []const u8, + aw: *Writer.Allocating, + values: *const std.array_hash_map.String(Value), + src_path: []const u8, +) !void { + const build = step.owner; + const allocator = build.allocator; + const bw = &aw.writer; + + const used = allocator.alloc(bool, values.count()) catch @panic("OOM"); + for (used) |*u| u.* = false; + defer allocator.free(used); + + var any_errors = false; + var line_index: u32 = 0; + var line_it = std.mem.splitScalar(u8, contents, '\n'); + while (line_it.next()) |line| : (line_index += 1) { + const last_line = line_it.index == line_it.buffer.len; + + const old_len = aw.written().len; + expand_variables_autoconf_at(bw, line, values, used) catch |err| switch (err) { + error.MissingValue => { + const name = aw.written()[old_len..]; + defer aw.shrinkRetainingCapacity(old_len); + try step.addError("{s}:{d}: error: unspecified config header value: '{s}'", .{ + src_path, line_index + 1, name, + }); + any_errors = true; + continue; + }, + else => { + try step.addError("{s}:{d}: unable to substitute variable: error: {s}", .{ + src_path, line_index + 1, @errorName(err), + }); + any_errors = true; + continue; + }, + }; + if (!last_line) try bw.writeByte('\n'); + } + + for (values.entries.slice().items(.key), used) |name, u| { + if (!u) { + try step.addError("{s}: error: config header value unused: '{s}'", .{ src_path, name }); + any_errors = true; + } + } + + if (any_errors) return error.MakeFailed; +} + +fn render_cmake( + step: *Step, + contents: []const u8, + bw: *Writer, + values: std.array_hash_map.String(Value), + src_path: []const u8, +) !void { + const build = step.owner; + const allocator = build.allocator; + + var values_copy = try values.clone(allocator); + defer values_copy.deinit(allocator); + + var any_errors = false; + var line_index: u32 = 0; + var line_it = std.mem.splitScalar(u8, contents, '\n'); + while (line_it.next()) |raw_line| : (line_index += 1) { + const last_line = line_it.index == line_it.buffer.len; + + const line = expand_variables_cmake(allocator, raw_line, values) catch |err| switch (err) { + error.InvalidCharacter => { + try step.addError("{s}:{d}: error: invalid character in a variable name", .{ + src_path, line_index + 1, + }); + any_errors = true; + continue; + }, + else => { + try step.addError("{s}:{d}: unable to substitute variable: error: {s}", .{ + src_path, line_index + 1, @errorName(err), + }); + any_errors = true; + continue; + }, + }; + defer allocator.free(line); + + const line_start = std.mem.findNone(u8, line, " \t\r") orelse { + try bw.writeAll(line); + if (!last_line) try bw.writeByte('\n'); + continue; + }; + const whitespace_prefix = line[0..line_start]; + const trimmed_line = line[line_start..]; + + if (!std.mem.startsWith(u8, trimmed_line, "#")) { + try bw.writeAll(line); + if (!last_line) try bw.writeByte('\n'); + continue; + } + + var it = std.mem.tokenizeAny(u8, trimmed_line[1..], " \t\r"); + const cmakedefine = it.next().?; + if (!std.mem.eql(u8, cmakedefine, "cmakedefine") and + !std.mem.eql(u8, cmakedefine, "cmakedefine01")) + { + try bw.writeAll(line); + if (!last_line) try bw.writeByte('\n'); + continue; + } + + const booldefine = std.mem.eql(u8, cmakedefine, "cmakedefine01"); + + const name = it.next() orelse { + try step.addError("{s}:{d}: error: missing define name", .{ + src_path, line_index + 1, + }); + any_errors = true; + continue; + }; + var value = values_copy.get(name) orelse blk: { + if (booldefine) { + break :blk Value{ .int = 0 }; + } + break :blk Value.undef; + }; + + value = blk: { + switch (value) { + .boolean => |b| { + if (!b) { + break :blk Value.undef; + } + }, + .int => |i| { + if (i == 0) { + break :blk Value.undef; + } + }, + .string => |string| { + if (string.len == 0) { + break :blk Value.undef; + } + }, + + else => {}, + } + break :blk value; + }; + + if (booldefine) { + value = blk: { + switch (value) { + .undef => { + break :blk Value{ .boolean = false }; + }, + .defined => { + break :blk Value{ .boolean = false }; + }, + .boolean => |b| { + break :blk Value{ .boolean = b }; + }, + .int => |i| { + break :blk Value{ .boolean = i != 0 }; + }, + .string => |string| { + break :blk Value{ .boolean = string.len != 0 }; + }, + + else => { + break :blk Value{ .boolean = false }; + }, + } + }; + } else if (value != Value.undef) { + value = Value{ .ident = it.rest() }; + } + + try bw.writeAll(whitespace_prefix); + try renderValueC(bw, name, value); + } + + if (any_errors) { + return error.HeaderConfigFailed; + } +} + +fn render_blank( + gpa: std.mem.Allocator, + bw: *Writer, + defines: std.array_hash_map.String(Value), + include_path: []const u8, + include_guard_override: ?[]const u8, +) !void { + const include_guard_name = include_guard_override orelse blk: { + const name = try gpa.dupe(u8, include_path); + for (name) |*byte| { + switch (byte.*) { + 'a'...'z' => byte.* = byte.* - 'a' + 'A', + 'A'...'Z', '0'...'9' => continue, + else => byte.* = '_', + } + } + break :blk name; + }; + defer if (include_guard_override == null) gpa.free(include_guard_name); + + try bw.print( + \\#ifndef {[0]s} + \\#define {[0]s} + \\ + , .{include_guard_name}); + + const values = defines.values(); + for (defines.keys(), 0..) |name, i| try renderValueC(bw, name, values[i]); + + try bw.print( + \\#endif /* {s} */ + \\ + , .{include_guard_name}); +} + +fn render_nasm(bw: *Writer, defines: std.array_hash_map.String(Value)) !void { + for (defines.keys(), defines.values()) |name, value| try renderValueNasm(bw, name, value); +} + +fn renderValueC(bw: *Writer, name: []const u8, value: Value) !void { + switch (value) { + .undef => try bw.print("/* #undef {s} */\n", .{name}), + .defined => try bw.print("#define {s}\n", .{name}), + .boolean => |b| try bw.print("#define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), + .int => |i| try bw.print("#define {s} {d}\n", .{ name, i }), + .ident => |ident| try bw.print("#define {s} {s}\n", .{ name, ident }), + // TODO: use C-specific escaping instead of zig string literals + .string => |string| try bw.print("#define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), + } +} + +fn renderValueNasm(bw: *Writer, name: []const u8, value: Value) !void { + switch (value) { + .undef => try bw.print("; %undef {s}\n", .{name}), + .defined => try bw.print("%define {s}\n", .{name}), + .boolean => |b| try bw.print("%define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), + .int => |i| try bw.print("%define {s} {d}\n", .{ name, i }), + .ident => |ident| try bw.print("%define {s} {s}\n", .{ name, ident }), + // TODO: use nasm-specific escaping instead of zig string literals + .string => |string| try bw.print("%define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), + } +} + +fn expand_variables_autoconf_at( + bw: *Writer, + contents: []const u8, + values: *const std.array_hash_map.String(Value), + used: []bool, +) !void { + const valid_varname_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"; + + var curr: usize = 0; + var source_offset: usize = 0; + while (curr < contents.len) : (curr += 1) { + if (contents[curr] != '@') continue; + if (std.mem.findScalarPos(u8, contents, curr + 1, '@')) |close_pos| { + if (close_pos == curr + 1) { + // closed immediately, preserve as a literal + continue; + } + const valid_varname_end = std.mem.findNonePos(u8, contents, curr + 1, valid_varname_chars) orelse 0; + if (valid_varname_end != close_pos) { + // contains invalid characters, preserve as a literal + continue; + } + + const key = contents[curr + 1 .. close_pos]; + const index = values.getIndex(key) orelse { + // Report the missing key to the caller. + try bw.writeAll(key); + return error.MissingValue; + }; + const value = values.entries.slice().items(.value)[index]; + used[index] = true; + try bw.writeAll(contents[source_offset..curr]); + switch (value) { + .undef, .defined => {}, + .boolean => |b| try bw.writeByte(@as(u8, '0') + @intFromBool(b)), + .int => |i| try bw.print("{d}", .{i}), + .ident, .string => |s| try bw.writeAll(s), + } + + curr = close_pos; + source_offset = close_pos + 1; + } + } + + try bw.writeAll(contents[source_offset..]); +} + +fn expand_variables_cmake( + allocator: Allocator, + contents: []const u8, + values: std.array_hash_map.String(Value), +) ![]const u8 { + var result: std.array_list.Managed(u8) = .init(allocator); + errdefer result.deinit(); + + const valid_varname_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789/_.+-"; + const open_var = "${"; + + var curr: usize = 0; + var source_offset: usize = 0; + const Position = struct { + source: usize, + target: usize, + }; + var var_stack: std.array_list.Managed(Position) = .init(allocator); + defer var_stack.deinit(); + loop: while (curr < contents.len) : (curr += 1) { + switch (contents[curr]) { + '@' => blk: { + if (std.mem.findScalarPos(u8, contents, curr + 1, '@')) |close_pos| { + if (close_pos == curr + 1) { + // closed immediately, preserve as a literal + break :blk; + } + const valid_varname_end = std.mem.findNonePos(u8, contents, curr + 1, valid_varname_chars) orelse 0; + if (valid_varname_end != close_pos) { + // contains invalid characters, preserve as a literal + break :blk; + } + + const key = contents[curr + 1 .. close_pos]; + const value = values.get(key) orelse return error.MissingValue; + const missing = contents[source_offset..curr]; + try result.appendSlice(missing); + switch (value) { + .undef, .defined => {}, + .boolean => |b| { + try result.append(if (b) '1' else '0'); + }, + .int => |i| { + try result.print("{d}", .{i}); + }, + .ident, .string => |s| { + try result.appendSlice(s); + }, + } + + curr = close_pos; + source_offset = close_pos + 1; + + continue :loop; + } + }, + '$' => blk: { + const next = curr + 1; + if (next == contents.len or contents[next] != '{') { + // no open bracket detected, preserve as a literal + break :blk; + } + const missing = contents[source_offset..curr]; + try result.appendSlice(missing); + try result.appendSlice(open_var); + + source_offset = curr + open_var.len; + curr = next; + try var_stack.append(Position{ + .source = curr, + .target = result.items.len - open_var.len, + }); + + continue :loop; + }, + '}' => blk: { + if (var_stack.items.len == 0) { + // no open bracket, preserve as a literal + break :blk; + } + const open_pos = var_stack.pop().?; + if (source_offset == open_pos.source) { + source_offset += open_var.len; + } + const missing = contents[source_offset..curr]; + try result.appendSlice(missing); + + const key_start = open_pos.target + open_var.len; + const key = result.items[key_start..]; + if (key.len == 0) { + return error.MissingKey; + } + const value = values.get(key) orelse return error.MissingValue; + result.shrinkRetainingCapacity(result.items.len - key.len - open_var.len); + switch (value) { + .undef, .defined => {}, + .boolean => |b| { + try result.append(if (b) '1' else '0'); + }, + .int => |i| { + try result.print("{d}", .{i}); + }, + .ident, .string => |s| { + try result.appendSlice(s); + }, + } + + source_offset = curr + 1; + + continue :loop; + }, + '\\' => { + // backslash is not considered a special character + continue :loop; + }, + else => {}, + } + + if (var_stack.items.len > 0 and std.mem.findScalar(u8, valid_varname_chars, contents[curr]) == null) { + return error.InvalidCharacter; + } + } + + if (source_offset != contents.len) { + const missing = contents[source_offset..]; + try result.appendSlice(missing); + } + + return result.toOwnedSlice(); +} + +fn testReplaceVariablesAutoconfAt( + allocator: Allocator, + contents: []const u8, + expected: []const u8, + values: std.array_hash_map.String(Value), +) !void { + var aw: Writer.Allocating = .init(allocator); + defer aw.deinit(); + + const used = try allocator.alloc(bool, values.count()); + for (used) |*u| u.* = false; + defer allocator.free(used); + + try expand_variables_autoconf_at(&aw.writer, contents, values, used); + + for (used) |u| if (!u) return error.UnusedValue; + try std.testing.expectEqualStrings(expected, aw.written()); +} + +fn testReplaceVariablesCMake( + allocator: Allocator, + contents: []const u8, + expected: []const u8, + values: std.array_hash_map.String(Value), +) !void { + const actual = try expand_variables_cmake(allocator, contents, values); + defer allocator.free(actual); + + try std.testing.expectEqualStrings(expected, actual); +} + +test "expand_variables_autoconf_at simple cases" { + const allocator = std.testing.allocator; + var values: std.array_hash_map.String(Value) = .init(allocator); + defer values.deinit(); + + // empty strings are preserved + try testReplaceVariablesAutoconfAt(allocator, "", "", values); + + // line with misc content is preserved + try testReplaceVariablesAutoconfAt(allocator, "no substitution", "no substitution", values); + + // empty @ sigils are preserved + try testReplaceVariablesAutoconfAt(allocator, "@", "@", values); + try testReplaceVariablesAutoconfAt(allocator, "@@", "@@", values); + try testReplaceVariablesAutoconfAt(allocator, "@@@", "@@@", values); + try testReplaceVariablesAutoconfAt(allocator, "@@@@", "@@@@", values); + + // simple substitution + try values.putNoClobber("undef", .undef); + try testReplaceVariablesAutoconfAt(allocator, "@undef@", "", values); + values.clearRetainingCapacity(); + + try values.putNoClobber("defined", .defined); + try testReplaceVariablesAutoconfAt(allocator, "@defined@", "", values); + values.clearRetainingCapacity(); + + try values.putNoClobber("true", Value{ .boolean = true }); + try testReplaceVariablesAutoconfAt(allocator, "@true@", "1", values); + values.clearRetainingCapacity(); + + try values.putNoClobber("false", Value{ .boolean = false }); + try testReplaceVariablesAutoconfAt(allocator, "@false@", "0", values); + values.clearRetainingCapacity(); + + try values.putNoClobber("int", Value{ .int = 42 }); + try testReplaceVariablesAutoconfAt(allocator, "@int@", "42", values); + values.clearRetainingCapacity(); + + try values.putNoClobber("ident", Value{ .string = "value" }); + try testReplaceVariablesAutoconfAt(allocator, "@ident@", "value", values); + values.clearRetainingCapacity(); + + try values.putNoClobber("string", Value{ .string = "text" }); + try testReplaceVariablesAutoconfAt(allocator, "@string@", "text", values); + values.clearRetainingCapacity(); + + // double packed substitution + try values.putNoClobber("string", Value{ .string = "text" }); + try testReplaceVariablesAutoconfAt(allocator, "@string@@string@", "texttext", values); + values.clearRetainingCapacity(); + + // triple packed substitution + try values.putNoClobber("int", Value{ .int = 42 }); + try values.putNoClobber("string", Value{ .string = "text" }); + try testReplaceVariablesAutoconfAt(allocator, "@string@@int@@string@", "text42text", values); + values.clearRetainingCapacity(); + + // double separated substitution + try values.putNoClobber("int", Value{ .int = 42 }); + try testReplaceVariablesAutoconfAt(allocator, "@int@.@int@", "42.42", values); + values.clearRetainingCapacity(); + + // triple separated substitution + try values.putNoClobber("true", Value{ .boolean = true }); + try values.putNoClobber("int", Value{ .int = 42 }); + try testReplaceVariablesAutoconfAt(allocator, "@int@.@true@.@int@", "42.1.42", values); + values.clearRetainingCapacity(); + + // misc prefix is preserved + try values.putNoClobber("false", Value{ .boolean = false }); + try testReplaceVariablesAutoconfAt(allocator, "false is @false@", "false is 0", values); + values.clearRetainingCapacity(); + + // misc suffix is preserved + try values.putNoClobber("true", Value{ .boolean = true }); + try testReplaceVariablesAutoconfAt(allocator, "@true@ is true", "1 is true", values); + values.clearRetainingCapacity(); + + // surrounding content is preserved + try values.putNoClobber("int", Value{ .int = 42 }); + try testReplaceVariablesAutoconfAt(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", values); + values.clearRetainingCapacity(); + + // incomplete key is preserved + try testReplaceVariablesAutoconfAt(allocator, "@undef", "@undef", values); + + // unknown key leads to an error + try std.testing.expectError(error.MissingValue, testReplaceVariablesAutoconfAt(allocator, "@bad@", "", values)); + + // unused key leads to an error + try values.putNoClobber("int", Value{ .int = 42 }); + try values.putNoClobber("false", Value{ .boolean = false }); + try std.testing.expectError(error.UnusedValue, testReplaceVariablesAutoconfAt(allocator, "@int", "", values)); + values.clearRetainingCapacity(); +} + +test "expand_variables_autoconf_at edge cases" { + const allocator = std.testing.allocator; + var values: std.array_hash_map.String(Value) = .init(allocator); + defer values.deinit(); + + // @-vars resolved only when they wrap valid characters, otherwise considered literals + try values.putNoClobber("string", Value{ .string = "text" }); + try testReplaceVariablesAutoconfAt(allocator, "@@string@@", "@text@", values); + values.clearRetainingCapacity(); + + // expanded variables are considered strings after expansion + try values.putNoClobber("string_at", Value{ .string = "@string@" }); + try testReplaceVariablesAutoconfAt(allocator, "@string_at@", "@string@", values); + values.clearRetainingCapacity(); +} + +test "expand_variables_cmake simple cases" { + const allocator = std.testing.allocator; + var values: std.array_hash_map.String(Value) = .init(allocator); + defer values.deinit(); + + try values.putNoClobber("undef", .undef); + try values.putNoClobber("defined", .defined); + try values.putNoClobber("true", Value{ .boolean = true }); + try values.putNoClobber("false", Value{ .boolean = false }); + try values.putNoClobber("int", Value{ .int = 42 }); + try values.putNoClobber("ident", Value{ .string = "value" }); + try values.putNoClobber("string", Value{ .string = "text" }); + + // empty strings are preserved + try testReplaceVariablesCMake(allocator, "", "", values); + + // line with misc content is preserved + try testReplaceVariablesCMake(allocator, "no substitution", "no substitution", values); + + // empty ${} wrapper leads to an error + try std.testing.expectError(error.MissingKey, testReplaceVariablesCMake(allocator, "${}", "", values)); + + // empty @ sigils are preserved + try testReplaceVariablesCMake(allocator, "@", "@", values); + try testReplaceVariablesCMake(allocator, "@@", "@@", values); + try testReplaceVariablesCMake(allocator, "@@@", "@@@", values); + try testReplaceVariablesCMake(allocator, "@@@@", "@@@@", values); + + // simple substitution + try testReplaceVariablesCMake(allocator, "@undef@", "", values); + try testReplaceVariablesCMake(allocator, "${undef}", "", values); + try testReplaceVariablesCMake(allocator, "@defined@", "", values); + try testReplaceVariablesCMake(allocator, "${defined}", "", values); + try testReplaceVariablesCMake(allocator, "@true@", "1", values); + try testReplaceVariablesCMake(allocator, "${true}", "1", values); + try testReplaceVariablesCMake(allocator, "@false@", "0", values); + try testReplaceVariablesCMake(allocator, "${false}", "0", values); + try testReplaceVariablesCMake(allocator, "@int@", "42", values); + try testReplaceVariablesCMake(allocator, "${int}", "42", values); + try testReplaceVariablesCMake(allocator, "@ident@", "value", values); + try testReplaceVariablesCMake(allocator, "${ident}", "value", values); + try testReplaceVariablesCMake(allocator, "@string@", "text", values); + try testReplaceVariablesCMake(allocator, "${string}", "text", values); + + // double packed substitution + try testReplaceVariablesCMake(allocator, "@string@@string@", "texttext", values); + try testReplaceVariablesCMake(allocator, "${string}${string}", "texttext", values); + + // triple packed substitution + try testReplaceVariablesCMake(allocator, "@string@@int@@string@", "text42text", values); + try testReplaceVariablesCMake(allocator, "@string@${int}@string@", "text42text", values); + try testReplaceVariablesCMake(allocator, "${string}@int@${string}", "text42text", values); + try testReplaceVariablesCMake(allocator, "${string}${int}${string}", "text42text", values); + + // double separated substitution + try testReplaceVariablesCMake(allocator, "@int@.@int@", "42.42", values); + try testReplaceVariablesCMake(allocator, "${int}.${int}", "42.42", values); + + // triple separated substitution + try testReplaceVariablesCMake(allocator, "@int@.@true@.@int@", "42.1.42", values); + try testReplaceVariablesCMake(allocator, "@int@.${true}.@int@", "42.1.42", values); + try testReplaceVariablesCMake(allocator, "${int}.@true@.${int}", "42.1.42", values); + try testReplaceVariablesCMake(allocator, "${int}.${true}.${int}", "42.1.42", values); + + // misc prefix is preserved + try testReplaceVariablesCMake(allocator, "false is @false@", "false is 0", values); + try testReplaceVariablesCMake(allocator, "false is ${false}", "false is 0", values); + + // misc suffix is preserved + try testReplaceVariablesCMake(allocator, "@true@ is true", "1 is true", values); + try testReplaceVariablesCMake(allocator, "${true} is true", "1 is true", values); + + // surrounding content is preserved + try testReplaceVariablesCMake(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", values); + try testReplaceVariablesCMake(allocator, "what is 6*7? ${int}!", "what is 6*7? 42!", values); + + // incomplete key is preserved + try testReplaceVariablesCMake(allocator, "@undef", "@undef", values); + try testReplaceVariablesCMake(allocator, "${undef", "${undef", values); + try testReplaceVariablesCMake(allocator, "{undef}", "{undef}", values); + try testReplaceVariablesCMake(allocator, "undef@", "undef@", values); + try testReplaceVariablesCMake(allocator, "undef}", "undef}", values); + + // unknown key leads to an error + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@bad@", "", values)); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${bad}", "", values)); +} + +test "expand_variables_cmake edge cases" { + const allocator = std.testing.allocator; + var values: std.array_hash_map.String(Value) = .init(allocator); + defer values.deinit(); + + // special symbols + try values.putNoClobber("at", Value{ .string = "@" }); + try values.putNoClobber("dollar", Value{ .string = "$" }); + try values.putNoClobber("underscore", Value{ .string = "_" }); + + // basic value + try values.putNoClobber("string", Value{ .string = "text" }); + + // proxy case values + try values.putNoClobber("string_proxy", Value{ .string = "string" }); + try values.putNoClobber("string_at", Value{ .string = "@string@" }); + try values.putNoClobber("string_curly", Value{ .string = "{string}" }); + try values.putNoClobber("string_var", Value{ .string = "${string}" }); + + // stack case values + try values.putNoClobber("nest_underscore_proxy", Value{ .string = "underscore" }); + try values.putNoClobber("nest_proxy", Value{ .string = "nest_underscore_proxy" }); + + // @-vars resolved only when they wrap valid characters, otherwise considered literals + try testReplaceVariablesCMake(allocator, "@@string@@", "@text@", values); + try testReplaceVariablesCMake(allocator, "@${string}@", "@text@", values); + + // @-vars are resolved inside ${}-vars + try testReplaceVariablesCMake(allocator, "${@string_proxy@}", "text", values); + + // expanded variables are considered strings after expansion + try testReplaceVariablesCMake(allocator, "@string_at@", "@string@", values); + try testReplaceVariablesCMake(allocator, "${string_at}", "@string@", values); + try testReplaceVariablesCMake(allocator, "$@string_curly@", "${string}", values); + try testReplaceVariablesCMake(allocator, "$${string_curly}", "${string}", values); + try testReplaceVariablesCMake(allocator, "${string_var}", "${string}", values); + try testReplaceVariablesCMake(allocator, "@string_var@", "${string}", values); + try testReplaceVariablesCMake(allocator, "${dollar}{${string}}", "${text}", values); + try testReplaceVariablesCMake(allocator, "@dollar@{${string}}", "${text}", values); + try testReplaceVariablesCMake(allocator, "@dollar@{@string@}", "${text}", values); + + // when expanded variables contain invalid characters, they prevent further expansion + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${${string_var}}", "", values)); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${@string_var@}", "", values)); + + // nested expanded variables are expanded from the inside out + try testReplaceVariablesCMake(allocator, "${string${underscore}proxy}", "string", values); + try testReplaceVariablesCMake(allocator, "${string@underscore@proxy}", "string", values); + + // nested vars are only expanded when ${} is closed + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@underscore@proxy@", "", values)); + try testReplaceVariablesCMake(allocator, "${nest${underscore}proxy}", "nest_underscore_proxy", values); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@@nest_underscore@underscore@proxy@@proxy@", "", values)); + try testReplaceVariablesCMake(allocator, "${nest${${nest_underscore${underscore}proxy}}proxy}", "nest_underscore_proxy", values); + + // invalid characters lead to an error + try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str*ing}", "", values)); + try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str$ing}", "", values)); + try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str@ing}", "", values)); +} + +test "expand_variables_cmake escaped characters" { + const allocator = std.testing.allocator; + var values: std.array_hash_map.String(Value) = .init(allocator); + defer values.deinit(); + + try values.putNoClobber("string", Value{ .string = "text" }); + + // backslash is an invalid character for @ lookup + try testReplaceVariablesCMake(allocator, "\\@string\\@", "\\@string\\@", values); + + // backslash is preserved, but doesn't affect ${} variable expansion + try testReplaceVariablesCMake(allocator, "\\${string}", "\\text", values); + + // backslash breaks ${} opening bracket identification + try testReplaceVariablesCMake(allocator, "$\\{string}", "$\\{string}", values); + + // backslash is skipped when checking for invalid characters, yet it mangles the key + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${string\\}", "", values)); +} diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index b5e874af0c869c88af49f7fa9aac30152611e68f..d4086f0cfffa7af7768dfec2ebd48d82f000c0e7 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -861,7 +861,12 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .install_dir => @panic("TODO"), .remove_dir => @panic("TODO"), - .fail => @panic("TODO"), + .fail => e: { + const sf: *Step.Fail = @fieldParentPtr("step", step); + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.Fail, .{ + .msg = sf.error_msg, + }))); + }, .find_program => @panic("TODO"), .fmt => @panic("TODO"), .translate_c => @panic("TODO"), diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 69212eee8694156edec6ce0fe5281598a7bf0eb6..4756be93688bad1039c6d3919dff7479d3870338 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -146,6 +146,22 @@ pub const Graph = struct { pub fn create(graph: *const Graph, comptime T: type) *T { return @ptrCast(graph.arena.allocBytesAligned(.of(T), @sizeOf(T), @returnAddress()) catch @panic("OOM")); } + + pub fn addBytesList(graph: *Graph, bytes_list: []const []const u8) []const Configuration.Bytes { + const result = graph.alloc(Configuration.Bytes, bytes_list.len); + for (result, bytes_list) |*d, s| d.* = addBytes(graph, s); + return result; + } + + pub fn addBytes(graph: *Graph, bytes: []const u8) Configuration.Bytes { + const wc = &graph.wip_configuration; + return wc.addBytes(bytes) catch @panic("OOM"); + } + + pub fn addString(graph: *Graph, bytes: []const u8) Configuration.String { + const wc = &graph.wip_configuration; + return wc.addString(bytes) catch @panic("OOM"); + } }; const AvailableDeps = []const struct { []const u8, []const u8 }; @@ -530,13 +546,17 @@ const OrderedUserValue = union(enum) { hasher.update(sp.sub_path); }, .generated => |gen| { - hasher.update(std.mem.asBytes(&gen.index)); - hasher.update(std.mem.asBytes(&gen.up)); + hasher.update(@ptrCast(&gen.index)); + hasher.update(@ptrCast(&gen.up)); hasher.update(gen.sub_path); }, .cwd_relative => |rel_path| { hasher.update(rel_path); }, + .relative => |r| { + hasher.update(@ptrCast(&r.base)); + hasher.update(@ptrCast(&r.sub_path)); + }, .dependency => |dep| { hasher.update(dep.dependency.builder.pkg_hash); hasher.update(dep.sub_path); @@ -1824,16 +1844,19 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c if (@hasDecl(pkg, "build_zig") and pkg.build_zig == asking_build_zig) break .{ pkg_hash, pkg.deps }; } else .{ "", deps.root_deps }; if (!std.mem.eql(u8, b_pkg_hash, b.pkg_hash)) { - panic("'{}' is not the struct that corresponds to '{s}'", .{ - asking_build_zig, b.pathFromRoot("build.zig"), + const build_zig_path = b.root.join("build.zig") catch @panic("OOM"); + panic("{} is not the struct that corresponds to {f}", .{ + asking_build_zig, build_zig_path, }); } comptime for (b_pkg_deps) |dep| { if (std.mem.eql(u8, dep[0], dep_name)) return dep[1]; }; - const full_path = b.pathFromRoot("build.zig.zon"); - panic("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file", .{ dep_name, full_path }); + const full_path = b.root.join("build.zig.zon") catch @panic("OOM"); + panic("no dependency named {s} in {f}. All packages used in build.zig must be declared in this file", .{ + dep_name, full_path, + }); } fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void { @@ -1935,6 +1958,8 @@ pub fn dependencyFromBuildZig( ) *Dependency { const build_runner = @import("root"); const deps = build_runner.dependencies; + const graph = b.graph; + const arena = graph.arena; find_dep: { const pkg, const pkg_hash = inline for (@typeInfo(deps.packages).@"struct".decls) |decl| { @@ -1948,8 +1973,8 @@ pub fn dependencyFromBuildZig( return dependencyInner(b, dep_name, pkg.build_root, pkg.build_zig, pkg_hash, pkg.deps, args); } - const full_path = b.pathFromRoot("build.zig.zon"); - panic("'{}' is not a build.zig struct of a dependency in '{s}'", .{ build_zig, full_path }); + const full_path = b.root.join(arena, "build.zig.zon") catch @panic("OOM"); + panic("{} is not a build.zig struct of a dependency in {f}", .{ build_zig, full_path }); } fn userValuesAreSame(lhs: UserValue, rhs: UserValue) bool { @@ -2024,6 +2049,7 @@ fn userLazyPathsAreTheSame(lhs_lp: LazyPath, rhs_lp: LazyPath) bool { if (!std.mem.eql(u8, lhs_rel_path, rhs_rel_path)) return false; }, + .relative => |lhs| return lhs.eql(rhs_lp.relative), .dependency => |lhs_dep| { const rhs_dep = rhs_lp.dependency; @@ -2150,6 +2176,10 @@ pub const LazyPath = union(enum) { relative: struct { base: Configuration.Path.Base, sub_path: Configuration.String = .empty, + + pub fn eql(a: @This(), b: @This()) bool { + return a.base == b.base and a.sub_path == b.sub_path; + } }, /// Path to the Zig executable being used to execute "zig build". @@ -2177,11 +2207,11 @@ pub const LazyPath = union(enum) { }, } }, .generated => |generated| .{ .generated = if (dirnameAllowEmpty(generated.sub_path)) |sub_dirname| .{ - .file = generated.file, + .index = generated.index, .up = generated.up, .sub_path = sub_dirname, } else .{ - .file = generated.file, + .index = generated.index, .up = generated.up + 1, .sub_path = "", } }, @@ -2208,6 +2238,9 @@ pub const LazyPath = union(enum) { } }, }, + .relative => .{ + .relative = @panic("TODO"), + }, .dependency => |dep| .{ .dependency = .{ .dependency = dep.dependency, .sub_path = dirnameAllowEmpty(dep.sub_path) orelse { @@ -2239,6 +2272,9 @@ pub const LazyPath = union(enum) { .cwd_relative => |cwd_relative| .{ .cwd_relative = try fs.path.resolve(arena, &.{ cwd_relative, sub_path }), }, + .relative => .{ + .relative = @panic("TODO"), + }, .dependency => |dep| .{ .dependency = .{ .dependency = dep.dependency, .sub_path = try fs.path.resolve(arena, &.{ dep.sub_path, sub_path }), diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index fc172609a42be5bb6d65940d6a2007b8d3aeb886..64a6113110fe40c568d2cd488a127f0d9a345cdb 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1011,10 +1011,17 @@ pub const Step = extern struct { pub const CheckFile = struct { flags: @This().Flags, + file: LazyPath.Index, + expected_exact: Storage.FlagOptional(.flags, .expected_exact, Bytes), + expected_matches: Storage.FlagLengthPrefixedList(.flags, .expected_matches, Bytes), + max_bytes: Storage.FlagOptional(.flags, .max_bytes, u32), pub const Flags = packed struct(u32) { tag: Tag = .check_file, - _: u27 = 0, + expected_exact: bool, + expected_matches: bool, + max_bytes: bool, + _: u24 = 0, }; }; @@ -1028,7 +1035,8 @@ pub const Step = extern struct { }; pub const Fail = struct { - flags: @This().Flags, + flags: @This().Flags = .{}, + msg: String, pub const Flags = packed struct(u32) { tag: Tag = .fail, diff --git a/lib/std/Build/Module.zig b/lib/std/Build/Module.zig index cdc0c134a56f9d8810946563fbaf4e41a31673bc..35a07f1eb88b911f2a0303ee556de7de1bb43e93 100644 --- a/lib/std/Build/Module.zig +++ b/lib/std/Build/Module.zig @@ -145,14 +145,13 @@ pub const RcSourceFile = struct { /// as `/I `. include_paths: []const LazyPath = &.{}, - pub fn dupe(file: RcSourceFile, b: *std.Build) RcSourceFile { - const graph = b.owner.graph; + pub fn dupe(file: RcSourceFile, graph: *const std.Build.Graph) RcSourceFile { const arena = graph.arena; const include_paths = arena.alloc(LazyPath, file.include_paths.len) catch @panic("OOM"); for (include_paths, file.include_paths) |*dest, lazy_path| dest.* = lazy_path.dupe(graph); return .{ .file = file.file.dupe(graph), - .flags = b.dupeStrings(file.flags), + .flags = graph.dupeStrings(file.flags), .include_paths = include_paths, }; } diff --git a/lib/std/Build/Step/CheckFile.zig b/lib/std/Build/Step/CheckFile.zig index 4cb968ff96ef2343153d1f522406ac35db088d7d..21ebbd4621c51319de1cc69579c0c59a4ec1df0d 100644 --- a/lib/std/Build/Step/CheckFile.zig +++ b/lib/std/Build/Step/CheckFile.zig @@ -1,7 +1,4 @@ //! Fail the build step if a file does not match certain checks. -//! TODO: make this more flexible, supporting more kinds of checks. -//! TODO: generalize the code in std.testing.expectEqualStrings and make this -//! CheckFile step produce those helpful diagnostics when there is not a match. const CheckFile = @This(); const std = @import("std"); @@ -9,83 +6,40 @@ const Io = std.Io; const Step = std.Build.Step; const fs = std.fs; const mem = std.mem; +const Configuration = std.Build.Configuration; step: Step, -expected_matches: []const []const u8, -expected_exact: ?[]const u8, -source: std.Build.LazyPath, -max_bytes: usize = 20 * 1024 * 1024, +file: std.Build.LazyPath, +expected_matches: []const Configuration.Bytes, +expected_exact: ?Configuration.Bytes, +max_bytes: ?u32, pub const base_tag: Step.Tag = .check_file; pub const Options = struct { expected_matches: []const []const u8 = &.{}, expected_exact: ?[]const u8 = null, + max_bytes: ?u32 = null, }; -pub fn create( - owner: *std.Build, - source: std.Build.LazyPath, - options: Options, -) *CheckFile { - const check_file = owner.allocator.create(CheckFile) catch @panic("OOM"); +pub fn create(owner: *std.Build, file: std.Build.LazyPath, options: Options) *CheckFile { + const graph = owner.graph; + const check_file = graph.create(CheckFile); check_file.* = .{ - .step = Step.init(.{ + .step = .init(.{ .tag = base_tag, .name = "CheckFile", .owner = owner, - .makeFn = make, }), - .source = source.dupe(owner), - .expected_matches = owner.dupeStrings(options.expected_matches), - .expected_exact = options.expected_exact, + .file = file.dupe(graph), + .expected_matches = graph.addBytesList(options.expected_matches), + .expected_exact = if (options.expected_exact) |b| graph.addBytes(b) else null, + .max_bytes = options.max_bytes, }; - check_file.source.addStepDependencies(&check_file.step); + file.addStepDependencies(&check_file.step); return check_file; } pub fn setName(check_file: *CheckFile, name: []const u8) void { check_file.step.name = name; } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const io = b.graph.io; - const check_file: *CheckFile = @fieldParentPtr("step", step); - try step.singleUnchangingWatchInput(check_file.source); - - const src_path = check_file.source.getPath2(b, step); - const contents = Io.Dir.cwd().readFileAlloc(io, src_path, b.allocator, .limited(check_file.max_bytes)) catch |err| { - return step.fail("unable to read '{s}': {s}", .{ - src_path, @errorName(err), - }); - }; - - for (check_file.expected_matches) |expected_match| { - if (mem.find(u8, contents, expected_match) == null) { - return step.fail( - \\ - \\========= expected to find: =================== - \\{s} - \\========= but file does not contain it: ======= - \\{s} - \\=============================================== - , .{ expected_match, contents }); - } - } - - if (check_file.expected_exact) |expected_exact| { - if (!mem.eql(u8, expected_exact, contents)) { - return step.fail( - \\ - \\========= expected: ===================== - \\{s} - \\========= but found: ==================== - \\{s} - \\========= from the following file: ====== - \\{s} - , .{ expected_exact, contents, src_path }); - } - } -} diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 2d0cec2bfcdc466e52c3021edf63e2aa0154179d..9a06ca87246bc344ab64166c7551a2c910854903 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -296,7 +296,7 @@ pub const HeaderInstallation = union(enum) { source: LazyPath, dest_rel_path: []const u8, - pub fn dupe(file: File, graph: *std.Build.Graph) File { + pub fn dupe(file: File, graph: *const std.Build.Graph) File { return .{ .source = file.source.dupe(graph), .dest_rel_path = graph.dupePath(file.dest_rel_path), @@ -317,7 +317,7 @@ pub const HeaderInstallation = union(enum) { /// `exclude_extensions` takes precedence over `include_extensions`. include_extensions: ?[]const []const u8 = &.{".h"}, - pub fn dupe(opts: Directory.Options, graph: *std.Build.Graph) Directory.Options { + pub fn dupe(opts: Directory.Options, graph: *const std.Build.Graph) Directory.Options { return .{ .exclude_extensions = graph.dupeStrings(opts.exclude_extensions), .include_extensions = if (opts.include_extensions) |incs| graph.dupeStrings(incs) else null, @@ -325,11 +325,11 @@ pub const HeaderInstallation = union(enum) { } }; - pub fn dupe(dir: Directory, b: *std.Build) Directory { + pub fn dupe(dir: Directory, graph: *const std.Build.Graph) Directory { return .{ - .source = dir.source.dupe(b), - .dest_rel_path = b.dupePath(dir.dest_rel_path), - .options = dir.options.dupe(b), + .source = dir.source.dupe(graph), + .dest_rel_path = graph.dupePath(dir.dest_rel_path), + .options = dir.options.dupe(graph), }; } }; @@ -340,10 +340,10 @@ pub const HeaderInstallation = union(enum) { }; } - pub fn dupe(installation: HeaderInstallation, b: *std.Build) HeaderInstallation { + pub fn dupe(installation: HeaderInstallation, graph: *const std.Build.Graph) HeaderInstallation { return switch (installation) { - .file => |f| .{ .file = f.dupe(b) }, - .directory => |d| .{ .directory = d.dupe(b) }, + .file => |f| .{ .file = f.dupe(graph) }, + .directory => |d| .{ .directory = d.dupe(graph) }, }; } }; diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index 8e76d39fa155de6d614c41b55ebd016e41063bc0..026496fe4a08378cbe89b1b8f2dda947b5108747 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -4,9 +4,20 @@ const std = @import("std"); const Io = std.Io; const Step = std.Build.Step; const Allocator = std.mem.Allocator; -const Writer = std.Io.Writer; const Configuration = std.Build.Configuration; +step: Step, +values: std.array_hash_map.String(Value), +/// This directory contains the generated file under the name `include_path`. +generated_dir: Configuration.GeneratedFileIndex, + +style: Style, +max_bytes: usize, +include_path: []const u8, +include_guard_override: ?[]const u8, + +pub const base_tag: Step.Tag = .config_header; + pub const Style = union(enum) { /// A configure format supported by autotools that uses `#undef foo` to /// mark lines that can be substituted with different values. @@ -38,18 +49,6 @@ pub const Value = union(enum) { string: []const u8, }; -step: Step, -values: std.array_hash_map.String(Value), -/// This directory contains the generated file under the name `include_path`. -generated_dir: Configuration.GeneratedFileIndex, - -style: Style, -max_bytes: usize, -include_path: []const u8, -include_guard_override: ?[]const u8, - -pub const base_tag: Step.Tag = .config_header; - pub const Options = struct { style: Style = .blank, max_bytes: usize = 2 * 1024 * 1024, @@ -66,10 +65,12 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { var include_path: []const u8 = "config.h"; if (options.style.getPath()) |s| default_include_path: { + const wc = &graph.wip_configuration; const sub_path = switch (s) { .src_path => |sp| sp.sub_path, .generated => break :default_include_path, .cwd_relative => |sub_path| sub_path, + .relative => |r| wc.stringSlice(r.sub_path), .dependency => |dependency| dependency.sub_path, }; const basename = std.fs.path.basename(sub_path); @@ -92,14 +93,13 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { .tag = base_tag, .name = name, .owner = owner, - .makeFn = make, .first_ret_addr = options.first_ret_addr orelse @returnAddress(), }), .style = options.style, .values = .empty, .max_bytes = options.max_bytes, - .include_path = include_path, + .include_path = graph.dupeString(include_path), .include_guard_override = options.include_guard_override, .generated_dir = graph.addGeneratedFile(&config_header.step), }; @@ -119,19 +119,6 @@ pub fn addValue(config_header: *ConfigHeader, name: []const u8, comptime T: type return addValueInner(config_header, name, T, value) catch @panic("OOM"); } -pub fn addValues(config_header: *ConfigHeader, values: anytype) void { - inline for (@typeInfo(@TypeOf(values)).@"struct".fields) |field| { - addValue(config_header, field.name, field.type, @field(values, field.name)); - } -} - -pub fn getOutputDir(ch: *ConfigHeader) std.Build.LazyPath { - return .{ .generated = .{ .index = &ch.generated_dir } }; -} -pub fn getOutputFile(ch: *ConfigHeader) std.Build.LazyPath { - return ch.getOutputDir().path(ch.step.owner, ch.include_path); -} - fn addValueInner(config_header: *ConfigHeader, name: []const u8, comptime T: type, value: T) !void { const arena = config_header.step.owner.allocator; switch (@typeInfo(T)) { @@ -183,895 +170,15 @@ fn addValueInner(config_header: *ConfigHeader, name: []const u8, comptime T: typ } } -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const config_header: *ConfigHeader = @fieldParentPtr("step", step); - if (config_header.style.getPath()) |lp| try step.singleUnchangingWatchInput(lp); - - const gpa = b.allocator; - const arena = b.allocator; - const io = b.graph.io; - - var man = b.graph.cache.obtain(); - defer man.deinit(); - - // Random bytes to make ConfigHeader unique. Refresh this with new - // random bytes when ConfigHeader implementation is modified in a - // non-backwards-compatible way. - man.hash.add(@as(u32, 0xdef08d23)); - man.hash.addBytes(config_header.include_path); - man.hash.addOptionalBytes(config_header.include_guard_override); - - var aw: Writer.Allocating = .init(gpa); - defer aw.deinit(); - const bw = &aw.writer; - - const header_text = "This file was generated by ConfigHeader using the Zig Build System."; - const c_generated_line = "/* " ++ header_text ++ " */\n"; - const asm_generated_line = "; " ++ header_text ++ "\n"; - - switch (config_header.style) { - .autoconf_undef, .autoconf_at => |file_source| { - try bw.writeAll(c_generated_line); - const src_path = file_source.getPath2(b, step); - const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(config_header.max_bytes)) catch |err| { - return step.fail("unable to read autoconf input file '{s}': {s}", .{ - src_path, @errorName(err), - }); - }; - switch (config_header.style) { - .autoconf_undef => try render_autoconf_undef(step, contents, bw, &config_header.values, src_path), - .autoconf_at => try render_autoconf_at(step, contents, &aw, &config_header.values, src_path), - else => unreachable, - } - }, - .cmake => |file_source| { - try bw.writeAll(c_generated_line); - const src_path = file_source.getPath2(b, step); - const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(config_header.max_bytes)) catch |err| { - return step.fail("unable to read cmake input file '{s}': {s}", .{ - src_path, @errorName(err), - }); - }; - try render_cmake(step, contents, bw, config_header.values, src_path); - }, - .blank => { - try bw.writeAll(c_generated_line); - try render_blank(gpa, bw, config_header.values, config_header.include_path, config_header.include_guard_override); - }, - .nasm => { - try bw.writeAll(asm_generated_line); - try render_nasm(bw, config_header.values); - }, - } - - const output = aw.written(); - man.hash.addBytes(output); - - if (try step.cacheHit(&man)) { - const digest = man.final(); - config_header.generated_dir.path = try b.cache_root.join(arena, &.{ "o", &digest }); - return; - } - - const digest = man.final(); - - // If output_path has directory parts, deal with them. Example: - // output_dir is zig-cache/o/HASH - // output_path is libavutil/avconfig.h - // We want to open directory zig-cache/o/HASH/libavutil/ - // but keep output_dir as zig-cache/o/HASH for -I include - const sub_path = b.pathJoin(&.{ "o", &digest, config_header.include_path }); - const sub_path_dirname = std.fs.path.dirname(sub_path).?; - - b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, sub_path_dirname, @errorName(err), - }); - }; - - b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = output }) catch |err| { - return step.fail("unable to write file '{f}{s}': {s}", .{ - b.cache_root, sub_path, @errorName(err), - }); - }; - - config_header.generated_dir.path = try b.cache_root.join(arena, &.{ "o", &digest }); - try man.writeManifest(); -} - -fn render_autoconf_undef( - step: *Step, - contents: []const u8, - bw: *Writer, - values: *const std.array_hash_map.String(Value), - src_path: []const u8, -) !void { - const build = step.owner; - const allocator = build.allocator; - - var is_used: std.bit_set.Dynamic = try .initEmpty(allocator, values.count()); - defer is_used.deinit(allocator); - - var any_errors = false; - var line_index: u32 = 0; - var line_it = std.mem.splitScalar(u8, contents, '\n'); - while (line_it.next()) |line| : (line_index += 1) { - if (!std.mem.startsWith(u8, line, "#")) { - try bw.writeAll(line); - try bw.writeByte('\n'); - continue; - } - var it = std.mem.tokenizeAny(u8, line[1..], " \t\r"); - const undef = it.next().?; - if (!std.mem.eql(u8, undef, "undef")) { - try bw.writeAll(line); - try bw.writeByte('\n'); - continue; - } - const name = it.next().?; - const index = values.getIndex(name) orelse { - try step.addError("{s}:{d}: error: unspecified config header value: '{s}'", .{ - src_path, line_index + 1, name, - }); - any_errors = true; - continue; - }; - is_used.set(index); - try renderValueC(bw, name, values.values()[index]); - } - - var unused_value_it = is_used.iterator(.{ .kind = .unset }); - while (unused_value_it.next()) |index| { - try step.addError("{s}: error: config header value unused: '{s}'", .{ src_path, values.keys()[index] }); - any_errors = true; - } - - if (any_errors) { - return error.MakeFailed; - } -} - -fn render_autoconf_at( - step: *Step, - contents: []const u8, - aw: *Writer.Allocating, - values: *const std.array_hash_map.String(Value), - src_path: []const u8, -) !void { - const build = step.owner; - const allocator = build.allocator; - const bw = &aw.writer; - - const used = allocator.alloc(bool, values.count()) catch @panic("OOM"); - for (used) |*u| u.* = false; - defer allocator.free(used); - - var any_errors = false; - var line_index: u32 = 0; - var line_it = std.mem.splitScalar(u8, contents, '\n'); - while (line_it.next()) |line| : (line_index += 1) { - const last_line = line_it.index == line_it.buffer.len; - - const old_len = aw.written().len; - expand_variables_autoconf_at(bw, line, values, used) catch |err| switch (err) { - error.MissingValue => { - const name = aw.written()[old_len..]; - defer aw.shrinkRetainingCapacity(old_len); - try step.addError("{s}:{d}: error: unspecified config header value: '{s}'", .{ - src_path, line_index + 1, name, - }); - any_errors = true; - continue; - }, - else => { - try step.addError("{s}:{d}: unable to substitute variable: error: {s}", .{ - src_path, line_index + 1, @errorName(err), - }); - any_errors = true; - continue; - }, - }; - if (!last_line) try bw.writeByte('\n'); - } - - for (values.entries.slice().items(.key), used) |name, u| { - if (!u) { - try step.addError("{s}: error: config header value unused: '{s}'", .{ src_path, name }); - any_errors = true; - } - } - - if (any_errors) return error.MakeFailed; -} - -fn render_cmake( - step: *Step, - contents: []const u8, - bw: *Writer, - values: std.array_hash_map.String(Value), - src_path: []const u8, -) !void { - const build = step.owner; - const allocator = build.allocator; - - var values_copy = try values.clone(allocator); - defer values_copy.deinit(allocator); - - var any_errors = false; - var line_index: u32 = 0; - var line_it = std.mem.splitScalar(u8, contents, '\n'); - while (line_it.next()) |raw_line| : (line_index += 1) { - const last_line = line_it.index == line_it.buffer.len; - - const line = expand_variables_cmake(allocator, raw_line, values) catch |err| switch (err) { - error.InvalidCharacter => { - try step.addError("{s}:{d}: error: invalid character in a variable name", .{ - src_path, line_index + 1, - }); - any_errors = true; - continue; - }, - else => { - try step.addError("{s}:{d}: unable to substitute variable: error: {s}", .{ - src_path, line_index + 1, @errorName(err), - }); - any_errors = true; - continue; - }, - }; - defer allocator.free(line); - - const line_start = std.mem.findNone(u8, line, " \t\r") orelse { - try bw.writeAll(line); - if (!last_line) try bw.writeByte('\n'); - continue; - }; - const whitespace_prefix = line[0..line_start]; - const trimmed_line = line[line_start..]; - - if (!std.mem.startsWith(u8, trimmed_line, "#")) { - try bw.writeAll(line); - if (!last_line) try bw.writeByte('\n'); - continue; - } - - var it = std.mem.tokenizeAny(u8, trimmed_line[1..], " \t\r"); - const cmakedefine = it.next().?; - if (!std.mem.eql(u8, cmakedefine, "cmakedefine") and - !std.mem.eql(u8, cmakedefine, "cmakedefine01")) - { - try bw.writeAll(line); - if (!last_line) try bw.writeByte('\n'); - continue; - } - - const booldefine = std.mem.eql(u8, cmakedefine, "cmakedefine01"); - - const name = it.next() orelse { - try step.addError("{s}:{d}: error: missing define name", .{ - src_path, line_index + 1, - }); - any_errors = true; - continue; - }; - var value = values_copy.get(name) orelse blk: { - if (booldefine) { - break :blk Value{ .int = 0 }; - } - break :blk Value.undef; - }; - - value = blk: { - switch (value) { - .boolean => |b| { - if (!b) { - break :blk Value.undef; - } - }, - .int => |i| { - if (i == 0) { - break :blk Value.undef; - } - }, - .string => |string| { - if (string.len == 0) { - break :blk Value.undef; - } - }, - - else => {}, - } - break :blk value; - }; - - if (booldefine) { - value = blk: { - switch (value) { - .undef => { - break :blk Value{ .boolean = false }; - }, - .defined => { - break :blk Value{ .boolean = false }; - }, - .boolean => |b| { - break :blk Value{ .boolean = b }; - }, - .int => |i| { - break :blk Value{ .boolean = i != 0 }; - }, - .string => |string| { - break :blk Value{ .boolean = string.len != 0 }; - }, - - else => { - break :blk Value{ .boolean = false }; - }, - } - }; - } else if (value != Value.undef) { - value = Value{ .ident = it.rest() }; - } - - try bw.writeAll(whitespace_prefix); - try renderValueC(bw, name, value); - } - - if (any_errors) { - return error.HeaderConfigFailed; +pub fn addValues(config_header: *ConfigHeader, values: anytype) void { + inline for (@typeInfo(@TypeOf(values)).@"struct".fields) |field| { + addValue(config_header, field.name, field.type, @field(values, field.name)); } } -fn render_blank( - gpa: std.mem.Allocator, - bw: *Writer, - defines: std.array_hash_map.String(Value), - include_path: []const u8, - include_guard_override: ?[]const u8, -) !void { - const include_guard_name = include_guard_override orelse blk: { - const name = try gpa.dupe(u8, include_path); - for (name) |*byte| { - switch (byte.*) { - 'a'...'z' => byte.* = byte.* - 'a' + 'A', - 'A'...'Z', '0'...'9' => continue, - else => byte.* = '_', - } - } - break :blk name; - }; - defer if (include_guard_override == null) gpa.free(include_guard_name); - - try bw.print( - \\#ifndef {[0]s} - \\#define {[0]s} - \\ - , .{include_guard_name}); - - const values = defines.values(); - for (defines.keys(), 0..) |name, i| try renderValueC(bw, name, values[i]); - - try bw.print( - \\#endif /* {s} */ - \\ - , .{include_guard_name}); -} - -fn render_nasm(bw: *Writer, defines: std.array_hash_map.String(Value)) !void { - for (defines.keys(), defines.values()) |name, value| try renderValueNasm(bw, name, value); +pub fn getOutputDir(ch: *ConfigHeader) std.Build.LazyPath { + return .{ .generated = .{ .index = ch.generated_dir } }; } - -fn renderValueC(bw: *Writer, name: []const u8, value: Value) !void { - switch (value) { - .undef => try bw.print("/* #undef {s} */\n", .{name}), - .defined => try bw.print("#define {s}\n", .{name}), - .boolean => |b| try bw.print("#define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), - .int => |i| try bw.print("#define {s} {d}\n", .{ name, i }), - .ident => |ident| try bw.print("#define {s} {s}\n", .{ name, ident }), - // TODO: use C-specific escaping instead of zig string literals - .string => |string| try bw.print("#define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), - } -} - -fn renderValueNasm(bw: *Writer, name: []const u8, value: Value) !void { - switch (value) { - .undef => try bw.print("; %undef {s}\n", .{name}), - .defined => try bw.print("%define {s}\n", .{name}), - .boolean => |b| try bw.print("%define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), - .int => |i| try bw.print("%define {s} {d}\n", .{ name, i }), - .ident => |ident| try bw.print("%define {s} {s}\n", .{ name, ident }), - // TODO: use nasm-specific escaping instead of zig string literals - .string => |string| try bw.print("%define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), - } -} - -fn expand_variables_autoconf_at( - bw: *Writer, - contents: []const u8, - values: *const std.array_hash_map.String(Value), - used: []bool, -) !void { - const valid_varname_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"; - - var curr: usize = 0; - var source_offset: usize = 0; - while (curr < contents.len) : (curr += 1) { - if (contents[curr] != '@') continue; - if (std.mem.findScalarPos(u8, contents, curr + 1, '@')) |close_pos| { - if (close_pos == curr + 1) { - // closed immediately, preserve as a literal - continue; - } - const valid_varname_end = std.mem.findNonePos(u8, contents, curr + 1, valid_varname_chars) orelse 0; - if (valid_varname_end != close_pos) { - // contains invalid characters, preserve as a literal - continue; - } - - const key = contents[curr + 1 .. close_pos]; - const index = values.getIndex(key) orelse { - // Report the missing key to the caller. - try bw.writeAll(key); - return error.MissingValue; - }; - const value = values.entries.slice().items(.value)[index]; - used[index] = true; - try bw.writeAll(contents[source_offset..curr]); - switch (value) { - .undef, .defined => {}, - .boolean => |b| try bw.writeByte(@as(u8, '0') + @intFromBool(b)), - .int => |i| try bw.print("{d}", .{i}), - .ident, .string => |s| try bw.writeAll(s), - } - - curr = close_pos; - source_offset = close_pos + 1; - } - } - - try bw.writeAll(contents[source_offset..]); -} - -fn expand_variables_cmake( - allocator: Allocator, - contents: []const u8, - values: std.array_hash_map.String(Value), -) ![]const u8 { - var result: std.array_list.Managed(u8) = .init(allocator); - errdefer result.deinit(); - - const valid_varname_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789/_.+-"; - const open_var = "${"; - - var curr: usize = 0; - var source_offset: usize = 0; - const Position = struct { - source: usize, - target: usize, - }; - var var_stack: std.array_list.Managed(Position) = .init(allocator); - defer var_stack.deinit(); - loop: while (curr < contents.len) : (curr += 1) { - switch (contents[curr]) { - '@' => blk: { - if (std.mem.findScalarPos(u8, contents, curr + 1, '@')) |close_pos| { - if (close_pos == curr + 1) { - // closed immediately, preserve as a literal - break :blk; - } - const valid_varname_end = std.mem.findNonePos(u8, contents, curr + 1, valid_varname_chars) orelse 0; - if (valid_varname_end != close_pos) { - // contains invalid characters, preserve as a literal - break :blk; - } - - const key = contents[curr + 1 .. close_pos]; - const value = values.get(key) orelse return error.MissingValue; - const missing = contents[source_offset..curr]; - try result.appendSlice(missing); - switch (value) { - .undef, .defined => {}, - .boolean => |b| { - try result.append(if (b) '1' else '0'); - }, - .int => |i| { - try result.print("{d}", .{i}); - }, - .ident, .string => |s| { - try result.appendSlice(s); - }, - } - - curr = close_pos; - source_offset = close_pos + 1; - - continue :loop; - } - }, - '$' => blk: { - const next = curr + 1; - if (next == contents.len or contents[next] != '{') { - // no open bracket detected, preserve as a literal - break :blk; - } - const missing = contents[source_offset..curr]; - try result.appendSlice(missing); - try result.appendSlice(open_var); - - source_offset = curr + open_var.len; - curr = next; - try var_stack.append(Position{ - .source = curr, - .target = result.items.len - open_var.len, - }); - - continue :loop; - }, - '}' => blk: { - if (var_stack.items.len == 0) { - // no open bracket, preserve as a literal - break :blk; - } - const open_pos = var_stack.pop().?; - if (source_offset == open_pos.source) { - source_offset += open_var.len; - } - const missing = contents[source_offset..curr]; - try result.appendSlice(missing); - - const key_start = open_pos.target + open_var.len; - const key = result.items[key_start..]; - if (key.len == 0) { - return error.MissingKey; - } - const value = values.get(key) orelse return error.MissingValue; - result.shrinkRetainingCapacity(result.items.len - key.len - open_var.len); - switch (value) { - .undef, .defined => {}, - .boolean => |b| { - try result.append(if (b) '1' else '0'); - }, - .int => |i| { - try result.print("{d}", .{i}); - }, - .ident, .string => |s| { - try result.appendSlice(s); - }, - } - - source_offset = curr + 1; - - continue :loop; - }, - '\\' => { - // backslash is not considered a special character - continue :loop; - }, - else => {}, - } - - if (var_stack.items.len > 0 and std.mem.findScalar(u8, valid_varname_chars, contents[curr]) == null) { - return error.InvalidCharacter; - } - } - - if (source_offset != contents.len) { - const missing = contents[source_offset..]; - try result.appendSlice(missing); - } - - return result.toOwnedSlice(); -} - -fn testReplaceVariablesAutoconfAt( - allocator: Allocator, - contents: []const u8, - expected: []const u8, - values: std.array_hash_map.String(Value), -) !void { - var aw: Writer.Allocating = .init(allocator); - defer aw.deinit(); - - const used = try allocator.alloc(bool, values.count()); - for (used) |*u| u.* = false; - defer allocator.free(used); - - try expand_variables_autoconf_at(&aw.writer, contents, values, used); - - for (used) |u| if (!u) return error.UnusedValue; - try std.testing.expectEqualStrings(expected, aw.written()); -} - -fn testReplaceVariablesCMake( - allocator: Allocator, - contents: []const u8, - expected: []const u8, - values: std.array_hash_map.String(Value), -) !void { - const actual = try expand_variables_cmake(allocator, contents, values); - defer allocator.free(actual); - - try std.testing.expectEqualStrings(expected, actual); -} - -test "expand_variables_autoconf_at simple cases" { - const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); - - // empty strings are preserved - try testReplaceVariablesAutoconfAt(allocator, "", "", values); - - // line with misc content is preserved - try testReplaceVariablesAutoconfAt(allocator, "no substitution", "no substitution", values); - - // empty @ sigils are preserved - try testReplaceVariablesAutoconfAt(allocator, "@", "@", values); - try testReplaceVariablesAutoconfAt(allocator, "@@", "@@", values); - try testReplaceVariablesAutoconfAt(allocator, "@@@", "@@@", values); - try testReplaceVariablesAutoconfAt(allocator, "@@@@", "@@@@", values); - - // simple substitution - try values.putNoClobber("undef", .undef); - try testReplaceVariablesAutoconfAt(allocator, "@undef@", "", values); - values.clearRetainingCapacity(); - - try values.putNoClobber("defined", .defined); - try testReplaceVariablesAutoconfAt(allocator, "@defined@", "", values); - values.clearRetainingCapacity(); - - try values.putNoClobber("true", Value{ .boolean = true }); - try testReplaceVariablesAutoconfAt(allocator, "@true@", "1", values); - values.clearRetainingCapacity(); - - try values.putNoClobber("false", Value{ .boolean = false }); - try testReplaceVariablesAutoconfAt(allocator, "@false@", "0", values); - values.clearRetainingCapacity(); - - try values.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@", "42", values); - values.clearRetainingCapacity(); - - try values.putNoClobber("ident", Value{ .string = "value" }); - try testReplaceVariablesAutoconfAt(allocator, "@ident@", "value", values); - values.clearRetainingCapacity(); - - try values.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@", "text", values); - values.clearRetainingCapacity(); - - // double packed substitution - try values.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@@string@", "texttext", values); - values.clearRetainingCapacity(); - - // triple packed substitution - try values.putNoClobber("int", Value{ .int = 42 }); - try values.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@@int@@string@", "text42text", values); - values.clearRetainingCapacity(); - - // double separated substitution - try values.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@.@int@", "42.42", values); - values.clearRetainingCapacity(); - - // triple separated substitution - try values.putNoClobber("true", Value{ .boolean = true }); - try values.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@.@true@.@int@", "42.1.42", values); - values.clearRetainingCapacity(); - - // misc prefix is preserved - try values.putNoClobber("false", Value{ .boolean = false }); - try testReplaceVariablesAutoconfAt(allocator, "false is @false@", "false is 0", values); - values.clearRetainingCapacity(); - - // misc suffix is preserved - try values.putNoClobber("true", Value{ .boolean = true }); - try testReplaceVariablesAutoconfAt(allocator, "@true@ is true", "1 is true", values); - values.clearRetainingCapacity(); - - // surrounding content is preserved - try values.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", values); - values.clearRetainingCapacity(); - - // incomplete key is preserved - try testReplaceVariablesAutoconfAt(allocator, "@undef", "@undef", values); - - // unknown key leads to an error - try std.testing.expectError(error.MissingValue, testReplaceVariablesAutoconfAt(allocator, "@bad@", "", values)); - - // unused key leads to an error - try values.putNoClobber("int", Value{ .int = 42 }); - try values.putNoClobber("false", Value{ .boolean = false }); - try std.testing.expectError(error.UnusedValue, testReplaceVariablesAutoconfAt(allocator, "@int", "", values)); - values.clearRetainingCapacity(); -} - -test "expand_variables_autoconf_at edge cases" { - const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); - - // @-vars resolved only when they wrap valid characters, otherwise considered literals - try values.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@@string@@", "@text@", values); - values.clearRetainingCapacity(); - - // expanded variables are considered strings after expansion - try values.putNoClobber("string_at", Value{ .string = "@string@" }); - try testReplaceVariablesAutoconfAt(allocator, "@string_at@", "@string@", values); - values.clearRetainingCapacity(); -} - -test "expand_variables_cmake simple cases" { - const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); - - try values.putNoClobber("undef", .undef); - try values.putNoClobber("defined", .defined); - try values.putNoClobber("true", Value{ .boolean = true }); - try values.putNoClobber("false", Value{ .boolean = false }); - try values.putNoClobber("int", Value{ .int = 42 }); - try values.putNoClobber("ident", Value{ .string = "value" }); - try values.putNoClobber("string", Value{ .string = "text" }); - - // empty strings are preserved - try testReplaceVariablesCMake(allocator, "", "", values); - - // line with misc content is preserved - try testReplaceVariablesCMake(allocator, "no substitution", "no substitution", values); - - // empty ${} wrapper leads to an error - try std.testing.expectError(error.MissingKey, testReplaceVariablesCMake(allocator, "${}", "", values)); - - // empty @ sigils are preserved - try testReplaceVariablesCMake(allocator, "@", "@", values); - try testReplaceVariablesCMake(allocator, "@@", "@@", values); - try testReplaceVariablesCMake(allocator, "@@@", "@@@", values); - try testReplaceVariablesCMake(allocator, "@@@@", "@@@@", values); - - // simple substitution - try testReplaceVariablesCMake(allocator, "@undef@", "", values); - try testReplaceVariablesCMake(allocator, "${undef}", "", values); - try testReplaceVariablesCMake(allocator, "@defined@", "", values); - try testReplaceVariablesCMake(allocator, "${defined}", "", values); - try testReplaceVariablesCMake(allocator, "@true@", "1", values); - try testReplaceVariablesCMake(allocator, "${true}", "1", values); - try testReplaceVariablesCMake(allocator, "@false@", "0", values); - try testReplaceVariablesCMake(allocator, "${false}", "0", values); - try testReplaceVariablesCMake(allocator, "@int@", "42", values); - try testReplaceVariablesCMake(allocator, "${int}", "42", values); - try testReplaceVariablesCMake(allocator, "@ident@", "value", values); - try testReplaceVariablesCMake(allocator, "${ident}", "value", values); - try testReplaceVariablesCMake(allocator, "@string@", "text", values); - try testReplaceVariablesCMake(allocator, "${string}", "text", values); - - // double packed substitution - try testReplaceVariablesCMake(allocator, "@string@@string@", "texttext", values); - try testReplaceVariablesCMake(allocator, "${string}${string}", "texttext", values); - - // triple packed substitution - try testReplaceVariablesCMake(allocator, "@string@@int@@string@", "text42text", values); - try testReplaceVariablesCMake(allocator, "@string@${int}@string@", "text42text", values); - try testReplaceVariablesCMake(allocator, "${string}@int@${string}", "text42text", values); - try testReplaceVariablesCMake(allocator, "${string}${int}${string}", "text42text", values); - - // double separated substitution - try testReplaceVariablesCMake(allocator, "@int@.@int@", "42.42", values); - try testReplaceVariablesCMake(allocator, "${int}.${int}", "42.42", values); - - // triple separated substitution - try testReplaceVariablesCMake(allocator, "@int@.@true@.@int@", "42.1.42", values); - try testReplaceVariablesCMake(allocator, "@int@.${true}.@int@", "42.1.42", values); - try testReplaceVariablesCMake(allocator, "${int}.@true@.${int}", "42.1.42", values); - try testReplaceVariablesCMake(allocator, "${int}.${true}.${int}", "42.1.42", values); - - // misc prefix is preserved - try testReplaceVariablesCMake(allocator, "false is @false@", "false is 0", values); - try testReplaceVariablesCMake(allocator, "false is ${false}", "false is 0", values); - - // misc suffix is preserved - try testReplaceVariablesCMake(allocator, "@true@ is true", "1 is true", values); - try testReplaceVariablesCMake(allocator, "${true} is true", "1 is true", values); - - // surrounding content is preserved - try testReplaceVariablesCMake(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", values); - try testReplaceVariablesCMake(allocator, "what is 6*7? ${int}!", "what is 6*7? 42!", values); - - // incomplete key is preserved - try testReplaceVariablesCMake(allocator, "@undef", "@undef", values); - try testReplaceVariablesCMake(allocator, "${undef", "${undef", values); - try testReplaceVariablesCMake(allocator, "{undef}", "{undef}", values); - try testReplaceVariablesCMake(allocator, "undef@", "undef@", values); - try testReplaceVariablesCMake(allocator, "undef}", "undef}", values); - - // unknown key leads to an error - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@bad@", "", values)); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${bad}", "", values)); -} - -test "expand_variables_cmake edge cases" { - const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); - - // special symbols - try values.putNoClobber("at", Value{ .string = "@" }); - try values.putNoClobber("dollar", Value{ .string = "$" }); - try values.putNoClobber("underscore", Value{ .string = "_" }); - - // basic value - try values.putNoClobber("string", Value{ .string = "text" }); - - // proxy case values - try values.putNoClobber("string_proxy", Value{ .string = "string" }); - try values.putNoClobber("string_at", Value{ .string = "@string@" }); - try values.putNoClobber("string_curly", Value{ .string = "{string}" }); - try values.putNoClobber("string_var", Value{ .string = "${string}" }); - - // stack case values - try values.putNoClobber("nest_underscore_proxy", Value{ .string = "underscore" }); - try values.putNoClobber("nest_proxy", Value{ .string = "nest_underscore_proxy" }); - - // @-vars resolved only when they wrap valid characters, otherwise considered literals - try testReplaceVariablesCMake(allocator, "@@string@@", "@text@", values); - try testReplaceVariablesCMake(allocator, "@${string}@", "@text@", values); - - // @-vars are resolved inside ${}-vars - try testReplaceVariablesCMake(allocator, "${@string_proxy@}", "text", values); - - // expanded variables are considered strings after expansion - try testReplaceVariablesCMake(allocator, "@string_at@", "@string@", values); - try testReplaceVariablesCMake(allocator, "${string_at}", "@string@", values); - try testReplaceVariablesCMake(allocator, "$@string_curly@", "${string}", values); - try testReplaceVariablesCMake(allocator, "$${string_curly}", "${string}", values); - try testReplaceVariablesCMake(allocator, "${string_var}", "${string}", values); - try testReplaceVariablesCMake(allocator, "@string_var@", "${string}", values); - try testReplaceVariablesCMake(allocator, "${dollar}{${string}}", "${text}", values); - try testReplaceVariablesCMake(allocator, "@dollar@{${string}}", "${text}", values); - try testReplaceVariablesCMake(allocator, "@dollar@{@string@}", "${text}", values); - - // when expanded variables contain invalid characters, they prevent further expansion - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${${string_var}}", "", values)); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${@string_var@}", "", values)); - - // nested expanded variables are expanded from the inside out - try testReplaceVariablesCMake(allocator, "${string${underscore}proxy}", "string", values); - try testReplaceVariablesCMake(allocator, "${string@underscore@proxy}", "string", values); - - // nested vars are only expanded when ${} is closed - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@underscore@proxy@", "", values)); - try testReplaceVariablesCMake(allocator, "${nest${underscore}proxy}", "nest_underscore_proxy", values); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@@nest_underscore@underscore@proxy@@proxy@", "", values)); - try testReplaceVariablesCMake(allocator, "${nest${${nest_underscore${underscore}proxy}}proxy}", "nest_underscore_proxy", values); - - // invalid characters lead to an error - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str*ing}", "", values)); - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str$ing}", "", values)); - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str@ing}", "", values)); -} - -test "expand_variables_cmake escaped characters" { - const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); - - try values.putNoClobber("string", Value{ .string = "text" }); - - // backslash is an invalid character for @ lookup - try testReplaceVariablesCMake(allocator, "\\@string\\@", "\\@string\\@", values); - - // backslash is preserved, but doesn't affect ${} variable expansion - try testReplaceVariablesCMake(allocator, "\\${string}", "\\text", values); - - // backslash breaks ${} opening bracket identification - try testReplaceVariablesCMake(allocator, "$\\{string}", "$\\{string}", values); - - // backslash is skipped when checking for invalid characters, yet it mangles the key - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${string\\}", "", values)); +pub fn getOutputFile(ch: *ConfigHeader) std.Build.LazyPath { + return ch.getOutputDir().path(ch.step.owner, ch.include_path); } diff --git a/lib/std/Build/Step/Fail.zig b/lib/std/Build/Step/Fail.zig index 50c7ed2789fb2b81434716cd4687c38eb0cb5a3c..b51ad51adc966b4bb2d28c363996f0c89356ba91 100644 --- a/lib/std/Build/Step/Fail.zig +++ b/lib/std/Build/Step/Fail.zig @@ -1,35 +1,25 @@ //! Fail the build with a given message. +const Fail = @This(); + const std = @import("std"); const Step = std.Build.Step; -const Fail = @This(); +const Configuration = std.Build.Configuration; step: Step, -error_msg: []const u8, +error_msg: Configuration.String, pub const base_tag: Step.Tag = .fail; pub fn create(owner: *std.Build, error_msg: []const u8) *Fail { - const fail = owner.allocator.create(Fail) catch @panic("OOM"); - + const graph = owner.graph; + const fail = graph.create(Fail); fail.* = .{ - .step = Step.init(.{ + .step = .init(.{ .tag = base_tag, .name = "fail", .owner = owner, - .makeFn = make, }), - .error_msg = owner.dupe(error_msg), + .error_msg = graph.addString(error_msg), }; - return fail; } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; // No progress to report. - - const fail: *Fail = @fieldParentPtr("step", step); - - try step.result_error_msgs.append(step.owner.allocator, fail.error_msg); - - return error.MakeFailed; -} diff --git a/test/standalone/libfuzzer/build.zig b/test/standalone/libfuzzer/build.zig index ea3ae12e454c98b1148117a9b11b0e757cea9918..85806f98e35109435170905e9a779d725f107a46 100644 --- a/test/standalone/libfuzzer/build.zig +++ b/test/standalone/libfuzzer/build.zig @@ -24,6 +24,6 @@ pub fn build(b: *std.Build) void { b.default_step = run_step; const run_artifact = b.addRunArtifact(exe); - run_artifact.addArg(b.cache_root.path orelse ""); + run_artifact.addFileArg(.cache_root); run_step.dependOn(&run_artifact.step); } diff --git a/test/standalone/windows_resources/build.zig b/test/standalone/windows_resources/build.zig index 3b140c90f79e650cc9fb4b944ac1452ac9996c97..8cd410917d9673137fbfa65bb34694ab132b512c 100644 --- a/test/standalone/windows_resources/build.zig +++ b/test/standalone/windows_resources/build.zig @@ -38,7 +38,7 @@ fn add( .file = b.path("res/zig.rc"), .flags = &.{"/c65001"}, // UTF-8 code page .include_paths = &.{ - .{ .generated = .{ .file = &generated_h_step.generated_directory } }, + .{ .generated = .{ .index = generated_h_step.generated_directory } }, }, }); exe.rc_includes = switch (rc_includes) { diff --git a/test/tests.zig b/test/tests.zig index 8591de270ed2773be1ae6c907eccfdc2ec7097d1..d7b74073902989f873ac038ac1ca0f6be4968e18 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2433,8 +2433,10 @@ pub fn addCliTests(b: *std.Build) *Step { }); run_test.addArg("--build-file"); run_test.addFileArg(b.path("test/cli/options/build.zig")); + run_test.addArg("--cache-dir"); - run_test.addFileArg(.{ .cwd_relative = b.cache_root.join(b.allocator, &.{}) catch @panic("OOM") }); + run_test.addFileArg(.cache_root); + run_test.setName("test build options"); step.dependOn(&run_test.step); @@ -2966,10 +2968,11 @@ pub fn addIncrementalTests(b: *std.Build, test_step: *Step, test_filters: []cons run.addArg(b.graph.zig_exe); run.addFileArg(b.path("test/incremental/").path(b, entry.path)); - run.addArgs(&.{ - "--zig-lib-dir", b.graph.zig_lib_directory.path orelse ".", - "--target", target_str, - }); + + run.addArg("--zig-lib-dir"); + run.addFileArg(.zig_lib); + + run.addArgs(&.{ "--target", target_str }); run.addArg("--quiet"); // don't fill stderr telling us about skipped tests etc -- 2.54.0 From affe5ed867009c889f2df7b624387c2bceefcc0e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 6 May 2026 23:17:34 -0700 Subject: [PATCH 085/179] std.Build: port UpdateSourceFiles step to new system --- BRANCH_TODO | 2 + lib/compiler/Maker/Step.zig | 7 +- lib/compiler/Maker/Step/ObjCopy.zig | 142 ++++++++++++++ lib/compiler/Maker/Step/UpdateSourceFiles.zig | 87 +++++++++ lib/compiler/configurer.zig | 2 +- lib/std/Build/Configuration.zig | 26 ++- lib/std/Build/Step.zig | 2 +- lib/std/Build/Step/ObjCopy.zig | 179 +++--------------- lib/std/Build/Step/Run.zig | 4 +- lib/std/Build/Step/TranslateC.zig | 2 +- lib/std/Build/Step/UpdateSourceFiles.zig | 49 +---- 11 files changed, 293 insertions(+), 209 deletions(-) create mode 100644 lib/compiler/Maker/Step/ObjCopy.zig create mode 100644 lib/compiler/Maker/Step/UpdateSourceFiles.zig diff --git a/BRANCH_TODO b/BRANCH_TODO index 846ce64fe6aa5416b2f854a9882ce65aa5dfdc46..616f7e81471c98be3954c3e6f7ee0e2968b66870 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -28,6 +28,8 @@ * no more "artifact arg" to run step. if you want to run the post-install binary, get the lazy path from the install step. * fmt step: import zig fmt code directly rather than child proc +* UpdateSourceFiles: introduce Group +* WriteFiles: introduce Group ## Already Filed Followup Issues * build system fmt step with check=false does not acquire a write lock on source files #35204 diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index edc7dd7edfff9d21b1109c3dc90ef65423dc987c..5404da509622d35f6ab6edcfe5aea9b2876e0f12 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -22,6 +22,7 @@ pub const Compile = @import("Step/Compile.zig"); pub const Run = @import("Step/Run.zig"); pub const InstallArtifact = @import("Step/InstallArtifact.zig"); pub const InstallFile = @import("Step/InstallFile.zig"); +pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); /// Avoid false sharing. _: void align(std.atomic.cache_line) = {}, @@ -75,13 +76,13 @@ pub const Extended = union(enum) { install_artifact: InstallArtifact, install_dir: Todo, install_file: InstallFile, - objcopy: Todo, + obj_copy: Todo, options: Todo, remove_dir: Todo, run: Run, top_level: TopLevel, translate_c: Todo, - update_source_files: Todo, + update_source_files: UpdateSourceFiles, write_file: Todo, pub fn init(tag: Configuration.Step.Tag) Extended { @@ -95,7 +96,7 @@ pub const Extended = union(enum) { .install_artifact => .{ .install_artifact = .{} }, .install_dir => .{ .install_dir = .{} }, .install_file => .{ .install_file = .{} }, - .objcopy => .{ .objcopy = .{} }, + .obj_copy => .{ .obj_copy = .{} }, .options => .{ .options = .{} }, .remove_dir => .{ .remove_dir = .{} }, .run => .{ .run = .{} }, diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig new file mode 100644 index 0000000000000000000000000000000000000000..517f58ce5ac1c2774de23a154863c202982df2ff --- /dev/null +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -0,0 +1,142 @@ +const ObjCopy = @This(); + +const std = @import("std"); +const Io = std.Io; +const allocPrint = std.fmt.allocPrint; +const Configuration = std.Build.Configuration; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + obj_copy: *ObjCopy, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + _ = obj_copy; + const graph = maker.graph; + const arena = maker.graph.arena; // TODO don't leak into process arena + const io = graph.io; + const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_oc = conf_step.extended.get(conf.extra).obj_copy; + const cache_root = graph.local_cache_root; + + try step.singleUnchangingWatchInput(maker, arena, conf_oc.input_file); + + var man = graph.cache.obtain(); + defer man.deinit(); + + const src_path = try maker.resolveLazyPathIndex(arena, conf_oc.input_file, step_index); + _ = try man.addFilePath(src_path, null); + man.hash.addOptionalBytes(conf_oc.only_section); + man.hash.addOptional(conf_oc.pad_to); + man.hash.addOptional(conf_oc.format); + man.hash.add(conf_oc.compress_debug); + man.hash.add(conf_oc.strip); + man.hash.add(conf_oc.output_file_debug != null); + + if (try step.cacheHit(&man)) { + // Cache hit, skip subprocess execution. + const digest = man.final(); + conf_oc.output_file.path = try cache_root.join(arena, &.{ + "o", &digest, conf_oc.basename, + }); + if (conf_oc.output_file_debug) |*file| { + file.path = try cache_root.join(arena, &.{ + "o", &digest, try allocPrint(arena, "{s}.debug", .{conf_oc.basename}), + }); + } + return; + } + + const digest = man.final(); + const cache_path = "o" ++ Io.Dir.path.sep_str ++ digest; + const full_dest_path = try cache_root.join(arena, &.{ cache_path, conf_oc.basename }); + const full_dest_path_debug = try cache_root.join(arena, &.{ + cache_path, try allocPrint(arena, "{s}.debug", .{conf_oc.basename}), + }); + cache_root.handle.createDirPath(io, cache_path) catch |err| + return step.fail("unable to make path {s}: {t}", .{ cache_path, err }); + + var argv: std.ArrayList([]const u8) = .empty; + try argv.ensureUnusedCapacity(arena, 11); + + argv.addManyAsArrayAssumeCapacity(2).* = .{ graph.zig_exe, "objcopy" }; + + if (conf_oc.only_section) |only_section| + argv.addManyAsArrayAssumeCapacity(2).* = .{ "-j", only_section }; + + switch (conf_oc.strip) { + .none => {}, + .debug => argv.appendAssumeCapacity("--strip-debug"), + .debug_and_symbols => argv.appendAssumeCapacity("--strip-all"), + } + + if (conf_oc.pad_to) |pad_to| { + argv.addManyAsArrayAssumeCapacity(2).* = .{ + "--pad-to", try allocPrint(arena, "{d}", .{pad_to}), + }; + } + + if (conf_oc.format) |format| { + argv.addManyAsArrayAssumeCapacity(2).* = .{ + "-O", + switch (format) { + .bin => "binary", + .hex => "hex", + .elf => "elf", + }, + }; + } + + if (conf_oc.compress_debug) + argv.appendAssumeCapacity("--compress-debug-sections"); + + if (conf_oc.output_file_debug != null) + argv.appendAssumeCapacity(try allocPrint(arena, "--extract-to={s}", .{full_dest_path_debug})); + + try argv.ensureUnusedCapacity(arena, 9); + + if (conf_oc.add_section) |section| { + argv.appendAssumeCapacity("--add-section"); + argv.appendAssumeCapacity(try allocPrint(arena, "{s}={f}", .{ + section.section_name, try maker.resolveLazyPathIndex(arena, section.file_path, step_index), + })); + } + + if (conf_oc.set_section_alignment) |set_align| { + argv.appendAssumeCapacity("--set-section-alignment"); + argv.appendAssumeCapacity(try allocPrint(arena, "{s}={d}", .{ set_align.section_name, set_align.alignment })); + } + + if (conf_oc.set_section_flags) |set_flags| { + const f = set_flags.flags; + // trailing comma is allowed + argv.appendAssumeCapacity("--set-section-flags"); + argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{ + set_flags.section_name, + if (f.alloc) "alloc," else "", + if (f.contents) "contents," else "", + if (f.load) "load," else "", + if (f.readonly) "readonly," else "", + if (f.code) "code," else "", + if (f.exclude) "exclude," else "", + if (f.large) "large," else "", + if (f.merge) "merge," else "", + if (f.strings) "strings," else "", + })); + } + + argv.appendAssumeCapacity(src_path); + argv.appendAssumeCapacity(full_dest_path); + + argv.appendAssumeCapacity("--listen=-"); + _ = try Step.evalZigProcess(step_index, maker, argv.items, progress_node, false); + + conf_oc.output_file.path = full_dest_path; + if (conf_oc.output_file_debug) |*file| file.path = full_dest_path_debug; + try man.writeManifest(); +} diff --git a/lib/compiler/Maker/Step/UpdateSourceFiles.zig b/lib/compiler/Maker/Step/UpdateSourceFiles.zig new file mode 100644 index 0000000000000000000000000000000000000000..779155cbab649a3acc6410b55bbaf62b2b159fe3 --- /dev/null +++ b/lib/compiler/Maker/Step/UpdateSourceFiles.zig @@ -0,0 +1,87 @@ +const UpdateSourceFiles = @This(); + +const std = @import("std"); +const Io = std.Io; +const Path = std.Build.Cache.Path; +const allocPrint = std.fmt.allocPrint; +const Configuration = std.Build.Configuration; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + usf: *UpdateSourceFiles, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + _ = usf; + const graph = maker.graph; + const arena = maker.graph.arena; // TODO don't leak into process arena + const io = graph.io; + const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_usf = conf_step.extended.get(conf.extra).update_source_files; + const build_root = graph.build_root_directory; + + if (conf_step.owner != .root) + return step.fail(maker, "non-root package attempted to update its source files", .{}); + + var any_miss = false; + + progress_node.setEstimatedTotalItems(conf_usf.embeds.slice.len + conf_usf.copies.slice.len); + + for (conf_usf.embeds.slice) |*embed| { + const dest_path: Path = .{ + .root_dir = build_root, + .sub_path = embed.dest_path.slice(conf), + }; + if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| { + const dirname_path: Path = .{ + .root_dir = build_root, + .sub_path = dirname, + }; + dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| + return step.fail(maker, "failed to create path {f}: {t}", .{ dirname_path, err }); + } + dest_path.root_dir.handle.writeFile(io, .{ + .sub_path = dest_path.sub_path, + .data = embed.bytes.slice(conf), + }) catch |err| return step.fail(maker, "failed to write file {f}: {t}", .{ dest_path, err }); + any_miss = true; + progress_node.completeOne(); + } + + for (conf_usf.copies.slice) |*copy| { + const dest_path: Path = .{ + .root_dir = build_root, + .sub_path = copy.dest_path.slice(conf), + }; + if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| { + const dirname_path: Path = .{ + .root_dir = build_root, + .sub_path = dirname, + }; + dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| + return step.fail(maker, "failed to create path {f}: {t}", .{ dirname_path, err }); + } + const src_lazy_path = copy.src_path.get(conf); + const source_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); + if (!step.inputs.populated()) try step.addWatchInput(maker, arena, src_lazy_path); + + const prev_status = source_path.root_dir.handle.updateFile( + io, + source_path.sub_path, + dest_path.root_dir.handle, + dest_path.sub_path, + .{}, + ) catch |err| return step.fail(maker, "unable to update file from {f} to {f}: {t}", .{ + source_path, dest_path, err, + }); + any_miss = any_miss or prev_status == .stale; + progress_node.completeOne(); + } + + step.result_cached = !any_miss; +} diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index d4086f0cfffa7af7768dfec2ebd48d82f000c0e7..4990478ac25d350817f470ba5d5c02ce570ccc83 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -967,7 +967,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .check_file => @panic("TODO"), .config_header => @panic("TODO"), - .objcopy => @panic("TODO"), + .obj_copy => @panic("TODO"), .options => @panic("TODO"), }, }); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 64a6113110fe40c568d2cd488a127f0d9a345cdb..82a215fc225723def25d676e22e5b6e35dd71cfc 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -456,7 +456,7 @@ pub const Step = extern struct { install_artifact: InstallArtifact, install_dir: InstallDir, install_file: InstallFile, - objcopy: Objcopy, + obj_copy: ObjCopy, options: Options, remove_dir: RemoveDir, run: Run, @@ -491,7 +491,7 @@ pub const Step = extern struct { install_artifact, install_dir, install_file, - objcopy, + obj_copy, options, remove_dir, run, @@ -1100,11 +1100,11 @@ pub const Step = extern struct { }; }; - pub const Objcopy = struct { + pub const ObjCopy = struct { flags: @This().Flags, pub const Flags = packed struct(u32) { - tag: Tag = .objcopy, + tag: Tag = .obj_copy, _: u27 = 0, }; }; @@ -1147,10 +1147,26 @@ pub const Step = extern struct { pub const UpdateSourceFiles = struct { flags: @This().Flags, + embeds: Storage.FlagLengthPrefixedList(.flags, .embeds, Embed), + copies: Storage.FlagLengthPrefixedList(.flags, .copies, Copy), + + pub const Embed = extern struct { + /// Relative to build root. + dest_path: String, + bytes: Bytes, + }; + + pub const Copy = extern struct { + /// Relative to build root. + dest_path: String, + src_path: LazyPath.Index, + }; pub const Flags = packed struct(u32) { tag: Tag = .update_source_files, - _: u27 = 0, + embeds: bool, + copies: bool, + _: u25 = 0, }; }; diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 7b8a04d66e923953d5fec8fec03e938e6e9f22ac..e115732d0346774d3ea059129a404d29d9375172 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -72,7 +72,7 @@ pub fn Type(comptime tag: Tag) type { .run => Run, .check_file => CheckFile, .config_header => ConfigHeader, - .objcopy => ObjCopy, + .obj_copy => ObjCopy, .options => Options, }; } diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index 94e6a405e84ee6d1ff26ba2a4f330c7f40f01ad3..65e58ffa06832ab6f9b36d36c6c19dd5db9b1a38 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -1,17 +1,26 @@ -const std = @import("std"); const ObjCopy = @This(); -const Allocator = std.mem.Allocator; -const ArenaAllocator = std.heap.ArenaAllocator; -const File = std.Io.File; -const InstallDir = std.Build.InstallDir; +const std = @import("std"); const Step = std.Build.Step; -const elf = std.elf; -const fs = std.fs; -const sort = std.sort; const Configuration = std.Build.Configuration; -pub const base_tag: Step.Tag = .objcopy; +step: Step, +input_file: std.Build.LazyPath, +basename: []const u8, +output_file: Configuration.GeneratedFileIndex, +output_file_debug: Configuration.OptionalGeneratedFileIndex, + +format: ?RawFormat, +only_section: ?[]const u8, +pad_to: ?u64, +strip: Strip, +compress_debug: bool, + +add_section: ?AddSection, +set_section_alignment: ?SetSectionAlignment, +set_section_flags: ?SetSectionFlags, + +pub const base_tag: Step.Tag = .obj_copy; pub const RawFormat = enum { bin, @@ -28,28 +37,20 @@ pub const Strip = enum { pub const SectionFlags = packed struct { /// add SHF_ALLOC alloc: bool = false, - /// if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing contents: bool = false, - /// if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing (same as contents) load: bool = false, - /// readonly: clear default SHF_WRITE flag readonly: bool = false, - /// add SHF_EXECINSTR code: bool = false, - /// add SHF_EXCLUDE exclude: bool = false, - /// add SHF_X86_64_LARGE. Fatal error if target is not x86_64 large: bool = false, - /// add SHF_MERGE merge: bool = false, - /// add SHF_STRINGS strings: bool = false, }; @@ -69,22 +70,6 @@ pub const SetSectionFlags = struct { flags: SectionFlags, }; -step: Step, -input_file: std.Build.LazyPath, -basename: []const u8, -output_file: Configuration.GeneratedFileIndex, -output_file_debug: Configuration.OptionalGeneratedFileIndex, - -format: ?RawFormat, -only_section: ?[]const u8, -pad_to: ?u64, -strip: Strip, -compress_debug: bool, - -add_section: ?AddSection, -set_section_alignment: ?SetSectionAlignment, -set_section_flags: ?SetSectionFlags, - pub const Options = struct { basename: ?[]const u8 = null, format: ?RawFormat = null, @@ -111,20 +96,19 @@ pub fn create( ) *ObjCopy { const graph = owner.graph; const arena = graph.arena; - - const objcopy = arena.create(ObjCopy) catch @panic("OOM"); - objcopy.* = ObjCopy{ - .step = Step.init(.{ + const obj_copy = graph.create(ObjCopy); + obj_copy.* = .{ + .step = .init(.{ .tag = base_tag, .name = owner.fmt("objcopy {f}", .{input_file.fmt(graph)}), .owner = owner, - .makeFn = make, }), .input_file = input_file, - .basename = options.basename orelse std.fmt.allocPrint("{f}", .{input_file.fmt(graph)}) catch @panic("OOM"), - .output_file = graph.addGeneratedFile(&objcopy.step), + .basename = options.basename orelse + std.fmt.allocPrint(arena, "{f}", .{input_file.fmt(graph)}) catch @panic("OOM"), + .output_file = graph.addGeneratedFile(&obj_copy.step), .output_file_debug = if (options.strip != .none and options.extract_to_separate_file) - .init(graph.addGeneratedFile(&objcopy.step)) + .init(graph.addGeneratedFile(&obj_copy.step)) else .none, .format = options.format, @@ -136,115 +120,14 @@ pub fn create( .set_section_alignment = options.set_section_alignment, .set_section_flags = options.set_section_flags, }; - input_file.addStepDependencies(&objcopy.step); - return objcopy; + input_file.addStepDependencies(&obj_copy.step); + return obj_copy; } -pub fn getOutput(objcopy: *const ObjCopy) std.Build.LazyPath { - return .{ .generated = .{ .index = objcopy.output_file } }; +pub fn getOutput(obj_copy: *const ObjCopy) std.Build.LazyPath { + return .{ .generated = .{ .index = obj_copy.output_file } }; } -pub fn getOutputSeparatedDebug(objcopy: *const ObjCopy) ?std.Build.LazyPath { - return if (objcopy.output_file_debug.unwrap()) |index| .{ .generated = .{ .index = index } } else null; -} - -fn make(step: *Step, options: Step.MakeOptions) !void { - const prog_node = options.progress_node; - const b = step.owner; - const io = b.graph.io; - const objcopy: *ObjCopy = @fieldParentPtr("step", step); - try step.singleUnchangingWatchInput(objcopy.input_file); - - var man = b.graph.cache.obtain(); - defer man.deinit(); - - const full_src_path = objcopy.input_file.getPath2(b, step); - _ = try man.addFile(full_src_path, null); - man.hash.addOptionalBytes(objcopy.only_section); - man.hash.addOptional(objcopy.pad_to); - man.hash.addOptional(objcopy.format); - man.hash.add(objcopy.compress_debug); - man.hash.add(objcopy.strip); - man.hash.add(objcopy.output_file_debug != null); - - if (try step.cacheHit(&man)) { - // Cache hit, skip subprocess execution. - const digest = man.final(); - objcopy.output_file.path = try b.cache_root.join(b.allocator, &.{ - "o", &digest, objcopy.basename, - }); - if (objcopy.output_file_debug) |*file| { - file.path = try b.cache_root.join(b.allocator, &.{ - "o", &digest, b.fmt("{s}.debug", .{objcopy.basename}), - }); - } - return; - } - - const digest = man.final(); - const cache_path = "o" ++ fs.path.sep_str ++ digest; - const full_dest_path = try b.cache_root.join(b.allocator, &.{ cache_path, objcopy.basename }); - const full_dest_path_debug = try b.cache_root.join(b.allocator, &.{ cache_path, b.fmt("{s}.debug", .{objcopy.basename}) }); - b.cache_root.handle.createDirPath(io, cache_path) catch |err| { - return step.fail("unable to make path {s}: {s}", .{ cache_path, @errorName(err) }); - }; - - var argv = std.array_list.Managed([]const u8).init(b.allocator); - try argv.appendSlice(&.{ b.graph.zig_exe, "objcopy" }); - - if (objcopy.only_section) |only_section| { - try argv.appendSlice(&.{ "-j", only_section }); - } - switch (objcopy.strip) { - .none => {}, - .debug => try argv.appendSlice(&.{"--strip-debug"}), - .debug_and_symbols => try argv.appendSlice(&.{"--strip-all"}), - } - if (objcopy.pad_to) |pad_to| { - try argv.appendSlice(&.{ "--pad-to", b.fmt("{d}", .{pad_to}) }); - } - if (objcopy.format) |format| switch (format) { - .bin => try argv.appendSlice(&.{ "-O", "binary" }), - .hex => try argv.appendSlice(&.{ "-O", "hex" }), - .elf => try argv.appendSlice(&.{ "-O", "elf" }), - }; - if (objcopy.compress_debug) { - try argv.appendSlice(&.{"--compress-debug-sections"}); - } - if (objcopy.output_file_debug != null) { - try argv.appendSlice(&.{b.fmt("--extract-to={s}", .{full_dest_path_debug})}); - } - if (objcopy.add_section) |section| { - try argv.append("--add-section"); - try argv.appendSlice(&.{b.fmt("{s}={s}", .{ section.section_name, section.file_path.getPath2(b, step) })}); - } - if (objcopy.set_section_alignment) |set_align| { - try argv.append("--set-section-alignment"); - try argv.appendSlice(&.{b.fmt("{s}={d}", .{ set_align.section_name, set_align.alignment })}); - } - if (objcopy.set_section_flags) |set_flags| { - const f = set_flags.flags; - // trailing comma is allowed - try argv.append("--set-section-flags"); - try argv.appendSlice(&.{b.fmt("{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{ - set_flags.section_name, - if (f.alloc) "alloc," else "", - if (f.contents) "contents," else "", - if (f.load) "load," else "", - if (f.readonly) "readonly," else "", - if (f.code) "code," else "", - if (f.exclude) "exclude," else "", - if (f.large) "large," else "", - if (f.merge) "merge," else "", - if (f.strings) "strings," else "", - })}); - } - - try argv.appendSlice(&.{ full_src_path, full_dest_path }); - - try argv.append("--listen=-"); - _ = try step.evalZigProcess(argv.items, prog_node, false, options.web_server, options.gpa); - objcopy.output_file.path = full_dest_path; - if (objcopy.output_file_debug) |*file| file.path = full_dest_path_debug; - try man.writeManifest(); +pub fn getOutputSeparatedDebug(obj_copy: *const ObjCopy) ?std.Build.LazyPath { + return if (obj_copy.output_file_debug.unwrap()) |index| .{ .generated = .{ .index = index } } else null; } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 1b9b519f9b70a338040c6917db083744f0cf32dd..35699051fbfd511c399cb7033e613abebd9cf9d5 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -421,7 +421,7 @@ pub fn addPrefixedOutputDirectoryArg( output.* = .{ .prefix = graph.dupeString(prefix), .basename = graph.dupeString(basename), - .generated_file = .{ .step = &run.step }, + .generated_file = graph.addGeneratedFile(&run.step), }; run.argv.append(arena, .{ .output_directory = output }) catch @panic("OOM"); @@ -429,7 +429,7 @@ pub fn addPrefixedOutputDirectoryArg( run.setName(std.fmt.allocPrint(arena, "{s} ({s})", .{ run.step.name, basename }) catch @panic("OOM")); } - return .{ .generated = .{ .file = &output.generated_file } }; + return .{ .generated = .{ .index = output.generated_file } }; } pub fn addDirectoryArg(run: *Run, lazy_directory: std.Build.LazyPath) void { diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index 0c0844d070f09ba8907f3a4af8f0e5e23a676818..28d94afe9234c5f29b90eeaa91ef5ef7dd7a0475 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -31,7 +31,7 @@ pub fn create(owner: *std.Build, options: Options) *TranslateC { const graph = owner.graph; const arena = graph.arena; const translate_c = arena.create(TranslateC) catch @panic("OOM"); - const source = options.root_source_file.dupe(owner); + const source = options.root_source_file.dupe(graph); translate_c.* = .{ .step = Step.init(.{ .tag = base_tag, diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig index b41cca59d9d82a6c959929101a4ebf203cc2cc1e..7615b5829b088c8213adb1928074c7e270c9e9cd 100644 --- a/lib/std/Build/Step/UpdateSourceFiles.zig +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -29,11 +29,10 @@ pub const Contents = union(enum) { pub fn create(owner: *std.Build) *UpdateSourceFiles { const usf = owner.allocator.create(UpdateSourceFiles) catch @panic("OOM"); usf.* = .{ - .step = Step.init(.{ + .step = .init(.{ .tag = base_tag, .name = "UpdateSourceFiles", .owner = owner, - .makeFn = make, }), .output_source_files = .empty, }; @@ -68,49 +67,3 @@ pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: [] .sub_path = sub_path, }) catch @panic("OOM"); } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const io = b.graph.io; - const usf: *UpdateSourceFiles = @fieldParentPtr("step", step); - - var any_miss = false; - for (usf.output_source_files.items) |output_source_file| { - if (fs.path.dirname(output_source_file.sub_path)) |dirname| { - b.build_root.handle.createDirPath(io, dirname) catch |err| { - return step.fail("unable to make path '{f}{s}': {t}", .{ b.build_root, dirname, err }); - }; - } - switch (output_source_file.contents) { - .bytes => |bytes| { - b.build_root.handle.writeFile(io, .{ .sub_path = output_source_file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{f}{s}': {t}", .{ - b.build_root, output_source_file.sub_path, err, - }); - }; - any_miss = true; - }, - .copy => |file_source| { - if (!step.inputs.populated()) try step.addWatchInput(file_source); - - const source_path = file_source.getPath2(b, step); - const prev_status = Io.Dir.updateFile( - .cwd(), - io, - source_path, - b.build_root.handle, - output_source_file.sub_path, - .{}, - ) catch |err| { - return step.fail("unable to update file from '{s}' to '{f}{s}': {t}", .{ - source_path, b.build_root, output_source_file.sub_path, err, - }); - }; - any_miss = any_miss or prev_status == .stale; - }, - } - } - - step.result_cached = !any_miss; -} -- 2.54.0 From 8a8bf5ad023451a22fd7abf438e6c7ee105ac6bf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 7 May 2026 15:32:02 -0700 Subject: [PATCH 086/179] maker: update ObjCopy to new system --- BRANCH_TODO | 1 + lib/compiler/Maker.zig | 5 +- lib/compiler/Maker/Step.zig | 10 +- lib/compiler/Maker/Step/ObjCopy.zig | 169 +++++++++++++++++----------- lib/std/Build/Configuration.zig | 99 +++++++++++++++- lib/std/Build/Step/ObjCopy.zig | 45 ++------ 6 files changed, 215 insertions(+), 114 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 616f7e81471c98be3954c3e6f7ee0e2968b66870..c137d947b47451eb44659a671d006b6ec175bc9a 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -20,6 +20,7 @@ - and adjust dependencyInner to not openDir() ## Followup Issues +* stop leaking into global process arena * reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make * link_eh_frame_hdr should be DefaultingBool * make --foo, --no-foo CLI args uniform (make them -f args instead) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index e8c409980087043a114cd3dc91131b727e53f9bb..b8bd2d0b7433091c84e04f051a1b31599c85bdc3 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -612,7 +612,7 @@ pub fn main(init: process.Init.Minimal) !void { step.state = .precheck_done; const deps = step_index.ptr(c).deps.slice(c); step.pending_deps = @intCast(deps.len); - step.reset(gpa); + step.reset(&maker); } continue :rebuild; }, @@ -1506,10 +1506,9 @@ fn constructGraphAndCheckForDependencyLoop( /// invalidated. pub fn invalidateResult(maker: *Maker, step: *Step) bool { if (step.state == .precheck_done) return false; - const gpa = maker.gpa; assert(step.pending_deps == 0); step.state = .precheck_done; - step.reset(gpa); + step.reset(maker); for (step.dependants.items) |dependant_index| { const dependant = maker.stepByIndex(dependant_index); _ = invalidateResult(maker, dependant); diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 5404da509622d35f6ab6edcfe5aea9b2876e0f12..9681c99814dc97caba3824e7ff13aef655471329 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -23,6 +23,7 @@ pub const Run = @import("Step/Run.zig"); pub const InstallArtifact = @import("Step/InstallArtifact.zig"); pub const InstallFile = @import("Step/InstallFile.zig"); pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); +pub const ObjCopy = @import("Step/ObjCopy.zig"); /// Avoid false sharing. _: void align(std.atomic.cache_line) = {}, @@ -76,7 +77,7 @@ pub const Extended = union(enum) { install_artifact: InstallArtifact, install_dir: Todo, install_file: InstallFile, - obj_copy: Todo, + obj_copy: ObjCopy, options: Todo, remove_dir: Todo, run: Run, @@ -306,8 +307,9 @@ pub fn make( } /// Prepares the step for being re-evaluated. -pub fn reset(step: *Step, gpa: Allocator) void { +pub fn reset(step: *Step, maker: *Maker) void { assert(step.state == .precheck_done); + const gpa = maker.gpa; if (step.result_failed_command) |cmd| gpa.free(cmd); @@ -318,7 +320,7 @@ pub fn reset(step: *Step, gpa: Allocator) void { step.result_peak_rss = 0; step.result_failed_command = null; step.test_results = .{}; - step.clearWatchInputs(); + step.clearWatchInputs(maker); step.result_error_bundle.deinit(gpa); step.result_error_bundle = std.zig.ErrorBundle.empty; @@ -732,7 +734,7 @@ fn failWithCacheError( pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { if (s.test_results.isSuccess()) { man.writeManifest() catch |err| { - try s.addError(maker, "unable to write cache manifest: {t}", .{err}); + try s.addError(maker, "failed writing cache manifest: {t}", .{err}); }; } } diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index 517f58ce5ac1c2774de23a154863c202982df2ff..208bd0953fb5735dab7034382cdb931070f4d653 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -2,6 +2,7 @@ const ObjCopy = @This(); const std = @import("std"); const Io = std.Io; +const Path = std.Build.Cache.Path; const allocPrint = std.fmt.allocPrint; const Configuration = std.Build.Configuration; @@ -23,120 +24,152 @@ pub fn make( const conf_step = step_index.ptr(conf); const conf_oc = conf_step.extended.get(conf.extra).obj_copy; const cache_root = graph.local_cache_root; + const input_lazy_path = conf_oc.input_file.get(conf); + const only_section: ?[]const u8 = if (conf_oc.only_section.value) |s| s.slice(conf) else null; + const opt_basename: ?[]const u8 = if (conf_oc.basename.value) |s| s.slice(conf) else null; + const opt_debug_basename: ?[]const u8 = if (conf_oc.debug_basename.value) |s| s.slice(conf) else null; - try step.singleUnchangingWatchInput(maker, arena, conf_oc.input_file); + try step.singleUnchangingWatchInput(maker, arena, input_lazy_path); var man = graph.cache.obtain(); defer man.deinit(); - const src_path = try maker.resolveLazyPathIndex(arena, conf_oc.input_file, step_index); - _ = try man.addFilePath(src_path, null); - man.hash.addOptionalBytes(conf_oc.only_section); - man.hash.addOptional(conf_oc.pad_to); - man.hash.addOptional(conf_oc.format); - man.hash.add(conf_oc.compress_debug); - man.hash.add(conf_oc.strip); - man.hash.add(conf_oc.output_file_debug != null); + const input_path = try maker.resolveLazyPath(arena, input_lazy_path, step_index); + _ = try man.addFilePath(input_path, null); + man.hash.addOptionalBytes(only_section); + man.hash.addOptionalBytes(opt_basename); + man.hash.addOptionalBytes(opt_debug_basename); + man.hash.addOptional(conf_oc.pad_to.value); + man.hash.add(conf_oc.flags.format); + man.hash.add(conf_oc.flags.compress_debug); + man.hash.add(conf_oc.flags.strip); + man.hash.add(conf_oc.debug_file.value != null); - if (try step.cacheHit(&man)) { + const basename = opt_basename orelse Io.Dir.path.basename(input_path.sub_path); + + if (try step.cacheHit(maker, &man)) { // Cache hit, skip subprocess execution. const digest = man.final(); - conf_oc.output_file.path = try cache_root.join(arena, &.{ - "o", &digest, conf_oc.basename, - }); - if (conf_oc.output_file_debug) |*file| { - file.path = try cache_root.join(arena, &.{ - "o", &digest, try allocPrint(arena, "{s}.debug", .{conf_oc.basename}), + maker.generatedPath(conf_oc.output_file).* = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), + }; + if (conf_oc.debug_file.value) |debug_file| { + const debug_basename = opt_debug_basename orelse try allocPrint(arena, "{s}.debug", .{ + Io.Dir.path.basename(input_path.sub_path), }); + maker.generatedPath(debug_file).* = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, debug_basename }), + }; } return; } + // We don't find out more input files while executing objcopy so we can + // already obtain the digest and use it directly as the output path. const digest = man.final(); - const cache_path = "o" ++ Io.Dir.path.sep_str ++ digest; - const full_dest_path = try cache_root.join(arena, &.{ cache_path, conf_oc.basename }); - const full_dest_path_debug = try cache_root.join(arena, &.{ - cache_path, try allocPrint(arena, "{s}.debug", .{conf_oc.basename}), - }); - cache_root.handle.createDirPath(io, cache_path) catch |err| - return step.fail("unable to make path {s}: {t}", .{ cache_path, err }); + const dest_path: Path = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), + }; + const dest_dirname = dest_path.dirname().?; + dest_dirname.root_dir.handle.createDirPath(io, dest_dirname.sub_path) catch |err| + return step.fail(maker, "failed to create path {f}: {t}", .{ dest_dirname, err }); var argv: std.ArrayList([]const u8) = .empty; try argv.ensureUnusedCapacity(arena, 11); argv.addManyAsArrayAssumeCapacity(2).* = .{ graph.zig_exe, "objcopy" }; - if (conf_oc.only_section) |only_section| - argv.addManyAsArrayAssumeCapacity(2).* = .{ "-j", only_section }; + if (only_section) |s| argv.addManyAsArrayAssumeCapacity(2).* = .{ "-j", s }; - switch (conf_oc.strip) { + switch (conf_oc.flags.strip) { .none => {}, .debug => argv.appendAssumeCapacity("--strip-debug"), .debug_and_symbols => argv.appendAssumeCapacity("--strip-all"), } - if (conf_oc.pad_to) |pad_to| { + if (conf_oc.pad_to.value) |pad_to| { argv.addManyAsArrayAssumeCapacity(2).* = .{ "--pad-to", try allocPrint(arena, "{d}", .{pad_to}), }; } - if (conf_oc.format) |format| { - argv.addManyAsArrayAssumeCapacity(2).* = .{ - "-O", - switch (format) { - .bin => "binary", - .hex => "hex", - .elf => "elf", - }, - }; + switch (conf_oc.flags.format) { + .default => {}, + else => |t| argv.addManyAsArrayAssumeCapacity(2).* = .{ "-O", @tagName(t) }, } - if (conf_oc.compress_debug) + if (conf_oc.flags.compress_debug) argv.appendAssumeCapacity("--compress-debug-sections"); - if (conf_oc.output_file_debug != null) - argv.appendAssumeCapacity(try allocPrint(arena, "--extract-to={s}", .{full_dest_path_debug})); + if (conf_oc.debug_file.value) |debug_file| { + const debug_basename = opt_debug_basename orelse try allocPrint(arena, "{s}.debug", .{ + Io.Dir.path.basename(input_path.sub_path), + }); + const debug_dest_path: Path = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, debug_basename }), + }; + argv.appendAssumeCapacity(try allocPrint(arena, "--extract-to={f}", .{debug_dest_path})); + maker.generatedPath(debug_file).* = debug_dest_path; + } - try argv.ensureUnusedCapacity(arena, 9); + try argv.ensureUnusedCapacity(arena, conf_oc.add_section.slice.len * 2); - if (conf_oc.add_section) |section| { + for (conf_oc.add_section.slice) |section| { argv.appendAssumeCapacity("--add-section"); argv.appendAssumeCapacity(try allocPrint(arena, "{s}={f}", .{ - section.section_name, try maker.resolveLazyPathIndex(arena, section.file_path, step_index), + section.section_name.slice(conf), + try maker.resolveLazyPathIndex(arena, section.file_path, step_index), })); } - if (conf_oc.set_section_alignment) |set_align| { - argv.appendAssumeCapacity("--set-section-alignment"); - argv.appendAssumeCapacity(try allocPrint(arena, "{s}={d}", .{ set_align.section_name, set_align.alignment })); - } + for (conf_oc.update_section.slice) |update| { + const name = update.section_name.slice(conf); - if (conf_oc.set_section_flags) |set_flags| { - const f = set_flags.flags; - // trailing comma is allowed - argv.appendAssumeCapacity("--set-section-flags"); - argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{ - set_flags.section_name, - if (f.alloc) "alloc," else "", - if (f.contents) "contents," else "", - if (f.load) "load," else "", - if (f.readonly) "readonly," else "", - if (f.code) "code," else "", - if (f.exclude) "exclude," else "", - if (f.large) "large," else "", - if (f.merge) "merge," else "", - if (f.strings) "strings," else "", - })); + try argv.ensureUnusedCapacity(arena, 4); + + if (update.flags.alignment.toBytes()) |a| { + argv.appendAssumeCapacity("--set-section-alignment"); + argv.appendAssumeCapacity(try allocPrint(arena, "{s}={d}", .{ name, a })); + } + + const f = update.flags.section_flags; + const default_flags: Configuration.Step.ObjCopy.SectionFlags = .{}; + + if (f != default_flags) { + // trailing comma is allowed + argv.appendAssumeCapacity("--set-section-flags"); + argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{ + name, + if (f.alloc) "alloc," else "", + if (f.contents) "contents," else "", + if (f.load) "load," else "", + if (f.readonly) "readonly," else "", + if (f.code) "code," else "", + if (f.exclude) "exclude," else "", + if (f.large) "large," else "", + if (f.merge) "merge," else "", + if (f.strings) "strings," else "", + })); + } } - argv.appendAssumeCapacity(src_path); - argv.appendAssumeCapacity(full_dest_path); + argv.appendAssumeCapacity(try allocPrint(arena, "{f}", .{input_path})); + argv.appendAssumeCapacity(try allocPrint(arena, "{f}", .{dest_path})); argv.appendAssumeCapacity("--listen=-"); - _ = try Step.evalZigProcess(step_index, maker, argv.items, progress_node, false); + _ = Step.evalZigProcess(step_index, maker, argv.items, progress_node, false) catch |err| switch (err) { + error.NeedCompileErrorCheck => unreachable, + else => |e| return e, + }; - conf_oc.output_file.path = full_dest_path; - if (conf_oc.output_file_debug) |*file| file.path = full_dest_path_debug; - try man.writeManifest(); + maker.generatedPath(conf_oc.output_file).* = dest_path; + + man.writeManifest() catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| try step.addError(maker, "failed writing cache manifest: {t}", .{e}), + }; } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 82a215fc225723def25d676e22e5b6e35dd71cfc..deb21cecb10f8d64df5a636b5e613cdc6fca5a38 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1102,10 +1102,87 @@ pub const Step = extern struct { pub const ObjCopy = struct { flags: @This().Flags, + input_file: LazyPath.Index, + output_file: GeneratedFileIndex, + basename: Storage.FlagOptional(.flags, .basename, String), + debug_file: Storage.FlagOptional(.flags, .debug_file, GeneratedFileIndex), + debug_basename: Storage.FlagOptional(.flags, .debug_basename, String), + only_section: Storage.FlagOptional(.flags, .only_section, String), + pad_to: Storage.FlagOptional(.flags, .pad_to, u64), + add_section: Storage.FlagLengthPrefixedList(.flags, .add_section, AddSection), + update_section: Storage.FlagLengthPrefixedList(.flags, .update_section, UpdateSection), + + pub const Format = enum(u2) { + binary, + hex, + elf, + default, + + pub fn init(f: ?std.Build.Step.ObjCopy.Format) @This() { + return switch (f orelse return .default) { + .binary => .binary, + .hex => .hex, + .elf => .elf, + }; + } + }; + + pub const Strip = enum(u2) { + none, + debug, + debug_and_symbols, + }; + + pub const AddSection = extern struct { + section_name: String, + file_path: LazyPath.Index, + }; + + pub const UpdateSection = extern struct { + section_name: String, + flags: @This().Flags, + + pub const Flags = packed struct(u32) { + section_flags: SectionFlags, + alignment: Alignment, + _: u17 = 0, + }; + }; + + pub const SectionFlags = packed struct(u9) { + /// add SHF_ALLOC + alloc: bool = false, + /// if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing + contents: bool = false, + /// if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing (same as contents) + load: bool = false, + /// readonly: clear default SHF_WRITE flag + readonly: bool = false, + /// add SHF_EXECINSTR + code: bool = false, + /// add SHF_EXCLUDE + exclude: bool = false, + /// add SHF_X86_64_LARGE. Fatal error if target is not x86_64 + large: bool = false, + /// add SHF_MERGE + merge: bool = false, + /// add SHF_STRINGS + strings: bool = false, + }; pub const Flags = packed struct(u32) { tag: Tag = .obj_copy, - _: u27 = 0, + basename: bool, + debug_file: bool, + debug_basename: bool, + format: Format, + strip: Strip, + compress_debug: bool, + only_section: bool, + pad_to: bool, + add_section: bool, + update_section: bool, + _: u15 = 0, }; }; @@ -1658,6 +1735,26 @@ pub const Bytes = extern struct { } }; +/// Stored as a power-of-two, with one special value to indicate none. +pub const Alignment = enum(u6) { + @"1" = 0, + @"2" = 1, + @"4" = 2, + @"8" = 3, + @"16" = 4, + @"32" = 5, + @"64" = 6, + none = std.math.maxInt(u6), + _, + + pub fn toBytes(a: @This()) ?u64 { + return switch (a) { + .none => null, + else => @as(u64, 1) << @intFromEnum(a), + }; + } +}; + pub const DefaultingBool = enum(u2) { false, true, diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index 65e58ffa06832ab6f9b36d36c6c19dd5db9b1a38..05d6442923e4351490920c98c84bb992be5ce3d4 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -6,11 +6,11 @@ const Configuration = std.Build.Configuration; step: Step, input_file: std.Build.LazyPath, -basename: []const u8, +basename: ?[]const u8, output_file: Configuration.GeneratedFileIndex, output_file_debug: Configuration.OptionalGeneratedFileIndex, -format: ?RawFormat, +format: ?Format, only_section: ?[]const u8, pad_to: ?u64, strip: Strip, @@ -22,38 +22,9 @@ set_section_flags: ?SetSectionFlags, pub const base_tag: Step.Tag = .obj_copy; -pub const RawFormat = enum { - bin, - hex, - elf, -}; - -pub const Strip = enum { - none, - debug, - debug_and_symbols, -}; - -pub const SectionFlags = packed struct { - /// add SHF_ALLOC - alloc: bool = false, - /// if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing - contents: bool = false, - /// if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing (same as contents) - load: bool = false, - /// readonly: clear default SHF_WRITE flag - readonly: bool = false, - /// add SHF_EXECINSTR - code: bool = false, - /// add SHF_EXCLUDE - exclude: bool = false, - /// add SHF_X86_64_LARGE. Fatal error if target is not x86_64 - large: bool = false, - /// add SHF_MERGE - merge: bool = false, - /// add SHF_STRINGS - strings: bool = false, -}; +pub const Format = enum { binary, hex, elf }; +pub const Strip = Configuration.Step.ObjCopy.Strip; +pub const SectionFlags = Configuration.Step.ObjCopy.SectionFlags; pub const AddSection = struct { section_name: []const u8, @@ -72,7 +43,7 @@ pub const SetSectionFlags = struct { pub const Options = struct { basename: ?[]const u8 = null, - format: ?RawFormat = null, + format: ?Format = null, only_section: ?[]const u8 = null, pad_to: ?u64 = null, @@ -95,7 +66,6 @@ pub fn create( options: Options, ) *ObjCopy { const graph = owner.graph; - const arena = graph.arena; const obj_copy = graph.create(ObjCopy); obj_copy.* = .{ .step = .init(.{ @@ -104,8 +74,7 @@ pub fn create( .owner = owner, }), .input_file = input_file, - .basename = options.basename orelse - std.fmt.allocPrint(arena, "{f}", .{input_file.fmt(graph)}) catch @panic("OOM"), + .basename = options.basename, .output_file = graph.addGeneratedFile(&obj_copy.step), .output_file_debug = if (options.strip != .none and options.extract_to_separate_file) .init(graph.addGeneratedFile(&obj_copy.step)) -- 2.54.0 From ecba6324bf47a68edf9c8d380f528088be68b5d7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 7 May 2026 18:08:31 -0700 Subject: [PATCH 087/179] configurer: update TranslateC step and get zig's build.zig script fully compiling --- BRANCH_TODO | 1 + lib/compiler/Maker.zig | 4 + lib/compiler/Maker/ScannedConfig.zig | 5 + lib/compiler/Maker/Step/TranslateC.zig | 122 ++++++++++++ lib/std/Build.zig | 18 +- lib/std/Build/Configuration.zig | 4 + lib/std/Build/Step/TranslateC.zig | 197 ++++--------------- lib/std/Build/Step/WriteFile.zig | 42 ++-- test/src/Cases.zig | 28 ++- test/src/Libc.zig | 4 +- test/standalone/dependency_options/build.zig | 2 +- test/standalone/dirname/build.zig | 19 -- test/standalone/install_headers/build.zig | 2 +- 13 files changed, 234 insertions(+), 214 deletions(-) create mode 100644 lib/compiler/Maker/Step/TranslateC.zig diff --git a/BRANCH_TODO b/BRANCH_TODO index c137d947b47451eb44659a671d006b6ec175bc9a..d65507f95d0a184fdbd5a8d9500460e1703cfab0 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,3 +1,4 @@ +* pass overridden pkg-dir to maker * finish migrating the rest of the build steps * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) * make zig-pkg path root configurable in maker (make sure --system still works) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index b8bd2d0b7433091c84e04f051a1b31599c85bdc3..48a7cf785d8c76f84206762bace9dc7ff9eb76f5 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1804,6 +1804,10 @@ pub fn relativePath(maker: *const Maker, relative: Configuration.LazyPath.Relati .root_dir = graph.zig_lib_directory, .sub_path = sub_path, }, + .install_prefix => maker.install_paths.prefix, + .install_lib => maker.install_paths.lib, + .install_bin => maker.install_paths.bin, + .install_include => maker.install_paths.include, }; } diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 487f0b16122ae7987ab048412cc86c7501b17ab6..7fc7cc9608fa268248b063d66f01f6ce4b9251ca 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -79,6 +79,11 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi try printStruct(sc, &sub_struct, Configuration.Step.Run.Arg, field_value.get(c)); try sub_struct.end(); }, + Configuration.Step.ObjCopy.UpdateSection.Flags => { + var sub_struct = try s.beginStruct(.{}); + try printStruct(sc, &sub_struct, Field, field_value); + try sub_struct.end(); + }, Configuration.LazyPath.Index => { switch (field_value.get(c)) { inline else => |u| { diff --git a/lib/compiler/Maker/Step/TranslateC.zig b/lib/compiler/Maker/Step/TranslateC.zig new file mode 100644 index 0000000000000000000000000000000000000000..1d29ba0703e463d393556bf9dbb118cf1e4843e8 --- /dev/null +++ b/lib/compiler/Maker/Step/TranslateC.zig @@ -0,0 +1,122 @@ + +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; + const b = step.owner; + const translate_c: *TranslateC = @fieldParentPtr("step", step); + const arena = b.graph.arena; + + var argv_list = std.array_list.Managed([]const u8).init(b.allocator); + try argv_list.append(b.graph.zig_exe); + try argv_list.append("translate-c"); + if (translate_c.link_libc) { + try argv_list.append("-lc"); + } + + try argv_list.append("--cache-dir"); + try argv_list.append(b.cache_root.path orelse "."); + + try argv_list.append("--global-cache-dir"); + try argv_list.append(b.graph.global_cache_root.path orelse "."); + + if (!translate_c.target.query.isNative()) { + try argv_list.append("-target"); + try argv_list.append(try translate_c.target.query.zigTriple(b.allocator)); + } + + switch (translate_c.optimize) { + .Debug => {}, // Skip since it's the default. + else => try argv_list.append(b.fmt("-O{s}", .{@tagName(translate_c.optimize)})), + } + + for (translate_c.include_dirs.items) |include_dir| { + try include_dir.appendZigProcessFlags(b, &argv_list, step); + } + + for (translate_c.c_macros.items) |c_macro| { + try argv_list.append("-D"); + try argv_list.append(c_macro); + } + + var prev_search_strategy: std.Build.Module.SystemLib.SearchStrategy = .paths_first; + var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; + + for (translate_c.system_libs.items) |*system_lib| { + var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; + const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); + if (system_lib_gop.found_existing) { + try argv_list.appendSlice(system_lib_gop.value_ptr.*); + continue; + } else { + system_lib_gop.value_ptr.* = &.{}; + } + + if (system_lib.search_strategy != prev_search_strategy or + system_lib.preferred_link_mode != prev_preferred_link_mode) + { + switch (system_lib.search_strategy) { + .no_fallback => switch (system_lib.preferred_link_mode) { + .dynamic => try argv_list.append("-search_dylibs_only"), + .static => try argv_list.append("-search_static_only"), + }, + .paths_first => switch (system_lib.preferred_link_mode) { + .dynamic => try argv_list.append("-search_paths_first"), + .static => try argv_list.append("-search_paths_first_static"), + }, + .mode_first => switch (system_lib.preferred_link_mode) { + .dynamic => try argv_list.append("-search_dylibs_first"), + .static => try argv_list.append("-search_static_first"), + }, + } + prev_search_strategy = system_lib.search_strategy; + prev_preferred_link_mode = system_lib.preferred_link_mode; + } + + const prefix: []const u8 = prefix: { + if (system_lib.needed) break :prefix "-needed-l"; + if (system_lib.weak) break :prefix "-weak-l"; + break :prefix "-l"; + }; + switch (system_lib.use_pkg_config) { + .no => try argv_list.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), + .yes, .force => { + if (Step.Compile.runPkgConfig(&translate_c.step, system_lib.name)) |result| { + try argv_list.appendSlice(result.cflags); + try argv_list.appendSlice(result.libs); + try seen_system_libs.put(arena, system_lib.name, result.cflags); + } else |err| switch (err) { + error.PkgConfigInvalidOutput, + error.PkgConfigCrashed, + error.PkgConfigFailed, + error.PkgConfigNotInstalled, + error.PackageNotFound, + => switch (system_lib.use_pkg_config) { + .yes => { + // pkg-config failed, so fall back to linking the library + // by name directly. + try argv_list.append(b.fmt("{s}{s}", .{ + prefix, + system_lib.name, + })); + }, + .force => { + std.debug.panic("pkg-config failed for library {s}", .{system_lib.name}); + }, + .no => unreachable, + }, + + else => |e| return e, + } + }, + } + } + + const c_source_path = translate_c.source.getPath2(b, step); + try argv_list.append(c_source_path); + + try argv_list.append("--listen=-"); + const output_dir = try step.evalZigProcess(argv_list.items, prog_node, false, options.web_server, options.gpa); + + const basename = std.fs.path.stem(std.fs.path.basename(c_source_path)); + translate_c.out_basename = b.fmt("{s}.zig", .{basename}); + translate_c.output_file.path = output_dir.?.joinString(b.allocator, translate_c.out_basename) catch @panic("OOM"); +} diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 4756be93688bad1039c6d3919dff7479d3870338..b5eeefa17189756fcb34d113a22f1646887f3f56 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -108,7 +108,10 @@ pub const Graph = struct { } pub fn dupePath(graph: *const Graph, bytes: []const u8) []const u8 { - const arena = graph.arena; + return dupePathInner(graph.arena, bytes); + } + + fn dupePathInner(arena: Allocator, bytes: []const u8) []const u8 { if (builtin.os.tag != .windows) return arena.dupe(u8, bytes) catch @panic("OOM"); const the_copy = arena.dupe(u8, bytes) catch @panic("OOM"); mem.replaceScalar(u8, the_copy, '/', '\\'); @@ -2331,21 +2334,24 @@ pub const LazyPath = union(enum) { /// Copies the internal strings. /// - /// The `b` parameter is only used for its allocator. All *Build instances - /// share the same allocator. + /// The `graph` parameter is only used for the global arena allocator. pub fn dupe(lazy_path: LazyPath, graph: *const Graph) LazyPath { + return dupeInner(lazy_path, graph.arena); + } + + fn dupeInner(lazy_path: LazyPath, arena: Allocator) LazyPath { return switch (lazy_path) { .src_path => |sp| .{ .src_path = .{ .owner = sp.owner, .sub_path = sp.owner.dupePath(sp.sub_path) } }, - .cwd_relative => |p| .{ .cwd_relative = graph.dupePath(p) }, + .cwd_relative => |p| .{ .cwd_relative = Graph.dupePathInner(arena, p) }, .relative => |r| .{ .relative = r }, .generated => |gen| .{ .generated = .{ .index = gen.index, .up = gen.up, - .sub_path = graph.dupePath(gen.sub_path), + .sub_path = Graph.dupePathInner(arena, gen.sub_path), } }, .dependency => |dep| .{ .dependency = .{ .dependency = dep.dependency, - .sub_path = graph.dupePath(dep.sub_path), + .sub_path = Graph.dupePathInner(arena, dep.sub_path), } }, }; } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index deb21cecb10f8d64df5a636b5e613cdc6fca5a38..f9cd339711573dfcdb592efeca4edb73e625911a 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1643,6 +1643,10 @@ pub const Path = extern struct { build_root, zig_exe, zig_lib, + install_prefix, + install_lib, + install_bin, + install_include, }; pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path { diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index 28d94afe9234c5f29b90eeaa91ef5ef7dd7a0475..837bdb314a68bd9d0bdb2fe5c101edb3b54349b3 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -1,25 +1,25 @@ const TranslateC = @This(); const std = @import("std"); -const Step = std.Build.Step; -const LazyPath = std.Build.LazyPath; const fs = std.fs; const mem = std.mem; +const allocPrint = std.fmt.allocPrint; +const Step = std.Build.Step; +const LazyPath = std.Build.LazyPath; const Configuration = std.Build.Configuration; -pub const base_tag: Step.Tag = .translate_c; - step: Step, source: std.Build.LazyPath, -include_dirs: std.array_list.Managed(std.Build.Module.IncludeDir), +include_dirs: std.ArrayList(std.Build.Module.IncludeDir), system_libs: std.ArrayList(std.Build.Module.SystemLib), -c_macros: std.array_list.Managed([]const u8), -out_basename: []const u8, +c_macros: std.ArrayList([]const u8), target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, output_file: Configuration.GeneratedFileIndex, link_libc: bool, +pub const base_tag: Step.Tag = .translate_c; + pub const Options = struct { root_source_file: std.Build.LazyPath, target: std.Build.ResolvedTarget, @@ -29,20 +29,17 @@ pub const Options = struct { pub fn create(owner: *std.Build, options: Options) *TranslateC { const graph = owner.graph; - const arena = graph.arena; - const translate_c = arena.create(TranslateC) catch @panic("OOM"); + const translate_c = graph.create(TranslateC); const source = options.root_source_file.dupe(graph); translate_c.* = .{ - .step = Step.init(.{ + .step = .init(.{ .tag = base_tag, .name = "translate-c", .owner = owner, - .makeFn = make, }), .source = source, - .include_dirs = std.array_list.Managed(std.Build.Module.IncludeDir).init(arena), - .c_macros = std.array_list.Managed([]const u8).init(arena), - .out_basename = undefined, + .include_dirs = .empty, + .c_macros = .empty, .target = options.target, .optimize = options.optimize, .output_file = graph.addGeneratedFile(&translate_c.step), @@ -90,8 +87,8 @@ pub fn createModule(translate_c: *TranslateC) *std.Build.Module { } fn setUpModule(translate_c: *TranslateC, module: *std.Build.Module) *std.Build.Module { - const b = translate_c.step.owner; - const arena = b.graph.arena; + const graph = translate_c.step.owner.graph; + const arena = graph.arena; if (translate_c.link_libc) module.link_libc = true; @@ -103,42 +100,49 @@ fn setUpModule(translate_c: *TranslateC, module: *std.Build.Module) *std.Build.M } pub fn addAfterIncludePath(translate_c: *TranslateC, lazy_path: LazyPath) void { - const b = translate_c.step.owner; - translate_c.include_dirs.append(.{ .path_after = lazy_path.dupe(b) }) catch + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + translate_c.include_dirs.append(arena, .{ .path_after = lazy_path.dupe(graph) }) catch @panic("OOM"); lazy_path.addStepDependencies(&translate_c.step); } pub fn addSystemIncludePath(translate_c: *TranslateC, lazy_path: LazyPath) void { - const b = translate_c.step.owner; - translate_c.include_dirs.append(.{ .path_system = lazy_path.dupe(b) }) catch + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + translate_c.include_dirs.append(arena, .{ .path_system = lazy_path.dupe(graph) }) catch @panic("OOM"); lazy_path.addStepDependencies(&translate_c.step); } pub fn addIncludePath(translate_c: *TranslateC, lazy_path: LazyPath) void { - const b = translate_c.step.owner; - translate_c.include_dirs.append(.{ .path = lazy_path.dupe(b) }) catch + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + translate_c.include_dirs.append(arena, .{ .path = lazy_path.dupe(graph) }) catch @panic("OOM"); lazy_path.addStepDependencies(&translate_c.step); } pub fn addConfigHeader(translate_c: *TranslateC, config_header: *Step.ConfigHeader) void { - translate_c.include_dirs.append(.{ .config_header_step = config_header }) catch + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + translate_c.include_dirs.append(arena, .{ .config_header_step = config_header }) catch @panic("OOM"); translate_c.step.dependOn(&config_header.step); } pub fn addSystemFrameworkPath(translate_c: *TranslateC, directory_path: LazyPath) void { - const b = translate_c.step.owner; - translate_c.include_dirs.append(.{ .framework_path_system = directory_path.dupe(b) }) catch + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + translate_c.include_dirs.append(arena, .{ .framework_path_system = directory_path.dupe(graph) }) catch @panic("OOM"); directory_path.addStepDependencies(&translate_c.step); } pub fn addFrameworkPath(translate_c: *TranslateC, directory_path: LazyPath) void { - const b = translate_c.step.owner; - translate_c.include_dirs.append(.{ .framework_path = directory_path.dupe(b) }) catch + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + translate_c.include_dirs.append(arena, .{ .framework_path = directory_path.dupe(graph) }) catch @panic("OOM"); directory_path.addStepDependencies(&translate_c.step); } @@ -154,135 +158,17 @@ pub fn addCheckFile(translate_c: *TranslateC, expected_matches: []const []const /// If the value is omitted, it is set to 1. /// `name` and `value` need not live longer than the function call. pub fn defineCMacro(translate_c: *TranslateC, name: []const u8, value: ?[]const u8) void { - const macro = translate_c.step.owner.fmt("{s}={s}", .{ name, value orelse "1" }); - translate_c.c_macros.append(macro) catch @panic("OOM"); + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + const macro = allocPrint(arena, "{s}={s}", .{ name, value orelse "1" }) catch @panic("OOM"); + translate_c.c_macros.append(arena, macro) catch @panic("OOM"); } /// name_and_value looks like [name]=[value]. If the value is omitted, it is set to 1. pub fn defineCMacroRaw(translate_c: *TranslateC, name_and_value: []const u8) void { - translate_c.c_macros.append(translate_c.step.owner.dupe(name_and_value)) catch @panic("OOM"); -} - -fn make(step: *Step, options: Step.MakeOptions) !void { - const prog_node = options.progress_node; - const b = step.owner; - const translate_c: *TranslateC = @fieldParentPtr("step", step); - const arena = b.graph.arena; - - var argv_list = std.array_list.Managed([]const u8).init(b.allocator); - try argv_list.append(b.graph.zig_exe); - try argv_list.append("translate-c"); - if (translate_c.link_libc) { - try argv_list.append("-lc"); - } - - try argv_list.append("--cache-dir"); - try argv_list.append(b.cache_root.path orelse "."); - - try argv_list.append("--global-cache-dir"); - try argv_list.append(b.graph.global_cache_root.path orelse "."); - - if (!translate_c.target.query.isNative()) { - try argv_list.append("-target"); - try argv_list.append(try translate_c.target.query.zigTriple(b.allocator)); - } - - switch (translate_c.optimize) { - .Debug => {}, // Skip since it's the default. - else => try argv_list.append(b.fmt("-O{s}", .{@tagName(translate_c.optimize)})), - } - - for (translate_c.include_dirs.items) |include_dir| { - try include_dir.appendZigProcessFlags(b, &argv_list, step); - } - - for (translate_c.c_macros.items) |c_macro| { - try argv_list.append("-D"); - try argv_list.append(c_macro); - } - - var prev_search_strategy: std.Build.Module.SystemLib.SearchStrategy = .paths_first; - var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; - - for (translate_c.system_libs.items) |*system_lib| { - var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; - const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); - if (system_lib_gop.found_existing) { - try argv_list.appendSlice(system_lib_gop.value_ptr.*); - continue; - } else { - system_lib_gop.value_ptr.* = &.{}; - } - - if (system_lib.search_strategy != prev_search_strategy or - system_lib.preferred_link_mode != prev_preferred_link_mode) - { - switch (system_lib.search_strategy) { - .no_fallback => switch (system_lib.preferred_link_mode) { - .dynamic => try argv_list.append("-search_dylibs_only"), - .static => try argv_list.append("-search_static_only"), - }, - .paths_first => switch (system_lib.preferred_link_mode) { - .dynamic => try argv_list.append("-search_paths_first"), - .static => try argv_list.append("-search_paths_first_static"), - }, - .mode_first => switch (system_lib.preferred_link_mode) { - .dynamic => try argv_list.append("-search_dylibs_first"), - .static => try argv_list.append("-search_static_first"), - }, - } - prev_search_strategy = system_lib.search_strategy; - prev_preferred_link_mode = system_lib.preferred_link_mode; - } - - const prefix: []const u8 = prefix: { - if (system_lib.needed) break :prefix "-needed-l"; - if (system_lib.weak) break :prefix "-weak-l"; - break :prefix "-l"; - }; - switch (system_lib.use_pkg_config) { - .no => try argv_list.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), - .yes, .force => { - if (Step.Compile.runPkgConfig(&translate_c.step, system_lib.name)) |result| { - try argv_list.appendSlice(result.cflags); - try argv_list.appendSlice(result.libs); - try seen_system_libs.put(arena, system_lib.name, result.cflags); - } else |err| switch (err) { - error.PkgConfigInvalidOutput, - error.PkgConfigCrashed, - error.PkgConfigFailed, - error.PkgConfigNotInstalled, - error.PackageNotFound, - => switch (system_lib.use_pkg_config) { - .yes => { - // pkg-config failed, so fall back to linking the library - // by name directly. - try argv_list.append(b.fmt("{s}{s}", .{ - prefix, - system_lib.name, - })); - }, - .force => { - std.debug.panic("pkg-config failed for library {s}", .{system_lib.name}); - }, - .no => unreachable, - }, - - else => |e| return e, - } - }, - } - } - - const c_source_path = translate_c.source.getPath2(b, step); - try argv_list.append(c_source_path); - - try argv_list.append("--listen=-"); - const output_dir = try step.evalZigProcess(argv_list.items, prog_node, false, options.web_server, options.gpa); - - const basename = std.fs.path.stem(std.fs.path.basename(c_source_path)); - translate_c.out_basename = b.fmt("{s}.zig", .{basename}); - translate_c.output_file.path = output_dir.?.joinString(b.allocator, translate_c.out_basename) catch @panic("OOM"); + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + translate_c.c_macros.append(arena, translate_c.step.owner.dupe(name_and_value)) catch @panic("OOM"); } pub fn linkSystemLibrary( @@ -290,9 +176,10 @@ pub fn linkSystemLibrary( name: []const u8, options: std.Build.Module.LinkSystemLibraryOptions, ) void { - const b = translate_c.step.owner; - translate_c.system_libs.append(b.allocator, .{ - .name = b.dupe(name), + const graph = translate_c.step.owner.graph; + const arena = graph.arena; + translate_c.system_libs.append(arena, .{ + .name = graph.dupeString(name), .needed = options.needed, .weak = options.weak, .use_pkg_config = options.use_pkg_config, diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 3e269ded6d5ffaf40da4606cce7f867bcfbc84c2..00ffe5e45675036c8c106d946020126ace98e076 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -119,34 +119,34 @@ pub fn add(write_file: *WriteFile, sub_path: []const u8, bytes: []const u8) std. }; } -/// Place the file into the generated directory within the local cache, -/// along with all the rest of the files added to this step. The parameter -/// here is the destination path relative to the local cache directory -/// associated with this WriteFile. It may be a basename, or it may -/// include sub-directories, in which case this step will ensure the -/// required sub-path exists. -/// This is the option expected to be used most commonly with `addCopyFile`. +/// Copies the provided file into the generated directory within the local +/// cache, along with all the rest of the files added to this step. +/// +/// `sub_path` is the destination path relative to the local cache directory +/// associated with this WriteFile. It may be a basename, or it may include +/// subdirectories, which are created as needed. pub fn addCopyFile(write_file: *WriteFile, source: std.Build.LazyPath, sub_path: []const u8) std.Build.LazyPath { - const b = write_file.step.owner; - const gpa = b.allocator; - const file = File{ - .sub_path = b.dupePath(sub_path), + const graph = write_file.step.owner.graph; + const duped_path = graph.dupePath(sub_path); + const arena = graph.arena; + + write_file.files.append(arena, .{ + .sub_path = duped_path, .contents = .{ .copy = source }, - }; - write_file.files.append(gpa, file) catch @panic("OOM"); + }) catch @panic("OOM"); write_file.maybeUpdateName(); source.addStepDependencies(&write_file.step); - return .{ - .generated = .{ - .index = write_file.generated_directory, - .sub_path = file.sub_path, - }, - }; + + return .{ .generated = .{ + .index = write_file.generated_directory, + .sub_path = duped_path, + } }; } -/// Copy files matching the specified exclude/include patterns to the specified subdirectory -/// relative to this step's generated directory. +/// Copy files matching the specified exclude/include patterns to the specified +/// subdirectory relative to this step's generated directory. +/// /// The returned value is a lazy path to the generated subdirectory. pub fn addCopyDirectory( write_file: *WriteFile, diff --git a/test/src/Cases.zig b/test/src/Cases.zig index ab28f2be6faf49a2189139ff6ffc01fc331a4536..2cc37fb656b8906887f5c0928787e9627d57cc1c 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -470,8 +470,9 @@ pub fn lowerToBuildSteps( options: CaseTestOptions, ) void { const io = self.io; + const graph = b.graph; + const arena = graph.arena; const host = b.resolveTargetQuery(.{}); - const cases_dir_path = b.build_root.join(b.allocator, &.{ "test", "cases" }) catch @panic("OOM"); for (self.cases.items) |case| { for (options.test_filters) |test_filter| { @@ -504,7 +505,7 @@ pub fn lowerToBuildSteps( ); if (options.skip_llvm and would_use_llvm) continue; - const triple_txt = case.target.query.zigTriple(b.allocator) catch @panic("OOM"); + const triple_txt = case.target.query.zigTriple(arena) catch @panic("OOM"); if (options.test_target_filters.len > 0) { for (options.test_target_filters) |filter| { @@ -516,7 +517,7 @@ pub fn lowerToBuildSteps( continue; const writefiles = b.addWriteFiles(); - var file_sources = std.StringHashMap(std.Build.LazyPath).init(b.allocator); + var file_sources = std.StringHashMap(std.Build.LazyPath).init(arena); defer file_sources.deinit(); const first_file = case.files.items[0]; const root_source_file = writefiles.add(first_file.path, first_file.src); @@ -526,12 +527,15 @@ pub fn lowerToBuildSteps( } for (case.imports) |import_rel| { - const import_abs = std.fs.path.join(b.allocator, &.{ - cases_dir_path, - case.import_path orelse @panic("import_path not set"), - import_rel, - }) catch @panic("OOM"); - _ = writefiles.addCopyFile(.{ .cwd_relative = import_abs }, import_rel); + _ = writefiles.addCopyFile(.{ .src_path = .{ + .owner = b, + .sub_path = b.pathJoin(&.{ + "test", + "cases", + case.import_path orelse @panic("import_path not set"), + import_rel, + }), + } }, import_rel); } const mod = b.createModule(.{ @@ -605,7 +609,11 @@ pub fn lowerToBuildSteps( }, .Execution => |expected_stdout| no_exec: { const run = if (case.target.result.ofmt == .c) run_step: { - if (getExternalExecutor(io, &host.result, &case.target.result, .{ .link_libc = true }) != .native) { + if (getExternalExecutor(io, &case.target.result, .{ + .host_cpu_arch = host.result.cpu.arch, + .host_os_tag = host.result.os.tag, + .link_libc = true, + }) != .native) { // We wouldn't be able to run the compiled C code. break :no_exec; } diff --git a/test/src/Libc.zig b/test/src/Libc.zig index c6cea0d57133e3f55c24d115fdc76585e066a102..e06651bb55c7e6ef2efb940feba81fe9922068b1 100644 --- a/test/src/Libc.zig +++ b/test/src/Libc.zig @@ -31,7 +31,9 @@ pub fn addLibcTestCase( supports_wasi_libc: bool, options: LibcTestCaseOption, ) void { - const name = libc.b.dupe(path[0 .. path.len - std.fs.path.extension(path).len]); + const graph = libc.b.graph; + const arena = graph.arena; + const name = arena.dupe(u8, path[0 .. path.len - std.fs.path.extension(path).len]) catch @panic("OOM"); std.mem.replaceScalar(u8, name, '/', '.'); libc.test_cases.append(libc.b.allocator, .{ .name = name, diff --git a/test/standalone/dependency_options/build.zig b/test/standalone/dependency_options/build.zig index 95cde3c8913fc3d8c33b07391bc0a77611623e33..8966920617d0a9bd1c851b96a83abc1e617e6ba0 100644 --- a/test/standalone/dependency_options/build.zig +++ b/test/standalone/dependency_options/build.zig @@ -10,7 +10,7 @@ pub fn build(b: *std.Build) !void { const none_specified_mod = none_specified.module("dummy"); if (!none_specified_mod.resolved_target.?.query.eql(b.graph.host.query)) return error.TestFailed; - const expected_optimize: std.builtin.OptimizeMode = switch (b.release_mode) { + const expected_optimize: std.builtin.OptimizeMode = switch (b.graph.release_mode) { .off => .Debug, .any => unreachable, .fast => .ReleaseFast, diff --git a/test/standalone/dirname/build.zig b/test/standalone/dirname/build.zig index 2dc9e31d3d4728681cf3bfce9f0eef6407679229..dc3be15cf2a9591958ec317aa13547979db0678c 100644 --- a/test/standalone/dirname/build.zig +++ b/test/standalone/dirname/build.zig @@ -27,15 +27,6 @@ pub fn build(b: *std.Build) void { }), }); - const has_basename = b.addExecutable(.{ - .name = "has_basename", - .root_module = b.createModule(.{ - .root_source_file = b.path("has_basename.zig"), - .optimize = .Debug, - .target = target, - }), - }); - // Known path: addTestRun(test_step, exists_in, touch_src.dirname(), &.{"touch.zig"}); @@ -47,16 +38,6 @@ pub fn build(b: *std.Build) void { "subdir" ++ std.fs.path.sep_str ++ "generated.txt", }); - // Cache root: - const cache_dir = b.cache_root.path orelse - (b.cache_root.join(b.allocator, &.{"."}) catch @panic("OOM")); - addTestRun( - test_step, - has_basename, - generated.dirname().dirname().dirname().dirname(), - &.{std.fs.path.basename(cache_dir)}, - ); - // Absolute path: const write_files = b.addWriteFiles(); _ = write_files.add("foo.txt", ""); diff --git a/test/standalone/install_headers/build.zig b/test/standalone/install_headers/build.zig index 60ecaf1d9e824c8c5138d73eec2cf5ab48e2a4df..fe855e32c20f17a64541cc7681bfb5dfcf23d031 100644 --- a/test/standalone/install_headers/build.zig +++ b/test/standalone/install_headers/build.zig @@ -106,7 +106,7 @@ pub fn build(b: *std.Build) void { "custom/include/foo/config.h", "custom/include/bar.h", }); - run_check_exists.setCwd(.{ .cwd_relative = b.getInstallPath(.prefix, "") }); + run_check_exists.setCwd(.{ .relative = .{ .base = .install_prefix } }); run_check_exists.expectExitCode(0); run_check_exists.step.dependOn(&install_libfoo.step); test_step.dependOn(&run_check_exists.step); -- 2.54.0 From f4ae918684975c191749edec26223b7dba7cd9ed Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 7 May 2026 22:43:20 -0700 Subject: [PATCH 088/179] configurer: implement serializing InstallDir --- lib/compiler/configurer.zig | 23 ++++++++++++++++++++++- lib/std/Build/Configuration.zig | 8 +++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 4990478ac25d350817f470ba5d5c02ce570ccc83..9ae2ab0a0efb659518ef3b581759f40d27997470 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -859,7 +859,28 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .dest_sub_path = try wc.addString(sif.dest_rel_path), }))); }, - .install_dir => @panic("TODO"), + .install_dir => e: { + const sid: *Step.InstallDir = @fieldParentPtr("step", step); + const dest_sub_path: ?[]const u8 = if (sid.options.install_subdir.len != 0) + sid.options.install_subdir + else + null; + const include_extensions = sid.options.include_extensions orelse &.{}; + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.InstallDir, .{ + .flags = .{ + .dest_sub_path = dest_sub_path != null, + .exclude_extensions = sid.options.exclude_extensions.len != 0, + .include_extensions = include_extensions.len != 0, + .blank_extensions = sid.options.blank_extensions.len != 0, + }, + .source_dir = try s.addLazyPath(sid.options.source_dir), + .dest_dir = try addInstallDir(wc, sid.options.install_dir), + .dest_sub_path = .{ .value = try s.addOptionalString(dest_sub_path) }, + .exclude_extensions = .{ .slice = try s.initStringList(sid.options.exclude_extensions) }, + .include_extensions = .{ .slice = try s.initStringList(include_extensions) }, + .blank_extensions = .{ .slice = try s.initStringList(sid.options.blank_extensions) }, + }))); + }, .remove_dir => @panic("TODO"), .fail => e: { const sf: *Step.Fail = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index f9cd339711573dfcdb592efeca4edb73e625911a..33e583402ef1041901d849e79cf6f91fba3dd343 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -210,9 +210,11 @@ pub const Wip = struct { } pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes { - _ = wip; - _ = bytes; - @panic("TODO"); + try wip.string_bytes.appendSlice(wip.gpa, bytes); + return .{ + .index = @intCast(wip.string_bytes.items.len - bytes.len), + .len = @intCast(bytes.len), + }; } pub fn addSemVer(wip: *Wip, sv: std.SemanticVersion) Allocator.Error!String { -- 2.54.0 From 1186a10d4e145f553a4b749042fcc02ba6efe18b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 12:20:21 -0700 Subject: [PATCH 089/179] configurer: serialize WriteFile --- lib/compiler/configurer.zig | 42 ++++++- lib/std/Build/Configuration.zig | 51 ++++++-- lib/std/Build/Step/WriteFile.zig | 199 +++++++++++++++---------------- 3 files changed, 182 insertions(+), 110 deletions(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 9ae2ab0a0efb659518ef3b581759f40d27997470..b3490936b7952908cf0521f9dcd7da3ae6f41d5c 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -891,7 +891,47 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .find_program => @panic("TODO"), .fmt => @panic("TODO"), .translate_c => @panic("TODO"), - .write_file => @panic("TODO"), + .write_file => e: { + const wf: *Step.WriteFile = @fieldParentPtr("step", step); + + const copies = try arena.alloc(Configuration.Step.WriteFile.Copy, wf.copies.items.len); + for (copies, wf.copies.items) |*dest, src| dest.* = .{ + .sub_path = src.sub_path, + .src_file = try s.addLazyPath(src.src_file), + }; + + const directories = try arena.alloc( + Configuration.Step.WriteFile.Directory, + wf.directories.items.len, + ); + for (directories, wf.directories.items) |*dest, src| dest.* = .{ + .sub_path = src.sub_path, + .src_path = try s.addLazyPath(src.src_path), + .exclude_extensions = src.exclude_extensions, + .include_extensions = src.include_extensions, + }; + + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.WriteFile, .{ + .flags = .{ + .embeds = wf.embeds.items.len != 0, + .copies = copies.len != 0, + .directories = directories.len != 0, + .mode = switch (wf.mode) { + .whole_cached => .whole_cached, + .tmp => .tmp, + .mutate => .mutate, + }, + }, + .generated_directory = wf.generated_directory, + .embeds = .{ .slice = wf.embeds.items }, + .copies = .{ .slice = copies }, + .directories = .{ .slice = directories }, + .mutate_path = .{ .value = switch (wf.mode) { + .mutate => |lp| try s.addLazyPath(lp), + .whole_cached, .tmp => null, + } }, + }))); + }, .update_source_files => @panic("TODO"), .run => e: { const run: *Step.Run = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 33e583402ef1041901d849e79cf6f91fba3dd343..266a36bf431be9ce8b8183a4db4edaa93fd2ce98 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1251,10 +1251,42 @@ pub const Step = extern struct { pub const WriteFile = struct { flags: @This().Flags, + generated_directory: GeneratedFileIndex, + embeds: Storage.FlagLengthPrefixedList(.flags, .embeds, Embed), + copies: Storage.FlagLengthPrefixedList(.flags, .copies, Copy), + directories: Storage.FlagLengthPrefixedList(.flags, .directories, Directory), + mutate_path: Storage.EnumOptional(.flags, .mode, .mutate, LazyPath.Index), + + pub const Embed = extern struct { + sub_path: String, + contents: Bytes, + }; + + pub const Copy = extern struct { + sub_path: String, + src_file: LazyPath.Index, + }; + + pub const Directory = extern struct { + sub_path: String, + src_path: LazyPath.Index, + exclude_extensions: OptionalStringList, + include_extensions: OptionalStringList, + }; + + pub const Mode = enum(u2) { + whole_cached, + tmp, + mutate, + }; pub const Flags = packed struct(u32) { tag: Tag = .write_file, - _: u27 = 0, + embeds: bool, + copies: bool, + directories: bool, + mode: Mode, + _: u22 = 0, }; }; @@ -2370,8 +2402,11 @@ pub const Storage = enum { }; } - /// A field in flags determines whether the length is zero or nonzero. If the length is - /// nonzero, then there is a length field followed by the list. + /// A field in flags determines whether the length is zero or nonzero. If + /// the length is nonzero, then there is a length field followed by the + /// list. The elements need well-defined memory layout but can otherwise be + /// any multiple of u32 length. The length is the number of elements, not + /// the number of u32s. pub fn FlagLengthPrefixedList( comptime flags_arg: @EnumLiteral(), comptime flag_arg: @EnumLiteral(), @@ -2767,14 +2802,16 @@ pub const Storage = enum { const len: u32 = @intCast(value.slice.len); if (len == 0) return 0; // Flag bit hides the length prefix. buffer[i] = len; - @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); - return len + 1; + const buf_len = len * @divExact(@sizeOf(Field.Elem), @sizeOf(u32)); + @memcpy(buffer[i + 1 ..][0..buf_len], @as([]const u32, @ptrCast(value.slice))); + return 1 + buf_len; }, .length_prefixed_list => { const len: u32 = @intCast(value.slice.len); buffer[i] = len; - @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice))); - return len + 1; + const buf_len = len * @divExact(@sizeOf(Field.Elem), @sizeOf(u32)); + @memcpy(buffer[i + 1 ..][0..buf_len], @as([]const u32, @ptrCast(value.slice))); + return 1 + buf_len; }, .flag_list => { const len: u32 = @intCast(value.slice.len); diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 00ffe5e45675036c8c106d946020126ace98e076..64a7a05666474cd56c31ed353e15be01a1f2c6ea 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -13,8 +13,9 @@ const Configuration = std.Build.Configuration; step: Step, -files: std.ArrayList(File), -directories: std.ArrayList(Directory), +embeds: std.ArrayList(Embed) = .empty, +copies: std.ArrayList(Copy) = .empty, +directories: std.ArrayList(Directory) = .empty, generated_directory: Configuration.GeneratedFileIndex, mode: Mode = .whole_cached, @@ -37,158 +38,152 @@ pub const Mode = union(enum) { mutate: std.Build.LazyPath, }; -pub const File = struct { - sub_path: []const u8, - contents: Contents, +pub const Embed = Configuration.Step.WriteFile.Embed; + +pub const Copy = struct { + sub_path: Configuration.String, + src_file: std.Build.LazyPath, }; pub const Directory = struct { - source: std.Build.LazyPath, - sub_path: []const u8, - options: Options, - - pub const Options = struct { - /// File paths that end in any of these suffixes will be excluded from copying. - exclude_extensions: []const []const u8 = &.{}, - /// Only file paths that end in any of these suffixes will be included in copying. - /// `null` means that all suffixes will be included. - /// `exclude_extensions` takes precedence over `include_extensions`. - include_extensions: ?[]const []const u8 = null, - - pub fn dupe(opts: Options, graph: *std.Build.Graph) Options { - return .{ - .exclude_extensions = graph.dupeStrings(opts.exclude_extensions), - .include_extensions = if (opts.include_extensions) |incs| graph.dupeStrings(incs) else null, - }; - } - - pub fn pathIncluded(opts: Options, path: []const u8) bool { - for (opts.exclude_extensions) |ext| { - if (std.mem.endsWith(u8, path, ext)) - return false; - } - if (opts.include_extensions) |incs| { - for (incs) |inc| { - if (std.mem.endsWith(u8, path, inc)) - return true; - } else { - return false; - } - } - return true; - } - }; -}; - -pub const Contents = union(enum) { - bytes: []const u8, - copy: std.Build.LazyPath, + sub_path: Configuration.String, + src_path: std.Build.LazyPath, + exclude_extensions: Configuration.OptionalStringList, + include_extensions: Configuration.OptionalStringList, }; pub fn create(owner: *std.Build) *WriteFile { const graph = owner.graph; - const arena = graph.arena; - const write_file = arena.create(WriteFile) catch @panic("OOM"); - write_file.* = .{ - .step = Step.init(.{ + const wf = graph.create(WriteFile); + wf.* = .{ + .step = .init(.{ .tag = base_tag, .name = "WriteFile", .owner = owner, }), - .files = .empty, - .directories = .empty, - .generated_directory = graph.addGeneratedFile(&write_file.step), + .generated_directory = graph.addGeneratedFile(&wf.step), }; - return write_file; + return wf; } -pub fn add(write_file: *WriteFile, sub_path: []const u8, bytes: []const u8) std.Build.LazyPath { - const graph = write_file.step.owner.graph; +/// Writes `contents` to a file at `sub_path` relative to the output +/// directory. +/// +/// `sub_path` may be a basename, or it may include subdirectories, which are +/// created as needed. +pub fn add(wf: *WriteFile, sub_path: []const u8, contents: []const u8) std.Build.LazyPath { + const graph = wf.step.owner.graph; + const wc = &graph.wip_configuration; const arena = graph.arena; - const file: File = .{ - .sub_path = graph.dupePath(sub_path), - .contents = .{ .bytes = graph.dupeString(bytes) }, - }; - write_file.files.append(arena, file) catch @panic("OOM"); - write_file.maybeUpdateName(); + + wf.embeds.append(arena, .{ + .sub_path = wc.addString(sub_path) catch @panic("OOM"), + .contents = wc.addBytes(contents) catch @panic("OOM"), + }) catch @panic("OOM"); + + wf.maybeUpdateName(); + return .{ .generated = .{ - .index = write_file.generated_directory, - .sub_path = file.sub_path, + .index = wf.generated_directory, + .sub_path = graph.dupeString(sub_path), }, }; } -/// Copies the provided file into the generated directory within the local -/// cache, along with all the rest of the files added to this step. +/// Copies the provided file to `sub_path` relative to the output directory. /// -/// `sub_path` is the destination path relative to the local cache directory -/// associated with this WriteFile. It may be a basename, or it may include -/// subdirectories, which are created as needed. -pub fn addCopyFile(write_file: *WriteFile, source: std.Build.LazyPath, sub_path: []const u8) std.Build.LazyPath { - const graph = write_file.step.owner.graph; - const duped_path = graph.dupePath(sub_path); +/// `sub_path` may be a basename, or it may include subdirectories, which are +/// created as needed. +pub fn addCopyFile(wf: *WriteFile, src_file: std.Build.LazyPath, sub_path: []const u8) std.Build.LazyPath { + const graph = wf.step.owner.graph; + const wc = &graph.wip_configuration; const arena = graph.arena; - write_file.files.append(arena, .{ - .sub_path = duped_path, - .contents = .{ .copy = source }, + wf.copies.append(arena, .{ + .sub_path = wc.addString(sub_path) catch @panic("OOM"), + .src_file = src_file.dupe(graph), }) catch @panic("OOM"); - write_file.maybeUpdateName(); - source.addStepDependencies(&write_file.step); + wf.maybeUpdateName(); + + src_file.addStepDependencies(&wf.step); return .{ .generated = .{ - .index = write_file.generated_directory, - .sub_path = duped_path, + .index = wf.generated_directory, + .sub_path = graph.dupePath(sub_path), } }; } +pub const CopyDirectoryOptions = struct { + /// File paths that end in any of these suffixes will be excluded from copying. + exclude_extensions: []const []const u8 = &.{}, + /// Only file paths that end in any of these suffixes will be included in copying. + /// `null` means that all suffixes will be included. + /// `exclude_extensions` takes precedence over `include_extensions`. + include_extensions: ?[]const []const u8 = null, +}; + /// Copy files matching the specified exclude/include patterns to the specified /// subdirectory relative to this step's generated directory. /// /// The returned value is a lazy path to the generated subdirectory. pub fn addCopyDirectory( - write_file: *WriteFile, - source: std.Build.LazyPath, + wf: *WriteFile, + src_path: std.Build.LazyPath, sub_path: []const u8, - options: Directory.Options, + options: CopyDirectoryOptions, ) std.Build.LazyPath { - const graph = write_file.step.owner.graph; + const graph = wf.step.owner.graph; + const wc = &graph.wip_configuration; const arena = graph.arena; - const dir = Directory{ - .source = source.dupe(graph), - .sub_path = graph.dupePath(sub_path), - .options = options.dupe(graph), - }; - write_file.directories.append(arena, dir) catch @panic("OOM"); - - write_file.maybeUpdateName(); - source.addStepDependencies(&write_file.step); + + wf.directories.append(arena, .{ + .sub_path = wc.addString(sub_path) catch @panic("OOM"), + .src_path = src_path.dupe(graph), + .exclude_extensions = if (options.exclude_extensions.len != 0) + .init(wc.addStringList(options.exclude_extensions) catch @panic("OOM")) + else + .none, + .include_extensions = if (options.include_extensions) |list| + .init(wc.addStringList(list) catch @panic("OOM")) + else + .none, + }) catch @panic("OOM"); + + wf.maybeUpdateName(); + + src_path.addStepDependencies(&wf.step); + return .{ .generated = .{ - .index = write_file.generated_directory, - .sub_path = dir.sub_path, + .index = wf.generated_directory, + .sub_path = graph.dupePath(sub_path), }, }; } /// Returns a `LazyPath` representing the base directory that contains all the /// files from this `WriteFile`. -pub fn getDirectory(write_file: *WriteFile) std.Build.LazyPath { - return .{ .generated = .{ .index = write_file.generated_directory } }; +pub fn getDirectory(wf: *WriteFile) std.Build.LazyPath { + return .{ .generated = .{ .index = wf.generated_directory } }; } -fn maybeUpdateName(write_file: *WriteFile) void { - if (write_file.files.items.len == 1 and write_file.directories.items.len == 0) { +fn maybeUpdateName(wf: *WriteFile) void { + const graph = wf.step.owner.graph; + const wc = &graph.wip_configuration; + const files_count = wf.embeds.items.len + wf.copies.items.len; + if (files_count == 1 and wf.directories.items.len == 0) { // First time adding a file; update name. - if (std.mem.eql(u8, write_file.step.name, "WriteFile")) { - write_file.step.name = write_file.step.owner.fmt("WriteFile {s}", .{write_file.files.items[0].sub_path}); + const sub_path = if (wf.embeds.items.len == 1) wf.embeds.items[0].sub_path else wf.copies.items[0].sub_path; + if (std.mem.eql(u8, wf.step.name, "WriteFile")) { + wf.step.name = wf.step.owner.fmt("WriteFile {s}", .{wc.stringSlice(sub_path)}); } - } else if (write_file.directories.items.len == 1 and write_file.files.items.len == 0) { + } else if (wf.directories.items.len == 1 and files_count == 0) { // First time adding a directory; update name. - if (std.mem.eql(u8, write_file.step.name, "WriteFile")) { - write_file.step.name = write_file.step.owner.fmt("WriteFile {s}", .{write_file.directories.items[0].sub_path}); + const dir_name = wc.stringSlice(wf.directories.items[0].sub_path); + if (std.mem.eql(u8, wf.step.name, "WriteFile")) { + wf.step.name = wf.step.owner.fmt("WriteFile {s}", .{dir_name}); } } } -- 2.54.0 From d45f792c91c9c43cf3dd228429205caab3a6f22d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 12:33:35 -0700 Subject: [PATCH 090/179] configurer: serialize Step.Fmt --- lib/compiler/configurer.zig | 13 ++++++++++++- lib/std/Build/Step/Fmt.zig | 3 +-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index b3490936b7952908cf0521f9dcd7da3ae6f41d5c..ec7195b37518d4b7f8c090e0d234f7db4fd7567e 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -889,7 +889,18 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }))); }, .find_program => @panic("TODO"), - .fmt => @panic("TODO"), + .fmt => e: { + const sf: *Step.Fmt = @fieldParentPtr("step", step); + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.Fmt, .{ + .flags = .{ + .paths = sf.paths.len != 0, + .exclude_paths = sf.exclude_paths.len != 0, + .check = sf.check, + }, + .paths = .{ .slice = try s.initLazyPathList(sf.paths) }, + .exclude_paths = .{ .slice = try s.initLazyPathList(sf.exclude_paths) }, + }))); + }, .translate_c => @panic("TODO"), .write_file => e: { const wf: *Step.WriteFile = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/Fmt.zig b/lib/std/Build/Step/Fmt.zig index 68f31e36d7fed6c6d39d7053bdcc22feb56d1636..294afdb9af1c23b1500598afe3f1c4bbc68080a7 100644 --- a/lib/std/Build/Step/Fmt.zig +++ b/lib/std/Build/Step/Fmt.zig @@ -26,8 +26,7 @@ pub const Options = struct { pub fn create(owner: *std.Build, options: Options) *Fmt { const graph = owner.graph; - const arena = graph.arena; - const fmt = arena.create(Fmt) catch @panic("OOM"); + const fmt = graph.create(Fmt); fmt.* = .{ .step = .init(.{ -- 2.54.0 From f9f00c2dee151231361df7aa45a71ca33f90dcd1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 12:38:14 -0700 Subject: [PATCH 091/179] configurer: serialize Step.CheckFile --- lib/compiler/configurer.zig | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index ec7195b37518d4b7f8c090e0d234f7db4fd7567e..37f813e2cdcad336402b4323fee0ed1c51dcdfa2 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -1037,7 +1037,20 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { })); break :e @enumFromInt(extra_index); }, - .check_file => @panic("TODO"), + .check_file => e: { + const cf: *Step.CheckFile = @fieldParentPtr("step", step); + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.CheckFile, .{ + .flags = .{ + .expected_exact = cf.expected_exact != null, + .expected_matches = cf.expected_matches.len != 0, + .max_bytes = cf.max_bytes != null, + }, + .file = try s.addLazyPath(cf.file), + .expected_exact = .{ .value = cf.expected_exact }, + .expected_matches = .{ .slice = cf.expected_matches }, + .max_bytes = .{ .value = cf.max_bytes }, + }))); + }, .config_header => @panic("TODO"), .obj_copy => @panic("TODO"), .options => @panic("TODO"), -- 2.54.0 From d7eab060db6aff326a957e4dbffcd09263f7ffbd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 13:18:16 -0700 Subject: [PATCH 092/179] configurer: serialize Step.UpdateSourceFiles --- lib/compiler/Maker/Step/UpdateSourceFiles.zig | 8 +-- lib/compiler/configurer.zig | 31 +++++--- lib/std/Build/Configuration.zig | 13 +--- lib/std/Build/Step/UpdateSourceFiles.zig | 71 +++++++++---------- lib/std/Build/Step/WriteFile.zig | 5 -- 5 files changed, 63 insertions(+), 65 deletions(-) diff --git a/lib/compiler/Maker/Step/UpdateSourceFiles.zig b/lib/compiler/Maker/Step/UpdateSourceFiles.zig index 779155cbab649a3acc6410b55bbaf62b2b159fe3..744fd35341e3c9d31fa9b5d4db2782011ddc2fbf 100644 --- a/lib/compiler/Maker/Step/UpdateSourceFiles.zig +++ b/lib/compiler/Maker/Step/UpdateSourceFiles.zig @@ -35,7 +35,7 @@ pub fn make( for (conf_usf.embeds.slice) |*embed| { const dest_path: Path = .{ .root_dir = build_root, - .sub_path = embed.dest_path.slice(conf), + .sub_path = embed.sub_path.slice(conf), }; if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| { const dirname_path: Path = .{ @@ -47,7 +47,7 @@ pub fn make( } dest_path.root_dir.handle.writeFile(io, .{ .sub_path = dest_path.sub_path, - .data = embed.bytes.slice(conf), + .data = embed.contents.slice(conf), }) catch |err| return step.fail(maker, "failed to write file {f}: {t}", .{ dest_path, err }); any_miss = true; progress_node.completeOne(); @@ -56,7 +56,7 @@ pub fn make( for (conf_usf.copies.slice) |*copy| { const dest_path: Path = .{ .root_dir = build_root, - .sub_path = copy.dest_path.slice(conf), + .sub_path = copy.sub_path.slice(conf), }; if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| { const dirname_path: Path = .{ @@ -66,7 +66,7 @@ pub fn make( dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| return step.fail(maker, "failed to create path {f}: {t}", .{ dirname_path, err }); } - const src_lazy_path = copy.src_path.get(conf); + const src_lazy_path = copy.src_file.get(conf); const source_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); if (!step.inputs.populated()) try step.addWatchInput(maker, arena, src_lazy_path); diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 37f813e2cdcad336402b4323fee0ed1c51dcdfa2..10d1875d0d49ab385ae120ac4aa29d259abbaae5 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -464,6 +464,15 @@ const Serialize = struct { return result; } + fn initCopyList(s: *Serialize, list: []const Step.WriteFile.Copy) ![]const Configuration.Step.WriteFile.Copy { + const result = try s.arena.alloc(Configuration.Step.WriteFile.Copy, list.len); + for (result, list) |*dest, src| dest.* = .{ + .sub_path = src.sub_path, + .src_file = try s.addLazyPath(src.src_file), + }; + return result; + } + fn initOptionalStringList(s: *Serialize, list: []const ?[]const u8) ![]const Configuration.OptionalString { const wc = s.wc; const result = try s.arena.alloc(Configuration.OptionalString, list.len); @@ -905,12 +914,6 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .write_file => e: { const wf: *Step.WriteFile = @fieldParentPtr("step", step); - const copies = try arena.alloc(Configuration.Step.WriteFile.Copy, wf.copies.items.len); - for (copies, wf.copies.items) |*dest, src| dest.* = .{ - .sub_path = src.sub_path, - .src_file = try s.addLazyPath(src.src_file), - }; - const directories = try arena.alloc( Configuration.Step.WriteFile.Directory, wf.directories.items.len, @@ -925,7 +928,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.WriteFile, .{ .flags = .{ .embeds = wf.embeds.items.len != 0, - .copies = copies.len != 0, + .copies = wf.copies.items.len != 0, .directories = directories.len != 0, .mode = switch (wf.mode) { .whole_cached => .whole_cached, @@ -935,7 +938,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .generated_directory = wf.generated_directory, .embeds = .{ .slice = wf.embeds.items }, - .copies = .{ .slice = copies }, + .copies = .{ .slice = try s.initCopyList(wf.copies.items) }, .directories = .{ .slice = directories }, .mutate_path = .{ .value = switch (wf.mode) { .mutate => |lp| try s.addLazyPath(lp), @@ -943,7 +946,17 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { } }, }))); }, - .update_source_files => @panic("TODO"), + .update_source_files => e: { + const usf: *Step.UpdateSourceFiles = @fieldParentPtr("step", step); + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.UpdateSourceFiles, .{ + .flags = .{ + .embeds = usf.embeds.items.len != 0, + .copies = usf.copies.items.len != 0, + }, + .embeds = .{ .slice = usf.embeds.items }, + .copies = .{ .slice = try s.initCopyList(usf.copies.items) }, + }))); + }, .run => e: { const run: *Step.Run = @fieldParentPtr("step", step); var expect_stderr_exact: ?Configuration.Bytes = null; diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 266a36bf431be9ce8b8183a4db4edaa93fd2ce98..b3d81c9ac91bbd090b318b561d8b155eba4fed24 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1229,17 +1229,8 @@ pub const Step = extern struct { embeds: Storage.FlagLengthPrefixedList(.flags, .embeds, Embed), copies: Storage.FlagLengthPrefixedList(.flags, .copies, Copy), - pub const Embed = extern struct { - /// Relative to build root. - dest_path: String, - bytes: Bytes, - }; - - pub const Copy = extern struct { - /// Relative to build root. - dest_path: String, - src_path: LazyPath.Index, - }; + pub const Embed = WriteFile.Embed; + pub const Copy = WriteFile.Copy; pub const Flags = packed struct(u32) { tag: Tag = .update_source_files, diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig index 7615b5829b088c8213adb1928074c7e270c9e9cd..cc5b114b8c2920303ccb90e029a00c691c90587d 100644 --- a/lib/std/Build/Step/UpdateSourceFiles.zig +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -6,64 +6,63 @@ const UpdateSourceFiles = @This(); const std = @import("std"); -const Io = std.Io; const Step = std.Build.Step; -const fs = std.fs; -const ArrayList = std.ArrayList; +const Configuration = std.Build.Configuration; step: Step, -output_source_files: std.ArrayList(OutputSourceFile), +embeds: std.ArrayList(Embed) = .empty, +copies: std.ArrayList(Copy) = .empty, pub const base_tag: Step.Tag = .update_source_files; -pub const OutputSourceFile = struct { - contents: Contents, - sub_path: []const u8, -}; - -pub const Contents = union(enum) { - bytes: []const u8, - copy: std.Build.LazyPath, -}; +pub const Embed = Step.WriteFile.Embed; +pub const Copy = Step.WriteFile.Copy; pub fn create(owner: *std.Build) *UpdateSourceFiles { - const usf = owner.allocator.create(UpdateSourceFiles) catch @panic("OOM"); + const graph = owner.graph; + const usf = graph.create(UpdateSourceFiles); usf.* = .{ .step = .init(.{ .tag = base_tag, .name = "UpdateSourceFiles", .owner = owner, }), - .output_source_files = .empty, }; return usf; } -/// A path relative to the package root. +/// Overwrites a path relative to the build root with the contents of another file. /// -/// Be careful with this because it updates source files. This should not be -/// used as part of the normal build process, but as a utility occasionally -/// run by a developer with intent to modify source files and then commit -/// those changes to version control. -pub fn addCopyFileToSource(usf: *UpdateSourceFiles, source: std.Build.LazyPath, sub_path: []const u8) void { - const b = usf.step.owner; - usf.output_source_files.append(b.allocator, .{ - .contents = .{ .copy = source }, - .sub_path = sub_path, +/// Because it updates source files, this should not be used as part of the +/// normal build process, but as a utility occasionally run by a developer with +/// intent to modify source files and then commit those changes to version +/// control. +pub fn addCopyFileToSource(usf: *UpdateSourceFiles, src_file: std.Build.LazyPath, sub_path: []const u8) void { + const graph = usf.step.owner.graph; + const wc = &graph.wip_configuration; + const arena = graph.arena; + + usf.copies.append(arena, .{ + .sub_path = wc.addString(sub_path) catch @panic("OOM"), + .src_file = src_file.dupe(graph), }) catch @panic("OOM"); - source.addStepDependencies(&usf.step); + + src_file.addStepDependencies(&usf.step); } -/// A path relative to the package root. +/// Overwrites a path relative to the package root with the provided bytes. /// -/// Be careful with this because it updates source files. This should not be -/// used as part of the normal build process, but as a utility occasionally -/// run by a developer with intent to modify source files and then commit -/// those changes to version control. -pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: []const u8) void { - const b = usf.step.owner; - usf.output_source_files.append(b.allocator, .{ - .contents = .{ .bytes = bytes }, - .sub_path = sub_path, +/// Because it updates source files, this should not be used as part of the +/// normal build process, but as a utility occasionally run by a developer with +/// intent to modify source files and then commit those changes to version +/// control. +pub fn addBytesToSource(usf: *UpdateSourceFiles, contents: []const u8, sub_path: []const u8) void { + const graph = usf.step.owner.graph; + const wc = &graph.wip_configuration; + const arena = graph.arena; + + usf.embeds.append(arena, .{ + .sub_path = wc.addString(sub_path) catch @panic("OOM"), + .contents = wc.addBytes(contents) catch @panic("OOM"), }) catch @panic("OOM"); } diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 64a7a05666474cd56c31ed353e15be01a1f2c6ea..a6b5f942d8fccb05c6f7824a6cd009367765aaba 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -4,15 +4,10 @@ const WriteFile = @This(); const std = @import("std"); -const Io = std.Io; -const Dir = std.Io.Dir; const Step = std.Build.Step; -const ArrayList = std.ArrayList; -const assert = std.debug.assert; const Configuration = std.Build.Configuration; step: Step, - embeds: std.ArrayList(Embed) = .empty, copies: std.ArrayList(Copy) = .empty, directories: std.ArrayList(Directory) = .empty, -- 2.54.0 From f158262b30665b22c571373907a14f11eec809f7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 13:36:51 -0700 Subject: [PATCH 093/179] configurer: serialize Step.Options --- lib/compiler/configurer.zig | 19 ++++++++++++++++++- lib/std/Build/Step/Options.zig | 24 ++++++++++-------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 10d1875d0d49ab385ae120ac4aa29d259abbaae5..6b7c2a982d439f056dcd22f20bb3fd77dc064a38 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -1066,7 +1066,24 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .config_header => @panic("TODO"), .obj_copy => @panic("TODO"), - .options => @panic("TODO"), + .options => e: { + const so: *Step.Options = @fieldParentPtr("step", step); + + const args = try arena.alloc(Configuration.Step.Options.Arg, so.args.items.len); + for (args, so.args.items) |*dest, src| dest.* = .{ + .name = src.name, + .path = try s.addLazyPath(src.path), + }; + + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.Options, .{ + .flags = .{ + .args = so.args.items.len != 0, + }, + .generated_file = so.generated_file, + .contents = try wc.addBytes(so.contents.items), + .args = .{ .slice = args }, + }))); + }, }, }); } diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 935aec618c3bc54f25970f7ef16b40afd0edc55a..e78baff95f83862c85b175b7fe1a10328fcd4b2b 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -11,14 +11,14 @@ const Configuration = std.Build.Configuration; step: Step, generated_file: Configuration.GeneratedFileIndex, -contents: std.ArrayList(u8), -args: std.ArrayList(Arg), +contents: std.ArrayList(u8) = .empty, +args: std.ArrayList(Arg) = .empty, encountered_types: std.StringHashMapUnmanaged(void), pub const base_tag: Step.Tag = .options; pub const Arg = struct { - name: []const u8, + name: Configuration.String, path: LazyPath, }; @@ -34,8 +34,6 @@ pub fn create(owner: *std.Build) *Options { .owner = owner, }), .generated_file = graph.addGeneratedFile(&options.step), - .contents = .empty, - .args = .empty, .encountered_types = .empty, }; @@ -416,16 +414,14 @@ fn printStructValue( } } -/// The value is the path in the cache dir. -/// Adds a dependency automatically. -pub fn addOptionPath( - options: *Options, - name: []const u8, - path: LazyPath, -) void { - const arena = options.step.owner.allocator; +/// The added option has type `[]const u8` and value of the provided path. +pub fn addOptionPath(options: *Options, name: []const u8, path: LazyPath) void { + const graph = options.step.owner.graph; + const arena = graph.arena; + const wc = &graph.wip_configuration; + options.args.append(arena, .{ - .name = options.step.owner.dupe(name), + .name = try wc.addString(name), .path = path.dupe(options.step.owner), }) catch @panic("OOM"); path.addStepDependencies(&options.step); -- 2.54.0 From 7e6be7ee6ee4223a4ef2b247002358cbfc714854 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 14:31:37 -0700 Subject: [PATCH 094/179] configurer: serialize Step.ObjCopy --- lib/compiler/Maker/Step/ObjCopy.zig | 4 +- lib/compiler/configurer.zig | 47 +++++++++++- lib/std/Build/Configuration.zig | 7 ++ lib/std/Build/Step/ObjCopy.zig | 108 +++++++++++++++++----------- 4 files changed, 120 insertions(+), 46 deletions(-) diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index 208bd0953fb5735dab7034382cdb931070f4d653..96fd1dfb041b22e572c35a2e5920770f7aa0b1bd 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -137,9 +137,7 @@ pub fn make( } const f = update.flags.section_flags; - const default_flags: Configuration.Step.ObjCopy.SectionFlags = .{}; - - if (f != default_flags) { + if (f != Configuration.Step.ObjCopy.SectionFlags.default) { // trailing comma is allowed argv.appendAssumeCapacity("--set-section-flags"); argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{ diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 6b7c2a982d439f056dcd22f20bb3fd77dc064a38..a7ef5429ff26a3ed8323209257333e2d8bf827a9 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -1065,7 +1065,52 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }))); }, .config_header => @panic("TODO"), - .obj_copy => @panic("TODO"), + .obj_copy => e: { + const oc: *Step.ObjCopy = @fieldParentPtr("step", step); + + const debug_basename: ?Configuration.String = if (oc.debug_file) |df| + df.basename.unwrap() + else + null; + + const debug_file: ?Configuration.GeneratedFileIndex = if (oc.debug_file) |df| + df.output_file + else + null; + + const add_sections = try arena.alloc( + Configuration.Step.ObjCopy.AddSection, + oc.add_sections.items.len, + ); + for (add_sections, oc.add_sections.items) |*dest, src| dest.* = .{ + .section_name = src.section_name, + .file_path = try s.addLazyPath(src.file_path), + }; + + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.ObjCopy, .{ + .flags = .{ + .basename = oc.basename != .none, + .debug_file = debug_file != null, + .debug_basename = debug_basename != null, + .format = .init(oc.format), + .strip = oc.strip, + .compress_debug = oc.compress_debug, + .only_section = oc.only_section != .none, + .pad_to = oc.pad_to != null, + .add_section = add_sections.len != 0, + .update_section = oc.update_sections.items.len != 0, + }, + .input_file = try s.addLazyPath(oc.input_file), + .output_file = oc.output_file, + .basename = .{ .value = oc.basename.unwrap() }, + .debug_file = .{ .value = debug_file }, + .debug_basename = .{ .value = debug_basename }, + .only_section = .{ .value = oc.only_section.unwrap() }, + .pad_to = .{ .value = oc.pad_to }, + .add_section = .{ .slice = add_sections }, + .update_section = .{ .slice = oc.update_sections.items }, + }))); + }, .options => e: { const so: *Step.Options = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index b3d81c9ac91bbd090b318b561d8b155eba4fed24..15986804057d8350d0e1ef8f652c5c7129704c3c 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1170,6 +1170,8 @@ pub const Step = extern struct { merge: bool = false, /// add SHF_STRINGS strings: bool = false, + + pub const default: @This() = .{}; }; pub const Flags = packed struct(u32) { @@ -1776,6 +1778,11 @@ pub const Alignment = enum(u6) { none = std.math.maxInt(u6), _, + pub fn init(optional_alignment: ?std.mem.Alignment) @This() { + const a = optional_alignment orelse return .none; + return @enumFromInt(@intFromEnum(a)); + } + pub fn toBytes(a: @This()) ?u64 { return switch (a) { .none => null, diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index 05d6442923e4351490920c98c84bb992be5ce3d4..d5a38ee91e2bfe9460aa625d56b0e9e1019a9903 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -6,19 +6,18 @@ const Configuration = std.Build.Configuration; step: Step, input_file: std.Build.LazyPath, -basename: ?[]const u8, +basename: Configuration.OptionalString, output_file: Configuration.GeneratedFileIndex, -output_file_debug: Configuration.OptionalGeneratedFileIndex, +debug_file: ?DebugFile, format: ?Format, -only_section: ?[]const u8, +only_section: Configuration.OptionalString, pad_to: ?u64, strip: Strip, compress_debug: bool, -add_section: ?AddSection, -set_section_alignment: ?SetSectionAlignment, -set_section_flags: ?SetSectionFlags, +add_sections: std.ArrayList(AddSection) = .empty, +update_sections: std.ArrayList(Configuration.Step.ObjCopy.UpdateSection) = .empty, pub const base_tag: Step.Tag = .obj_copy; @@ -27,18 +26,13 @@ pub const Strip = Configuration.Step.ObjCopy.Strip; pub const SectionFlags = Configuration.Step.ObjCopy.SectionFlags; pub const AddSection = struct { - section_name: []const u8, + section_name: Configuration.String, file_path: std.Build.LazyPath, }; -pub const SetSectionAlignment = struct { - section_name: []const u8, - alignment: u32, -}; - -pub const SetSectionFlags = struct { - section_name: []const u8, - flags: SectionFlags, +pub const DebugFile = struct { + basename: Configuration.OptionalString, + output_file: Configuration.GeneratedFileIndex, }; pub const Options = struct { @@ -53,50 +47,80 @@ pub const Options = struct { /// Put the stripped out debug sections in a separate file. /// note: the `basename` is baked into the elf file to specify the link to the separate debug file. /// see https://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html - extract_to_separate_file: bool = false, + /// + /// Makes `getOutputSeparatedDebug` return non-null. + separate_debug_file: ?SeparateDebugFile = null, - add_section: ?AddSection = null, - set_section_alignment: ?SetSectionAlignment = null, - set_section_flags: ?SetSectionFlags = null, + pub const SeparateDebugFile = struct { + basename: ?[]const u8, + }; }; -pub fn create( - owner: *std.Build, - input_file: std.Build.LazyPath, - options: Options, -) *ObjCopy { +pub fn create(owner: *std.Build, input_file: std.Build.LazyPath, options: Options) *ObjCopy { const graph = owner.graph; - const obj_copy = graph.create(ObjCopy); - obj_copy.* = .{ + const wc = &graph.wip_configuration; + const oc = graph.create(ObjCopy); + oc.* = .{ .step = .init(.{ .tag = base_tag, .name = owner.fmt("objcopy {f}", .{input_file.fmt(graph)}), .owner = owner, }), .input_file = input_file, - .basename = options.basename, - .output_file = graph.addGeneratedFile(&obj_copy.step), - .output_file_debug = if (options.strip != .none and options.extract_to_separate_file) - .init(graph.addGeneratedFile(&obj_copy.step)) - else - .none, + .basename = if (options.basename) |s| .init(wc.addString(s) catch @panic("OOM")) else .none, + .output_file = graph.addGeneratedFile(&oc.step), + .debug_file = if (options.separate_debug_file) |df| .{ + .basename = if (df.basename) |s| .init(wc.addString(s) catch @panic("OOM")) else .none, + .output_file = graph.addGeneratedFile(&oc.step), + } else null, .format = options.format, - .only_section = options.only_section, + .only_section = if (options.only_section) |s| .init(wc.addString(s) catch @panic("OOM")) else .none, .pad_to = options.pad_to, .strip = options.strip, .compress_debug = options.compress_debug, - .add_section = options.add_section, - .set_section_alignment = options.set_section_alignment, - .set_section_flags = options.set_section_flags, }; - input_file.addStepDependencies(&obj_copy.step); - return obj_copy; + input_file.addStepDependencies(&oc.step); + return oc; } -pub fn getOutput(obj_copy: *const ObjCopy) std.Build.LazyPath { - return .{ .generated = .{ .index = obj_copy.output_file } }; +pub const UpdateSectionOptions = struct { + alignment: ?std.mem.Alignment = null, + flags: SectionFlags = .default, +}; + +pub fn updateSection(oc: *ObjCopy, section_name: []const u8, options: UpdateSectionOptions) void { + const graph = oc.owner.graph; + const arena = graph.arena; + const wc = &graph.wip_configuration; + oc.update_sections.append(arena, .{ + .flags = .{ + .section_flags = options.flags, + .alignment = .init(options.alignment), + }, + .section_name = wc.addString(section_name) catch @panic("OOM"), + }) catch @panic("OOM"); +} + +pub const AddSectionOptions = struct { + file_path: std.Build.LazyPath, +}; + +pub fn addSection(oc: *ObjCopy, section_name: []const u8, options: AddSectionOptions) void { + const graph = oc.owner.graph; + const arena = graph.arena; + const wc = &graph.wip_configuration; + oc.add_sections.append(arena, .{ + .section_name = wc.addString(section_name) catch @panic("OOM"), + .file_path = options.file_path, + }) catch @panic("OOM"); + options.file_path.addStepDependencies(&oc.step); +} + +pub fn getOutput(oc: *const ObjCopy) std.Build.LazyPath { + return .{ .generated = .{ .index = oc.output_file } }; } -pub fn getOutputSeparatedDebug(obj_copy: *const ObjCopy) ?std.Build.LazyPath { - return if (obj_copy.output_file_debug.unwrap()) |index| .{ .generated = .{ .index = index } } else null; +pub fn getOutputSeparatedDebug(oc: *const ObjCopy) ?std.Build.LazyPath { + const df = oc.debug_file orelse return null; + return .{ .generated = .{ .index = df.output_file } }; } -- 2.54.0 From 315d6ee59b42c07ac0a4e31924ce229d7c5afc32 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 16:03:03 -0700 Subject: [PATCH 095/179] configurer: serialize Step.ConfigHeader --- BRANCH_TODO | 1 + lib/compiler/configurer.zig | 58 +++++++++++++++- lib/std/Build/Configuration.zig | 104 +++++++++++++++++++++++++++- lib/std/Build/Step/ConfigHeader.zig | 65 ++++++++--------- 4 files changed, 195 insertions(+), 33 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index d65507f95d0a184fdbd5a8d9500460e1703cfab0..b54140d4a0fee66c9fc59a3dca66b906cf3f3d53 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -76,3 +76,4 @@ closes #31397 ### std.Build API * `b.build_root` (Directory) -> `b.root` (Path) +* `ConfigHeader.Options`: `include_guard_override` -> `include_guard` diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index a7ef5429ff26a3ed8323209257333e2d8bf827a9..b8a0f6097133c3f6178e52f37246084840e15b5a 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -1064,7 +1064,63 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .max_bytes = .{ .value = cf.max_bytes }, }))); }, - .config_header => @panic("TODO"), + .config_header => e: { + const ch: *Step.ConfigHeader = @fieldParentPtr("step", step); + const lazy_path: ?std.Build.LazyPath = ch.style.getPath(); + const pairs = try arena.alloc(Configuration.Step.ConfigHeader.Value.Pair, ch.values.count()); + for (pairs, ch.values.keys(), ch.values.values()) |*pair, key, value| pair.* = .{ + .key = try wc.addString(key), + .index = switch (value) { + .undef => .undef, + .defined => .defined, + .boolean => |x| switch (x) { + false => .bool_false, + true => .bool_true, + }, + .int => |x| switch (x) { + 0 => .int_0, + 1 => .int_1, + else => @enumFromInt(try wc.addExtra( + Configuration.Step.ConfigHeader.Value.initSigned(x), + )), + }, + .ident => |x| @enumFromInt(try wc.addExtra(@as(Configuration.Step.ConfigHeader.Value, .{ + .flags = .{ + .tag = .ident, + .small = 0, + }, + .i64 = .{ .value = null }, + .u64 = .{ .value = null }, + .ident = .{ .value = try wc.addString(x) }, + .string = .{ .value = null }, + }))), + .string => |x| @enumFromInt(try wc.addExtra(@as(Configuration.Step.ConfigHeader.Value, .{ + .flags = .{ + .tag = .string, + .small = 0, + }, + .i64 = .{ .value = null }, + .u64 = .{ .value = null }, + .ident = .{ .value = null }, + .string = .{ .value = try wc.addString(x) }, + }))), + }, + }; + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.ConfigHeader, .{ + .flags = .{ + .template_file = lazy_path != null, + .style = .init(ch.style), + .input_size_limit = ch.input_size_limit != null, + .include_guard = ch.include_guard != .none, + }, + .template_file = .{ .value = try s.addOptionalLazyPath(lazy_path) }, + .generated_dir = ch.generated_dir, + .input_size_limit = .{ .value = ch.input_size_limit }, + .include_path = try wc.addString(ch.include_path), + .include_guard = .{ .value = ch.include_guard.unwrap() }, + .values = .{ .slice = pairs }, + }))); + }, .obj_copy => e: { const oc: *Step.ObjCopy = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 15986804057d8350d0e1ef8f652c5c7129704c3c..2a15f3362d36a8b917d65b4d670c3bfa9685c477 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1029,10 +1029,112 @@ pub const Step = extern struct { pub const ConfigHeader = struct { flags: @This().Flags, + template_file: Storage.FlagOptional(.flags, .template_file, LazyPath.Index), + generated_dir: GeneratedFileIndex, + input_size_limit: Storage.FlagOptional(.flags, .input_size_limit, u64), + include_path: String, + include_guard: Storage.FlagOptional(.flags, .include_guard, String), + values: Storage.LengthPrefixedList(Value.Pair), + + pub const Style = enum(u3) { + autoconf_undef, + autoconf_at, + cmake, + blank, + nasm, + + pub fn init(s: std.Build.Step.ConfigHeader.Style) Style { + return switch (s) { + .autoconf_undef => .autoconf_undef, + .autoconf_at => .autoconf_at, + .cmake => .cmake, + .blank => .blank, + .nasm => .nasm, + }; + } + }; + + pub const Value = struct { + flags: @This().Flags, + i64: Storage.EnumOptional(.flags, .tag, .i64, i64), + u64: Storage.EnumOptional(.flags, .tag, .u64, u64), + ident: Storage.EnumOptional(.flags, .tag, .ident, String), + string: Storage.EnumOptional(.flags, .tag, .string, String), + + pub const Flags = packed struct(u32) { + tag: Value.Tag, + small: u29, + }; + + pub const Tag = enum(u3) { + ident, + string, + small_unsigned, + small_signed, + i64, + u64, + }; + + pub const Pair = extern struct { + key: String, + index: Value.Index, + }; + + pub const Index = enum(u32) { + int_0 = max_u32 - 5, + int_1 = max_u32 - 4, + bool_false = max_u32 - 3, + bool_true = max_u32 - 2, + undef = max_u32 - 1, + defined = max_u32, + _, + }; + + pub fn initSigned(x: i64) @This() { + return switch (x) { + 0 => unreachable, // should have been an Index + 1 => unreachable, // should have been an Index + 2...std.math.maxInt(u29) => .{ + .flags = .{ + .tag = .small_unsigned, + .small = @intCast(x), + }, + .i64 = .{ .value = null }, + .u64 = .{ .value = null }, + .ident = .{ .value = null }, + .string = .{ .value = null }, + }, + std.math.minInt(i29)...-1 => .{ + .flags = .{ + .tag = .small_signed, + .small = @bitCast(@as(i29, @intCast(x))), + }, + .i64 = .{ .value = null }, + .u64 = .{ .value = null }, + .ident = .{ .value = null }, + .string = .{ .value = null }, + }, + else => .{ + .flags = .{ + .tag = .i64, + .small = 0, + }, + .i64 = .{ .value = x }, + .u64 = .{ .value = null }, + .ident = .{ .value = null }, + .string = .{ .value = null }, + }, + }; + } + }; pub const Flags = packed struct(u32) { tag: Tag = .config_header, - _: u27 = 0, + template_file: bool, + style: Style, + input_size_limit: bool, + include_guard: bool, + _: u21 = 0, }; }; diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index 026496fe4a08378cbe89b1b8f2dda947b5108747..55b9a617f2e1670e3c32140b43f2aba3c450000a 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -5,16 +5,17 @@ const Io = std.Io; const Step = std.Build.Step; const Allocator = std.mem.Allocator; const Configuration = std.Build.Configuration; +const allocPrint = std.fmt.allocPrint; step: Step, -values: std.array_hash_map.String(Value), +values: std.array_hash_map.String(Value) = .empty, /// This directory contains the generated file under the name `include_path`. generated_dir: Configuration.GeneratedFileIndex, style: Style, -max_bytes: usize, +input_size_limit: ?u64, include_path: []const u8, -include_guard_override: ?[]const u8, +include_guard: Configuration.OptionalString, pub const base_tag: Step.Tag = .config_header; @@ -51,42 +52,45 @@ pub const Value = union(enum) { pub const Options = struct { style: Style = .blank, - max_bytes: usize = 2 * 1024 * 1024, + max_bytes: ?u64 = null, include_path: ?[]const u8 = null, + include_guard: ?[]const u8 = null, first_ret_addr: ?usize = null, - include_guard_override: ?[]const u8 = null, }; pub fn create(owner: *std.Build, options: Options) *ConfigHeader { const graph = owner.graph; const arena = graph.arena; - const config_header = arena.create(ConfigHeader) catch @panic("OOM"); + const wc = &graph.wip_configuration; + const config_header = graph.create(ConfigHeader); - var include_path: []const u8 = "config.h"; + const include_path: []const u8 = p: { + if (options.include_path) |p| + break :p graph.dupeString(p); - if (options.style.getPath()) |s| default_include_path: { - const wc = &graph.wip_configuration; - const sub_path = switch (s) { - .src_path => |sp| sp.sub_path, - .generated => break :default_include_path, - .cwd_relative => |sub_path| sub_path, - .relative => |r| wc.stringSlice(r.sub_path), - .dependency => |dependency| dependency.sub_path, - }; - const basename = std.fs.path.basename(sub_path); - if (std.mem.endsWith(u8, basename, ".h.in")) { - include_path = basename[0 .. basename.len - 3]; + if (options.style.getPath()) |s| default: { + const sub_path = switch (s) { + .src_path => |sp| sp.sub_path, + .generated => break :default, + .cwd_relative => |sub_path| sub_path, + .relative => |r| wc.stringSlice(r.sub_path), + .dependency => |dependency| dependency.sub_path, + }; + const basename = Io.Dir.path.basename(sub_path); + if (std.mem.endsWith(u8, basename, ".h.in")) + break :p graph.dupeString(basename[0 .. basename.len - 3]); } - } - - if (options.include_path) |p| { - include_path = p; - } + break :p "config.h"; + }; const name = if (options.style.getPath()) |s| - owner.fmt("configure {t} header {f} to {s}", .{ options.style, s.fmt(graph), include_path }) + allocPrint(arena, "configure {t} header {f} to {s}", .{ + options.style, s.fmt(graph), include_path, + }) catch @panic("OOM") else - owner.fmt("configure {t} header to {s}", .{ options.style, include_path }); + allocPrint(arena, "configure {t} header to {s}", .{ + options.style, include_path, + }) catch @panic("OOM"); config_header.* = .{ .step = .init(.{ @@ -96,11 +100,9 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { .first_ret_addr = options.first_ret_addr orelse @returnAddress(), }), .style = options.style, - .values = .empty, - - .max_bytes = options.max_bytes, - .include_path = graph.dupeString(include_path), - .include_guard_override = options.include_guard_override, + .input_size_limit = options.max_bytes, + .include_path = include_path, + .include_guard = if (options.include_guard) |s| .init(wc.addString(s) catch @panic("OOM")) else .none, .generated_dir = graph.addGeneratedFile(&config_header.step), }; @@ -179,6 +181,7 @@ pub fn addValues(config_header: *ConfigHeader, values: anytype) void { pub fn getOutputDir(ch: *ConfigHeader) std.Build.LazyPath { return .{ .generated = .{ .index = ch.generated_dir } }; } + pub fn getOutputFile(ch: *ConfigHeader) std.Build.LazyPath { return ch.getOutputDir().path(ch.step.owner, ch.include_path); } -- 2.54.0 From 42be6c0088ece7f3df2a30cfbaee0d77151f762a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 17:11:18 -0700 Subject: [PATCH 096/179] configurer: serialize Step.TranslateC --- lib/compiler/configurer.zig | 57 +++++++++++++++++++++++-------- lib/std/Build/Configuration.zig | 13 ++++++- lib/std/Build/Step/TranslateC.zig | 19 ++++++----- 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index b8a0f6097133c3f6178e52f37246084840e15b5a..214dd4cc61b2e1fee76584cc71f83c4a701b6991 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -451,6 +451,24 @@ const Serialize = struct { return result; } + fn initIncludeDirList( + s: *Serialize, + list: []const std.Build.Module.IncludeDir, + ) ![]const Configuration.Module.IncludeDir { + const result = try s.arena.alloc(Configuration.Module.IncludeDir, list.len); + for (result, list) |*dest, src| dest.* = switch (src) { + .path => |lp| .{ .path = try addLazyPath(s, lp) }, + .path_system => |lp| .{ .path_system = try addLazyPath(s, lp) }, + .path_after => |lp| .{ .path_after = try addLazyPath(s, lp) }, + .framework_path => |lp| .{ .framework_path = try addLazyPath(s, lp) }, + .framework_path_system => |lp| .{ .framework_path_system = try addLazyPath(s, lp) }, + .embed_path => |lp| .{ .embed_path = try addLazyPath(s, lp) }, + .other_step => |cs| .{ .other_step = stepIndex(s, &cs.step) }, + .config_header_step => |chs| .{ .config_header_step = stepIndex(s, &chs.step) }, + }; + return result; + } + fn initLazyPathList(s: *Serialize, list: []const std.Build.LazyPath) ![]const Configuration.LazyPath.Index { const result = try s.arena.alloc(Configuration.LazyPath.Index, list.len); for (result, list) |*dest, src| dest.* = try addLazyPath(s, src); @@ -486,18 +504,6 @@ const Serialize = struct { const wc = s.wc; const arena = s.arena; - const include_dirs = try arena.alloc(Configuration.Module.IncludeDir, m.include_dirs.items.len); - for (include_dirs, m.include_dirs.items) |*dest, src| dest.* = switch (src) { - .path => |lp| .{ .path = try addLazyPath(s, lp) }, - .path_system => |lp| .{ .path_system = try addLazyPath(s, lp) }, - .path_after => |lp| .{ .path_after = try addLazyPath(s, lp) }, - .framework_path => |lp| .{ .framework_path = try addLazyPath(s, lp) }, - .framework_path_system => |lp| .{ .framework_path_system = try addLazyPath(s, lp) }, - .embed_path => |lp| .{ .embed_path = try addLazyPath(s, lp) }, - .other_step => |cs| .{ .other_step = stepIndex(s, &cs.step) }, - .config_header_step => |chs| .{ .config_header_step = stepIndex(s, &chs.step) }, - }; - const rpaths = try arena.alloc(Configuration.Module.RPath, m.rpaths.items.len); for (rpaths, m.rpaths.items) |*dest, src| dest.* = switch (src) { .lazy_path => |lp| .{ .lazy_path = try addLazyPath(s, lp) }, @@ -542,7 +548,7 @@ const Serialize = struct { .fuzz = .init(m.fuzz), .code_model = m.code_model, .c_macros = c_macros.len != 0, - .include_dirs = include_dirs.len != 0, + .include_dirs = m.include_dirs.items.len != 0, .lib_paths = lib_paths.len != 0, .rpaths = rpaths.len != 0, .frameworks = frameworks.len != 0, @@ -566,7 +572,7 @@ const Serialize = struct { .c_macros = .{ .slice = c_macros }, .lib_paths = .{ .slice = lib_paths }, .export_symbol_names = .{ .slice = export_symbol_names }, - .include_dirs = .init(include_dirs), + .include_dirs = .init(try s.initIncludeDirList(m.include_dirs.items)), .rpaths = .init(rpaths), .link_objects = .init(link_objects), .frameworks = .{ .slice = frameworks }, @@ -910,7 +916,28 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .exclude_paths = .{ .slice = try s.initLazyPathList(sf.exclude_paths) }, }))); }, - .translate_c => @panic("TODO"), + .translate_c => e: { + const tc: *Step.TranslateC = @fieldParentPtr("step", step); + + const system_libs = try arena.alloc(Configuration.SystemLib.Index, tc.system_libs.items.len); + for (system_libs, tc.system_libs.items) |*dest, *src| dest.* = try s.addSystemLib(src); + + break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.TranslateC, .{ + .flags = .{ + .include_dirs = tc.include_dirs.items.len != 0, + .system_libs = system_libs.len != 0, + .c_macros = tc.c_macros.items.len != 0, + .link_libc = tc.link_libc, + .optimize = .init(tc.optimize), + }, + .src_path = try s.addLazyPath(tc.source), + .output_file = tc.output_file, + .include_dirs = .init(try s.initIncludeDirList(tc.include_dirs.items)), + .system_libs = .{ .slice = system_libs }, + .c_macros = .{ .slice = tc.c_macros.items }, + .target = try addOptionalResolvedTarget(wc, tc.target), + }))); + }, .write_file => e: { const wf: *Step.WriteFile = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 2a15f3362d36a8b917d65b4d670c3bfa9685c477..b08fb39543fca86db2db39a5809fb5eb1b7d2566 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1321,10 +1321,21 @@ pub const Step = extern struct { pub const TranslateC = struct { flags: @This().Flags, + src_path: LazyPath.Index, + output_file: GeneratedFileIndex, + include_dirs: Storage.UnionList(.flags, .include_dirs, Module.IncludeDir), + system_libs: Storage.FlagLengthPrefixedList(.flags, .system_libs, SystemLib.Index), + c_macros: Storage.FlagLengthPrefixedList(.flags, .c_macros, String), + target: ResolvedTarget.OptionalIndex, pub const Flags = packed struct(u32) { tag: Tag = .translate_c, - _: u27 = 0, + include_dirs: bool, + system_libs: bool, + c_macros: bool, + link_libc: bool, + optimize: Module.Optimize, + _: u20 = 0, }; }; diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index 837bdb314a68bd9d0bdb2fe5c101edb3b54349b3..5698111e044860f573a9dcfc9fcb5ba5393c8590 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -10,9 +10,9 @@ const Configuration = std.Build.Configuration; step: Step, source: std.Build.LazyPath, -include_dirs: std.ArrayList(std.Build.Module.IncludeDir), -system_libs: std.ArrayList(std.Build.Module.SystemLib), -c_macros: std.ArrayList([]const u8), +include_dirs: std.ArrayList(std.Build.Module.IncludeDir) = .empty, +system_libs: std.ArrayList(std.Build.Module.SystemLib) = .empty, +c_macros: std.ArrayList(Configuration.String) = .empty, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, output_file: Configuration.GeneratedFileIndex, @@ -38,13 +38,10 @@ pub fn create(owner: *std.Build, options: Options) *TranslateC { .owner = owner, }), .source = source, - .include_dirs = .empty, - .c_macros = .empty, .target = options.target, .optimize = options.optimize, .output_file = graph.addGeneratedFile(&translate_c.step), .link_libc = options.link_libc, - .system_libs = .empty, }; source.addStepDependencies(&translate_c.step); return translate_c; @@ -160,15 +157,19 @@ pub fn addCheckFile(translate_c: *TranslateC, expected_matches: []const []const pub fn defineCMacro(translate_c: *TranslateC, name: []const u8, value: ?[]const u8) void { const graph = translate_c.step.owner.graph; const arena = graph.arena; + const wc = &graph.wip_configuration; const macro = allocPrint(arena, "{s}={s}", .{ name, value orelse "1" }) catch @panic("OOM"); - translate_c.c_macros.append(arena, macro) catch @panic("OOM"); + const macro_string = wc.addString(macro) catch @panic("OOM"); + translate_c.c_macros.append(arena, macro_string) catch @panic("OOM"); } -/// name_and_value looks like [name]=[value]. If the value is omitted, it is set to 1. +/// name_and_value looks like [name]=[value]. pub fn defineCMacroRaw(translate_c: *TranslateC, name_and_value: []const u8) void { const graph = translate_c.step.owner.graph; const arena = graph.arena; - translate_c.c_macros.append(arena, translate_c.step.owner.dupe(name_and_value)) catch @panic("OOM"); + const wc = &graph.wip_configuration; + const macro_string = wc.addString(name_and_value) catch @panic("OOM"); + translate_c.c_macros.append(arena, macro_string) catch @panic("OOM"); } pub fn linkSystemLibrary( -- 2.54.0 From a249201aecdb0a342ba871ce7adc86926297dcee Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 May 2026 17:26:44 -0700 Subject: [PATCH 097/179] maker: fix Step.Fmt --- lib/compiler/Maker/Step.zig | 7 ++++--- lib/compiler/Maker/Step/ConfigHeader.zig | 1 - lib/compiler/Maker/Step/Fmt.zig | 12 ++++++------ lib/compiler/Maker/Step/Options.zig | 2 -- lib/compiler/Maker/Step/TranslateC.zig | 1 - lib/compiler/Maker/Step/WriteFile.zig | 1 - 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 9681c99814dc97caba3824e7ff13aef655471329..cc04acf566745ebe0d2a28d1ca69357c25b8407e 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -19,11 +19,12 @@ const WebServer = @import("WebServer.zig"); const Maker = @import("../Maker.zig"); pub const Compile = @import("Step/Compile.zig"); -pub const Run = @import("Step/Run.zig"); +pub const Fmt = @import("Step/Fmt.zig"); pub const InstallArtifact = @import("Step/InstallArtifact.zig"); pub const InstallFile = @import("Step/InstallFile.zig"); -pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); pub const ObjCopy = @import("Step/ObjCopy.zig"); +pub const Run = @import("Step/Run.zig"); +pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); /// Avoid false sharing. _: void align(std.atomic.cache_line) = {}, @@ -73,7 +74,7 @@ pub const Extended = union(enum) { config_header: Todo, fail: Fail, find_program: Todo, - fmt: Todo, + fmt: Fmt, install_artifact: InstallArtifact, install_dir: Todo, install_file: InstallFile, diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index 89c8134400d280875b4e0148f847356f97da3b4a..f823d5346807113e7ae69bc3f537240f4958a60a 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -106,7 +106,6 @@ pub fn make( try man.writeManifest(); } - fn render_autoconf_undef( step: *Step, contents: []const u8, diff --git a/lib/compiler/Maker/Step/Fmt.zig b/lib/compiler/Maker/Step/Fmt.zig index 292a172ac1a4bf4c38da868a3f7467fbb40e6242..addd39a969491bf74c51b0db20235cfa26b18196 100644 --- a/lib/compiler/Maker/Step/Fmt.zig +++ b/lib/compiler/Maker/Step/Fmt.zig @@ -24,7 +24,7 @@ pub fn make( const conf_step = step_index.ptr(conf); const conf_fmt = conf_step.extended.get(conf.extra).fmt; const paths = conf_fmt.paths.slice; - const exclude_paths = conf_fmt.paths.exclude_paths; + const exclude_paths = conf_fmt.exclude_paths.slice; argv.clearRetainingCapacity(); try argv.ensureUnusedCapacity(gpa, 2 + 1 + paths.len + 2 * exclude_paths.len); @@ -32,23 +32,23 @@ pub fn make( argv.appendAssumeCapacity(graph.zig_exe); argv.appendAssumeCapacity("fmt"); - if (fmt.check) + if (conf_fmt.flags.check) argv.appendAssumeCapacity("--check"); - for (fmt.paths) |lp| + for (paths) |lp| argv.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, step_index)); - for (fmt.exclude_paths) |lp| { + for (exclude_paths) |lp| { argv.appendAssumeCapacity("--exclude"); argv.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, step_index)); } const run_result = try step.captureChildProcess(maker, progress_node, argv.items); - if (fmt.check) switch (run_result.term) { + if (conf_fmt.flags.check) switch (run_result.term) { .exited => |code| if (code != 0 and run_result.stdout.len != 0) { var it = std.mem.tokenizeScalar(u8, run_result.stdout, '\n'); while (it.next()) |bad_file_name| { - try step.addError("{s}: non-conforming formatting", .{bad_file_name}); + try step.addError(maker, "{s}: non-conforming formatting", .{bad_file_name}); } }, else => {}, diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index f6f3d532c2769be522dba1e0a456400e1f3c7d2a..7cca163c0da70d2e01449deabee2bcd8b124d0ff 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -6,7 +6,6 @@ const Configuration = std.Build.Configuration; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); - pub fn make( options: *Options, step_index: Configuration.Step.Index, @@ -81,4 +80,3 @@ pub fn make( }), } } - diff --git a/lib/compiler/Maker/Step/TranslateC.zig b/lib/compiler/Maker/Step/TranslateC.zig index 1d29ba0703e463d393556bf9dbb118cf1e4843e8..37063cb22a912f8cbb6cacf2e224d74c2d81ad8d 100644 --- a/lib/compiler/Maker/Step/TranslateC.zig +++ b/lib/compiler/Maker/Step/TranslateC.zig @@ -1,4 +1,3 @@ - fn make(step: *Step, options: Step.MakeOptions) !void { const prog_node = options.progress_node; const b = step.owner; diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index d594f8983fe5ead949a5efdd0ba106c176777d34..e392a56b01cf196db1a6cf11a45e419bb9470e53 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -1,4 +1,3 @@ - fn make(step: *Step, options: Step.MakeOptions) !void { _ = options; const b = step.owner; -- 2.54.0 From bd4c1e34d28bb7ab88ada31bb0fa01fda6e4b201 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 9 May 2026 19:30:08 -0700 Subject: [PATCH 098/179] configurer: add search_prefixes back It is generally best practice to avoid calling this function, instead relying on the user to provide these paths via the standard build system interface. However, when integrating with other build systems, the user may have already provided the information to the other build system, and thus it is desirable to use that same information without requiring the user to provide it again. --- BRANCH_TODO | 23 +++++++++++++++++++++++ lib/compiler/Maker.zig | 3 +++ lib/compiler/Maker/ScannedConfig.zig | 6 ++++++ lib/std/Build.zig | 24 +++++++++++++++--------- lib/std/Build/Configuration.zig | 8 ++++++++ 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index b54140d4a0fee66c9fc59a3dca66b906cf3f3d53..ffc063b7cc33675994c68b3ac6b7ea81889b2c80 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -77,3 +77,26 @@ closes #31397 * `b.build_root` (Directory) -> `b.root` (Path) * `ConfigHeader.Options`: `include_guard_override` -> `include_guard` + +### Perf Data Point: `zig build -h` (cached) + +``` +Benchmark 1 (34 runs): master/zig build -h + measurement mean ± σ min … max outliers delta + wall_time 150ms ± 5.52ms 145ms … 165ms 4 (12%) 0% + peak_rss 84.8MB ± 275KB 84.2MB … 85.1MB 0 ( 0%) 0% + cpu_cycles 593M ± 4.01M 588M … 608M 2 ( 6%) 0% + instructions 995M ± 52.5K 995M … 995M 0 ( 0%) 0% + cache_references 25.8M ± 165K 25.4M … 26.1M 0 ( 0%) 0% + cache_misses 651K ± 20.1K 619K … 697K 0 ( 0%) 0% + branch_misses 918K ± 7.44K 906K … 935K 0 ( 0%) 0% +Benchmark 2 (348 runs): branch/zig build -h + measurement mean ± σ min … max outliers delta + wall_time 14.3ms ± 744us 13.2ms … 23.3ms 8 ( 2%) ⚡- 90.4% ± 0.4% + peak_rss 78.5MB ± 562KB 77.1MB … 81.4MB 7 ( 2%) ⚡- 7.4% ± 0.2% + cpu_cycles 24.1M ± 821K 22.8M … 27.1M 3 ( 1%) ⚡- 95.9% ± 0.1% + instructions 43.7M ± 23.8K 43.7M … 43.8M 56 (16%) ⚡- 95.6% ± 0.0% + cache_references 1.46M ± 14.6K 1.40M … 1.50M 19 ( 5%) ⚡- 94.3% ± 0.1% + cache_misses 142K ± 4.87K 127K … 157K 2 ( 1%) ⚡- 78.1% ± 0.4% + branch_misses 126K ± 1.37K 120K … 129K 12 ( 3%) ⚡- 86.3% ± 0.1% +``` diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 48a7cf785d8c76f84206762bace9dc7ff9eb76f5..d443a6ae4d52dff0da3c323648fdf88be500ffc7 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -446,6 +446,9 @@ pub fn main(init: process.Init.Minimal) !void { else => {}, } } + for (c.search_prefixes) |search_prefix| { + try graph.search_prefixes.append(arena, search_prefix.slice(c)); + } break :sc .{ .configuration = configuration, .top_level_steps = top_level_steps, diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 7fc7cc9608fa268248b063d66f01f6ce4b9251ca..5bb3a9871f53a1c50bb994cd73483315673f23c5 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -20,6 +20,12 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { var serializer: Serializer = .{ .writer = w }; var s = try serializer.beginStruct(.{}); + { + var tf = try s.beginTupleField("search_prefixes", .{}); + for (c.search_prefixes) |string| try tf.field(string.slice(c), .{}); + try tf.end(); + } + try s.field("default_step", @intFromEnum(c.default_step), .{}); { var sf = try s.beginStructField("top_level_steps", .{}); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index b5eeefa17189756fcb34d113a22f1646887f3f56..ad87bc246627a68fb3e769d9a8649d2dd3936edc 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -295,12 +295,12 @@ fn createChild( pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap, ) error{OutOfMemory}!*Build { - const allocator = parent.allocator; - const child = try allocator.create(Build); + const arena = parent.graph.arena; + const child = try arena.create(Build); child.* = .{ .graph = parent.graph, .root = root, - .allocator = allocator, + .allocator = arena, .install_tls = .{ .step = .init(.{ .tag = .top_level, @@ -334,8 +334,8 @@ fn createChild( .pkg_hash = pkg_hash, .available_deps = pkg_deps, }; - try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls); - try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls); + try child.top_level_steps.put(arena, child.install_tls.step.name, &child.install_tls); + try child.top_level_steps.put(arena, child.uninstall_tls.step.name, &child.uninstall_tls); child.default_step = &child.install_tls.step; return child; } @@ -1773,10 +1773,16 @@ pub fn run(b: *Build, argv: []const []const u8) []u8 { /// it is desirable to use that same information without requiring the user to /// provide it again. pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { - _ = b; - _ = search_prefix; - @panic("TODO"); - //b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); + if (b.isRoot()) { + const graph = b.graph; + const wc = &graph.wip_configuration; + const string = wc.addString(search_prefix) catch @panic("OOM"); + wc.search_prefixes.append(wc.gpa, string) catch @panic("OOM"); + } +} + +pub fn isRoot(b: *const Build) bool { + return b.pkg_hash.len == 0; } pub const Dependency = struct { diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index b08fb39543fca86db2db39a5809fb5eb1b7d2566..c008dcc90f31e445ea8b4701edf7ac0507e8e460 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -13,6 +13,7 @@ path_deps_sub: []String, unlazy_deps: []String, system_integrations: []SystemIntegration, available_options: []AvailableOption, +search_prefixes: []String, extra: []u32, default_step: Step.Index, generated_files_len: u32, @@ -26,6 +27,7 @@ pub const Header = extern struct { unlazy_deps_len: u32, system_integrations_len: u32, available_options_len: u32, + search_prefixes_len: u32, extra_len: u32, default_step: Step.Index, @@ -47,6 +49,7 @@ pub const Wip = struct { available_options: std.ArrayList(AvailableOption) = .empty, steps: std.ArrayList(Step) = .empty, path_deps: std.MultiArrayList(Path) = .empty, + search_prefixes: std.ArrayList(String) = .empty, extra: std.ArrayList(u32) = .empty, next_generated_file_index: u32 = 0, @@ -126,6 +129,7 @@ pub const Wip = struct { wip.available_options.deinit(gpa); wip.steps.deinit(gpa); wip.path_deps.deinit(gpa); + wip.search_prefixes.deinit(gpa); wip.extra.deinit(gpa); wip.* = undefined; } @@ -143,6 +147,7 @@ pub const Wip = struct { .unlazy_deps_len = @intCast(wip.unlazy_deps.items.len), .system_integrations_len = @intCast(wip.system_integrations.items.len), .available_options_len = @intCast(wip.available_options.items.len), + .search_prefixes_len = @intCast(wip.search_prefixes.items.len), .extra_len = @intCast(wip.extra.items.len), .default_step = static.default_step, @@ -157,6 +162,7 @@ pub const Wip = struct { @ptrCast(wip.unlazy_deps.items), @ptrCast(wip.system_integrations.items), @ptrCast(wip.available_options.items), + @ptrCast(wip.search_prefixes.items), @ptrCast(wip.extra.items), }; try w.writeVecAll(&buffers); @@ -3017,6 +3023,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), .system_integrations = try arena.alloc(SystemIntegration, header.system_integrations_len), .available_options = try arena.alloc(AvailableOption, header.available_options_len), + .search_prefixes = try arena.alloc(String, header.search_prefixes_len), .extra = try arena.alloc(u32, header.extra_len), .default_step = header.default_step, .generated_files_len = header.generated_files_len, @@ -3029,6 +3036,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { @ptrCast(result.unlazy_deps), @ptrCast(result.system_integrations), @ptrCast(result.available_options), + @ptrCast(result.search_prefixes), @ptrCast(result.extra), }; try reader.readVecAll(&vecs); -- 2.54.0 From 43209551b73b670cbb1505fd5fc45980d763aa9c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 14:49:43 -0700 Subject: [PATCH 099/179] maker: implement Step.Options also revert #35224 --- BRANCH_TODO | 3 + lib/compiler/Maker/Step.zig | 6 +- lib/compiler/Maker/Step/Options.zig | 126 ++++++++++++++++------------ 3 files changed, 81 insertions(+), 54 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index ffc063b7cc33675994c68b3ac6b7ea81889b2c80..acecc8d1e583a2196780ee842adbc9d81d9ffd5c 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -20,6 +20,8 @@ * make the generated dependencies.zig be dependencies.zon and don't put absolute paths in there - and adjust dependencyInner to not openDir() +* re-evaluate https://codeberg.org/ziglang/zig/pulls/35224 + ## Followup Issues * stop leaking into global process arena * reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make @@ -32,6 +34,7 @@ * fmt step: import zig fmt code directly rather than child proc * UpdateSourceFiles: introduce Group * WriteFiles: introduce Group +* re-examine the use case of adding file paths to Options steps ## Already Filed Followup Issues * build system fmt step with check=false does not acquire a write lock on source files #35204 diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index cc04acf566745ebe0d2a28d1ca69357c25b8407e..59fdabcfaa90ee2b5f78bc9118cdf4a980468df9 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -23,6 +23,7 @@ pub const Fmt = @import("Step/Fmt.zig"); pub const InstallArtifact = @import("Step/InstallArtifact.zig"); pub const InstallFile = @import("Step/InstallFile.zig"); pub const ObjCopy = @import("Step/ObjCopy.zig"); +pub const Options = @import("Step/Options.zig"); pub const Run = @import("Step/Run.zig"); pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); @@ -79,7 +80,7 @@ pub const Extended = union(enum) { install_dir: Todo, install_file: InstallFile, obj_copy: ObjCopy, - options: Todo, + options: Options, remove_dir: Todo, run: Run, top_level: TopLevel, @@ -321,7 +322,8 @@ pub fn reset(step: *Step, maker: *Maker) void { step.result_peak_rss = 0; step.result_failed_command = null; step.test_results = .{}; - step.clearWatchInputs(maker); + // We do not clearWatchInputs here because each step manages that choice + // independently. step.result_error_bundle.deinit(gpa); step.result_error_bundle = std.zig.ErrorBundle.empty; diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index 7cca163c0da70d2e01449deabee2bcd8b124d0ff..b44c9af4057e8d3a7b50a0d58ae705da397df79c 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -1,7 +1,9 @@ const Options = @This(); const std = @import("std"); +const Io = std.Io; const Configuration = std.Build.Configuration; +const Cache = std.Build.Cache; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); @@ -12,6 +14,8 @@ pub fn make( maker: *Maker, progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { + _ = options; + // This step completes so quickly that no progress reporting is necessary. _ = progress_node; @@ -19,64 +23,82 @@ pub fn make( const step = maker.stepByIndex(step_index); const io = graph.io; const cache_root = graph.local_cache_root; + const arena = graph.arena; // TODO don't leak into the process arena + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_options = conf_step.extended.get(conf.extra).options; + const contents = conf_options.contents.slice(conf); - for (options.args.items) |arg| { - options.addOption( - []const u8, - arg.name, - arg.path.getPath2(b, step), - ); + // This step operates under the assumption that all contents of the + // generated zig file are observable by dependant steps, as well as the + // contents of files added via Options.Arg. + + step.clearWatchInputs(maker); + + var man = graph.cache.obtain(); + defer man.deinit(); + + var args_bytes: std.ArrayList(u8) = .empty; + + for (conf_options.args.slice) |arg| { + const name = arg.name.slice(conf); + const lazy_path = arg.path.get(conf); + try step.addWatchInput(maker, arena, lazy_path); + const arg_path = try maker.resolveLazyPath(arena, lazy_path, step_index); + _ = try man.addFilePath(arg_path, null); + try args_bytes.print(arena, "pub const {f}: []const u8 = \"{f}\";\n", .{ + std.zig.fmtId(name), arg_path.fmtEscapeString(), + }); } - if (!step.inputs.populated()) for (options.args.items) |arg| { - try step.addWatchInput(arg.path); - }; + + man.hash.addBytes(contents); + man.hash.addBytes(args_bytes.items); const basename = "options.zig"; - // Hash contents to file name. - var hash = graph.cache.hash; - // Random bytes to make unique. Refresh this with new random bytes when - // implementation is modified in a non-backwards-compatible way. - hash.add(@as(u32, 0xad95e922)); - hash.addBytes(options.contents.items); - const sub_path = "c" ++ fs.path.sep_str ++ hash.final() ++ fs.path.sep_str ++ basename; - - options.generated_file.path = try cache_root.join(arena, &.{sub_path}); - - // Optimize for the hot path. Stat the file, and if it already exists, - // cache hit. - if (cache_root.handle.access(io, sub_path, .{})) |_| { - // This is the hot path, success. + if (try step.cacheHitAndWatch(maker, &man)) { + const digest = man.final(); + maker.generatedPath(conf_options.generated_file).* = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), + }; step.result_cached = true; return; - } else |outer_err| switch (outer_err) { - error.FileNotFound => { - var atomic_file = cache_root.handle.createFileAtomic(io, sub_path, .{ - .replace = false, - .make_path = true, - }) catch |err| return step.fail("failed to create temporary path for '{f}{s}': {t}", .{ - cache_root, sub_path, err, - }); - defer atomic_file.deinit(io); - - atomic_file.file.writeStreamingAll(io, options.contents.items) catch |err| { - return step.fail("failed to write options to temporary path for '{f}{s}': {t}", .{ - cache_root, sub_path, err, - }); - }; - - atomic_file.link(io) catch |err| switch (err) { - error.PathAlreadyExists => { - step.result_cached = true; - return; - }, - else => return step.fail("failed to link temporary file into '{f}{s}': {t}", .{ - cache_root, sub_path, err, - }), - }; - }, - else => |e| return step.fail("unable to access options file '{f}{s}': {t}", .{ - cache_root, sub_path, e, - }), } + + const digest = man.final(); + const out_path: Cache.Path = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), + }; + + var file: Io.File = out_path.root_dir.handle.createFile(io, out_path.sub_path, .{}) catch |err| switch (err) { + error.Canceled => |e| return e, + error.FileNotFound => f: { + out_path.root_dir.handle.createDirPath(io, Io.Dir.path.dirname(out_path.sub_path).?) catch |inner| switch (inner) { + error.Canceled => |e| return e, + else => |e| return step.fail(maker, "failed to create {f}: {t}", .{ out_path, e }), + }; + break :f out_path.root_dir.handle.createFile(io, out_path.sub_path, .{}) catch |inner| switch (inner) { + error.Canceled => |e| return e, + else => |e| return step.fail(maker, "failed to create {f}: {t}", .{ out_path, e }), + }; + }, + else => |e| return step.fail(maker, "failed to create {f}: {t}", .{ out_path, e }), + }; + defer file.close(io); + + // No buffer because we already have all contents buffered. + var file_writer = file.writer(io, &.{}); + var data: [2][]const u8 = .{ contents, args_bytes.items }; + file_writer.interface.writeVecAll(&data) catch |write_err| switch (write_err) { + error.WriteFailed => switch (file_writer.err.?) { + error.Canceled => |e| return e, + else => |e| return step.fail(maker, "failed to write to {f}: {t}", .{ out_path, e }), + }, + }; + + try step.writeManifestAndWatch(maker, &man); + + maker.generatedPath(conf_options.generated_file).* = out_path; } -- 2.54.0 From 5644d68f147159d3be14378d6cce06e1fa200dc3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 14:56:52 -0700 Subject: [PATCH 100/179] more colorful wip panics --- lib/compiler/Maker.zig | 2 +- lib/compiler/Maker/Fuzz.zig | 12 ++++++------ lib/compiler/Maker/Step/Compile.zig | 10 +++++----- lib/compiler/Maker/Step/Run.zig | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index d443a6ae4d52dff0da3c323648fdf88be500ffc7..8c83f403003a20bf644a69775a6353b8cd4db1ff 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1781,7 +1781,7 @@ pub fn relativePath(maker: *const Maker, relative: Configuration.LazyPath.Relati const graph = maker.graph; const c = &maker.scanned_config.configuration; const sub_path = relative.sub_path.slice(c); - if (relative.flags.base == .zig_exe and sub_path.len != 0) @panic("TODO"); + if (relative.flags.base == .zig_exe and sub_path.len != 0) @panic("TODO relativePath zig_exe"); return switch (relative.flags.base) { .cwd => .{ .root_dir = .cwd(), diff --git a/lib/compiler/Maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig index 73d1447966f349b8bb1f09a41f2ec5518c33d17e..def3b923150cb16240ef7041840b41265af29c1a 100644 --- a/lib/compiler/Maker/Fuzz.zig +++ b/lib/compiler/Maker/Fuzz.zig @@ -93,7 +93,7 @@ pub fn init( defer rebuild_group.cancel(io); for (all_steps) |step| { - if (true) @panic("TODO"); + if (true) @panic("TODO update the fuzzer"); const run = step.cast(std.Build.Step.Run) orelse continue; if (run.producer == null) continue; if (run.fuzz_tests.items.len == 0) continue; @@ -110,7 +110,7 @@ pub fn init( errdefer gpa.free(run_steps); for (run_steps) |run_step_index| { - if (true) @panic("TODO"); + if (true) @panic("TODO update the fuzzer"); assert(run_step_index.fuzz_tests.items.len > 0); if (run_step_index.rebuilt_executable == null) fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); @@ -144,7 +144,7 @@ pub fn start(fuzz: *Fuzz) void { fatal("unable to spawn coverage task: {t}", .{err}); } - if (true) @panic("TODO"); + if (true) @panic("TODO update the fuzzer"); for (fuzz.run_steps) |run| { assert(run.rebuilt_executable != null); @@ -221,7 +221,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { } pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { - if (true) @panic("TODO"); + if (true) @panic("TODO update the fuzzer"); assert(fuzz.mode == .forever); const gpa = fuzz.maker.gpa; @@ -373,7 +373,7 @@ fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { } } fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { - if (true) @panic("TODO"); + if (true) @panic("TODO update the fuzzer"); assert(fuzz.mode == .forever); const ws = fuzz.mode.forever.ws; const maker = fuzz.maker; @@ -538,7 +538,7 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte } pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { - if (true) @panic("TODO"); + if (true) @panic("TODO update the fuzzer"); assert(fuzz.mode == .limit); const maker = fuzz.maker; const graph = maker.graph; diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 5beb76ae5078ee6ec336421c4c35feef09fe815d..ec1db818f6d988eaaef9f9c18fded9d4793d8a47 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -1037,7 +1037,7 @@ const PkgConfigResult = struct { /// Run pkg-config for the given library name and parse the output, returning the arguments /// that should be passed to zig to link the given library. fn runPkgConfig(compile: *const Compile, maker: *const Maker, lib_name: []const u8) !PkgConfigResult { - if (true) @panic("TODO"); + if (true) @panic("TODO runPkgConfig"); const graph = maker.graph; const wl_rpath_prefix = "-Wl,-rpath,"; @@ -1149,7 +1149,7 @@ fn runPkgConfig(compile: *const Compile, maker: *const Maker, lib_name: []const } fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { - if (true) @panic("TODO"); + if (true) @panic("TODO checkCompileErrors"); // Clear this field so that it does not get printed by the build runner. const actual_eb = compile.step.result_error_bundle; compile.step.result_error_bundle = .empty; @@ -1452,7 +1452,7 @@ fn appendModuleFlags( if (target.query.get(conf)) |query| { try zig_args.ensureUnusedCapacity(gpa, 6); - if (true) @panic("TODO"); + if (true) @panic("TODO appendModuleFlags"); zig_args.appendAssumeCapacity("-target"); zig_args.appendAssumeCapacity(try query.zigTriple(arena)); @@ -1535,12 +1535,12 @@ fn appendIncludeDirFlags( }, .config_header_step => |ch| { zig_args.appendAssumeCapacity("-I"); - if (true) @panic("TODO"); + if (true) @panic("TODO appendIncludeDirFlags"); ch.getOutputDir(); }, .other_step => |comp| { zig_args.appendAssumeCapacity("-I"); - if (true) @panic("TODO"); + if (true) @panic("TODO appendIncludeDirFlags"); comp.installed_headers_include_tree.?.getDirectory(); }, .embed_path => |lazy_path| { diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 30ffc13dc45d0e885c40129f4b11e41267606e74..2bc3ff73c6d9685a976878f2a50d1ab917ab788a 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -2296,12 +2296,12 @@ fn convertPathArg(run_index: Configuration.Step.Index, maker: *Maker, path: Path } fn addPathForDynLibs(artifact: Configuration.Step.Index) void { - if (true) @panic("TODO"); + if (true) @panic("TODO addPathForDynLibs"); for (artifact.getCompileDependencies(true)) |compile| { if (compile.root_module.resolved_target.?.result.os.tag == .windows and compile.isDynamicLibrary()) { - @panic("TODO"); + @panic("TODO addPathForDynLibs"); //addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, step)).?); } } -- 2.54.0 From fb9118195e4a738aff9556389fcbdb9ebacbd020 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 16:42:37 -0700 Subject: [PATCH 101/179] maker: implement pkg-config integration featuring: * better error reporting * including PKG_CONFIG environment variable in `zig env` * memoizing the output of `pkg-config --list-all` --- lib/compiler/Maker.zig | 7 + lib/compiler/Maker/PkgConfig.zig | 203 ++++++++++++++++++++++++ lib/compiler/Maker/Step/Compile.zig | 231 ++++------------------------ lib/std/zig.zig | 1 + 4 files changed, 239 insertions(+), 203 deletions(-) create mode 100644 lib/compiler/Maker/PkgConfig.zig diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 8c83f403003a20bf644a69775a6353b8cd4db1ff..dcc8cb21f72b8b572cac85e45aefea69b7893a96 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -23,6 +23,7 @@ const Step = @import("Maker/Step.zig"); const Watch = @import("Maker/Watch.zig"); const WebServer = @import("Maker/WebServer.zig"); const ScannedConfig = @import("Maker/ScannedConfig.zig"); +const PkgConfig = @import("Maker/PkgConfig.zig"); pub const std_options: std.Options = .{ .side_channels_mitigations = .none, @@ -48,6 +49,7 @@ web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, memory_blocked_steps: std.ArrayList(Configuration.Step.Index), /// Allocated into `gpa`. step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), +pkg_config: PkgConfig, error_style: ErrorStyle, multiline_errors: MultilineErrors, @@ -540,6 +542,11 @@ pub fn main(init: process.Init.Minimal) !void { .web_server = undefined, // set after `prepare` .memory_blocked_steps = .empty, .step_stack = .empty, + .pkg_config = .{ + .mutex = .init, + .list = null, + .debug = debug_pkg_config, + }, .error_style = error_style, .multiline_errors = multiline_errors, diff --git a/lib/compiler/Maker/PkgConfig.zig b/lib/compiler/Maker/PkgConfig.zig new file mode 100644 index 0000000000000000000000000000000000000000..1e2837e91c9c0a2de4377faff2dd40d5fb0efc78 --- /dev/null +++ b/lib/compiler/Maker/PkgConfig.zig @@ -0,0 +1,203 @@ +const std = @import("std"); +const Io = std.Io; +const mem = std.mem; + +const Maker = @import("../Maker.zig"); +const Step = @import("Step.zig"); +const Graph = @import("Graph.zig"); + +pub const Pkg = struct { + name: []const u8, + desc: []const u8, +}; + +mutex: Io.Mutex = .init, +list: ?[]const Pkg = null, +debug: bool = false, + +pub const RunError = error{ + PackageNotFound, + PkgConfigUnavailable, +} || Step.ExtendedMakeError; + +pub const Result = struct { + cflags: []const []const u8, + libs: []const []const u8, +}; + +/// Run pkg-config for the given library name and parse the output, returning the arguments +/// that should be passed to zig to link the given library. +pub fn run( + maker: *Maker, + step: *Step, + progress_node: std.Progress.Node, + lib_name: []const u8, + /// If true, reports failure error messages on step rather than returning + /// error.PackageNotFound or error.PkgConfigInvalidOutput, + force: bool, +) RunError!Result { + const pc = &maker.pkg_config; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into process arena + const wl_rpath_prefix = "-Wl,-rpath,"; + + const pkg_name = match: { + // First we have to map the library name to pkg config name. Unfortunately, + // there are several examples where this is not straightforward: + // -lSDL2 -> pkg-config sdl2 + // -lgdk-3 -> pkg-config gdk-3.0 + // -latk-1.0 -> pkg-config atk + // -lpulse -> pkg-config libpulse + const pkgs = try getList(maker, step, progress_node, force); + + // Exact match means instant winner. + for (pkgs) |pkg| { + if (mem.eql(u8, pkg.name, lib_name)) { + break :match pkg.name; + } + } + + // Next we'll try ignoring case. + for (pkgs) |pkg| { + if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { + break :match pkg.name; + } + } + + // Prefixed "lib" or suffixed ".0". + for (pkgs) |pkg| { + if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { + const prefix = pkg.name[0..pos]; + const suffix = pkg.name[pos + lib_name.len ..]; + if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; + if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; + break :match pkg.name; + } + } + + // Trimming "-1.0". + if (mem.endsWith(u8, lib_name, "-1.0")) { + const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; + for (pkgs) |pkg| { + if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { + break :match pkg.name; + } + } + } + + if (force) return step.fail(maker, "{s}: package not found: {s}", .{ + getExe(graph), lib_name, + }); + + return error.PackageNotFound; + }; + + const pkg_config_exe = getExe(graph); + const captured = try step.captureChildProcess(maker, progress_node, &.{ + pkg_config_exe, pkg_name, "--cflags", "--libs", + }); + try step.handleChildProcessTerm(maker, captured.term); + + var zig_cflags: std.ArrayList([]const u8) = .empty; + var zig_libs: std.ArrayList([]const u8) = .empty; + var arg_it = mem.tokenizeAny(u8, captured.stdout, " \r\n\t"); + + while (arg_it.next()) |arg| { + if (mem.eql(u8, arg, "-I")) { + const dir = arg_it.next() orelse return missingArg(maker, step, pkg_config_exe, lib_name, arg, force); + try zig_cflags.appendSlice(arena, &.{ "-I", dir }); + } else if (mem.startsWith(u8, arg, "-I")) { + try zig_cflags.append(arena, arg); + } else if (mem.eql(u8, arg, "-L")) { + const dir = arg_it.next() orelse return missingArg(maker, step, pkg_config_exe, lib_name, arg, force); + try zig_libs.appendSlice(arena, &.{ "-L", dir }); + } else if (mem.startsWith(u8, arg, "-L")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-l")) { + const lib = arg_it.next() orelse return missingArg(maker, step, pkg_config_exe, lib_name, arg, force); + try zig_libs.appendSlice(arena, &.{ "-l", lib }); + } else if (mem.startsWith(u8, arg, "-l")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-D")) { + const macro = arg_it.next() orelse return missingArg(maker, step, pkg_config_exe, lib_name, arg, force); + try zig_cflags.appendSlice(arena, &.{ "-D", macro }); + } else if (mem.startsWith(u8, arg, "-D")) { + try zig_cflags.append(arena, arg); + } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { + try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); + } else if (force or pc.debug) { + return step.fail(maker, "{s} package {s} unknown flag: {s}", .{ pkg_config_exe, lib_name, arg }); + } + } + + try zig_cflags.shrinkToLen(arena); + try zig_libs.shrinkToLen(arena); + + return .{ + .cflags = zig_cflags.toOwnedSliceAssert(), + .libs = zig_libs.toOwnedSliceAssert(), + }; +} + +fn missingArg( + maker: *Maker, + step: *Step, + pkg_config_exe: []const u8, + lib_name: []const u8, + arg: []const u8, + force: bool, +) RunError { + if (force) return step.fail(maker, "{s} package {s} missing arg after flag: {s}", .{ + pkg_config_exe, lib_name, arg, + }); + return error.PkgConfigUnavailable; +} + +fn getExe(graph: *const Graph) []const u8 { + return std.zig.EnvVar.PKG_CONFIG.get(&graph.environ_map) orelse "pkg-config"; +} + +fn getList(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force: bool) RunError![]const Pkg { + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into process arena + const io = graph.io; + const pc = &maker.pkg_config; + + try pc.mutex.lock(io); + defer pc.mutex.unlock(io); + + if (pc.list) |list| return list; + + const pkg_config_exe = getExe(graph); + const captured = try step.captureChildProcess(maker, progress_node, &.{ pkg_config_exe, "--list-all" }); + if (force) { + try step.handleChildProcessTerm(maker, captured.term); + } else switch (captured.term) { + .exited => |code| if (code != 0) return error.PkgConfigUnavailable, + else => { + try step.handleChildProcessTerm(maker, captured.term); + unreachable; + }, + } + + var list: std.ArrayList(Pkg) = .empty; + var line_it = mem.tokenizeAny(u8, captured.stdout, "\r\n"); + while (line_it.next()) |line| { + if (mem.trim(u8, line, " \t").len == 0) continue; + var tok_it = mem.tokenizeAny(u8, line, " \t"); + try list.append(arena, .{ + .name = tok_it.next() orelse { + if (force) return step.fail(maker, "{s}: invalid line: {s}", .{ + pkg_config_exe, line, + }); + return error.PkgConfigUnavailable; + }, + .desc = tok_it.rest(), + }); + } + try list.shrinkToLen(arena); + + const result = list.toOwnedSliceAssert(); + pc.list = result; + return result; +} diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index ec1db818f6d988eaaef9f9c18fded9d4793d8a47..a01b548269dee83ef04a81a73524a3dbd1215328 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -14,6 +14,7 @@ const allocPrint = std.fmt.allocPrint; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); +const PkgConfig = @import("../PkgConfig.zig"); /// Populated when there is compiler process that lives across multiple calls /// to `make`. @@ -40,7 +41,7 @@ pub fn make( // Reset / repopulate persistent state. compile.zig_args.clearRetainingCapacity(); - try lowerZigArgs(compile, compile_index, maker, &compile.zig_args, false); + try lowerZigArgs(compile, compile_index, maker, progress_node, &compile.zig_args, false); const maybe_output_dir = Step.evalZigProcess( compile_index, @@ -148,10 +149,11 @@ const ModuleListContext = struct { fn lowerZigArgs( compile: *Compile, compile_index: Configuration.Step.Index, - maker: *const Maker, + maker: *Maker, + progress_node: std.Progress.Node, zig_args: *std.ArrayList([]const u8), fuzz: bool, -) error{ OutOfMemory, MakeFailed }!void { +) Step.ExtendedMakeError!void { const step = maker.stepByIndex(compile_index); const graph = maker.graph; const arena = graph.arena; // TODO don't leak into the process arena @@ -312,40 +314,36 @@ fn lowerZigArgs( if (system_lib.flags.weak) break :prefix "-weak-l"; break :prefix "-l"; }; - switch (system_lib.flags.use_pkg_config) { - .no => try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ - prefix, system_lib_name, - })), - .yes, .force => { - if (compile.runPkgConfig(maker, system_lib_name)) |result| { + l: { + pc: { + const force = switch (system_lib.flags.use_pkg_config) { + .no => break :pc, + .yes => false, + .force => true, + }; + + const pkg_conf_node = progress_node.start("pkg-config", 0); + defer pkg_conf_node.end(); + + if (PkgConfig.run(maker, step, pkg_conf_node, system_lib_name, force)) |result| { try zig_args.appendSlice(gpa, result.cflags); try zig_args.appendSlice(gpa, result.libs); try seen_system_libs.put(arena, system_lib.name, result.cflags); + break :l; } else |err| switch (err) { - error.PkgConfigInvalidOutput, - error.PkgConfigCrashed, - error.PkgConfigFailed, - error.PkgConfigNotInstalled, + error.PkgConfigUnavailable, error.PackageNotFound, - => switch (system_lib.flags.use_pkg_config) { - .yes => { - // pkg-config failed, so fall back to linking the library - // by name directly. - try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ - prefix, system_lib_name, - })); - }, - .force => { - return step.fail(maker, "pkg-config failed for library {s}", .{ - system_lib_name, - }); - }, - .no => unreachable, + => { + // pkg-config failed, so fall back to linking the library by name directly. + assert(!force); + break :pc; }, - else => |e| return e, } - }, + } + try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ + prefix, system_lib_name, + })); } }, .other_step => |other_step_index| { @@ -961,65 +959,11 @@ pub fn rebuildInFuzzMode(compile: *Compile, maker: *Maker, progress_node: std.Pr const zig_args = &compile.zig_args; zig_args.clearRetainingCapacity(); - try lowerZigArgs(compile, maker, zig_args, true); + try lowerZigArgs(compile, maker, progress_node, zig_args, true); const maybe_output_bin_path = try compile.step.evalZigProcess(zig_args.items, progress_node, false, maker); return maybe_output_bin_path.?; } -pub const PkgConfigError = error{ - PkgConfigCrashed, - PkgConfigFailed, - PkgConfigNotInstalled, - PkgConfigInvalidOutput, -}; - -pub const PkgConfigPkg = struct { - name: []const u8, - desc: []const u8, -}; - -fn execPkgConfigList(maker: *Maker, out_code: *u8) (PkgConfigError || Maker.RunError)![]const PkgConfigPkg { - const graph = maker.graph; - const process_arena = graph.arena; // TODO don't leak into process arena - const pkg_config_exe = graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = try maker.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); - var list = std.array_list.Managed(PkgConfigPkg).init(process_arena); - errdefer list.deinit(); - var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); - while (line_it.next()) |line| { - if (mem.trim(u8, line, " \t").len == 0) continue; - var tok_it = mem.tokenizeAny(u8, line, " \t"); - try list.append(PkgConfigPkg{ - .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, - .desc = tok_it.rest(), - }); - } - return list.toOwnedSlice(); -} - -fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { - if (b.pkg_config_pkg_list) |res| { - return res; - } - var code: u8 = undefined; - if (execPkgConfigList(b, &code)) |list| { - b.pkg_config_pkg_list = list; - return list; - } else |err| { - const result = switch (err) { - error.ProcessTerminated => error.PkgConfigCrashed, - error.ExecNotSupported => error.PkgConfigFailed, - error.ExitCodeFailure => error.PkgConfigFailed, - error.FileNotFound => error.PkgConfigNotInstalled, - error.InvalidName => error.PkgConfigNotInstalled, - error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, - else => return err, - }; - b.pkg_config_pkg_list = result; - return result; - } -} - fn addBool(gpa: Allocator, args: *std.ArrayList([]const u8), arg: []const u8, opt: bool) !void { if (opt) try args.append(gpa, arg); } @@ -1029,125 +973,6 @@ fn addFlag(gpa: Allocator, args: *std.ArrayList([]const u8), comptime name: []co try args.append(gpa, if (cond) "-f" ++ name else "-fno-" ++ name); } -const PkgConfigResult = struct { - cflags: []const []const u8, - libs: []const []const u8, -}; - -/// Run pkg-config for the given library name and parse the output, returning the arguments -/// that should be passed to zig to link the given library. -fn runPkgConfig(compile: *const Compile, maker: *const Maker, lib_name: []const u8) !PkgConfigResult { - if (true) @panic("TODO runPkgConfig"); - const graph = maker.graph; - const wl_rpath_prefix = "-Wl,-rpath,"; - - const b = compile.step.owner; - const arena = b.allocator; - const pkg_name = match: { - // First we have to map the library name to pkg config name. Unfortunately, - // there are several examples where this is not straightforward: - // -lSDL2 -> pkg-config sdl2 - // -lgdk-3 -> pkg-config gdk-3.0 - // -latk-1.0 -> pkg-config atk - // -lpulse -> pkg-config libpulse - const pkgs = try getPkgConfigList(b); - - // Exact match means instant winner. - for (pkgs) |pkg| { - if (mem.eql(u8, pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Next we'll try ignoring case. - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Prefixed "lib" or suffixed ".0". - for (pkgs) |pkg| { - if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { - const prefix = pkg.name[0..pos]; - const suffix = pkg.name[pos + lib_name.len ..]; - if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; - if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; - break :match pkg.name; - } - } - - // Trimming "-1.0". - if (mem.endsWith(u8, lib_name, "-1.0")) { - const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { - break :match pkg.name; - } - } - } - - return error.PackageNotFound; - }; - - var code: u8 = undefined; - const pkg_config_exe = graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = if (b.runAllowFail(&[_][]const u8{ - pkg_config_exe, - pkg_name, - "--cflags", - "--libs", - }, &code, .ignore)) |stdout| stdout else |err| switch (err) { - error.ProcessTerminated => return error.PkgConfigCrashed, - error.ExecNotSupported => return error.PkgConfigFailed, - error.ExitCodeFailure => return error.PkgConfigFailed, - error.FileNotFound => return error.PkgConfigNotInstalled, - else => return err, - }; - - var zig_cflags: std.ArrayList([]const u8) = .empty; - defer zig_cflags.deinit(arena); - var zig_libs: std.ArrayList([]const u8) = .empty; - defer zig_libs.deinit(arena); - - var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); - while (arg_it.next()) |arg| { - if (mem.eql(u8, arg, "-I")) { - const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_cflags.appendSlice(arena, &.{ "-I", dir }); - } else if (mem.startsWith(u8, arg, "-I")) { - try zig_cflags.append(arena, arg); - } else if (mem.eql(u8, arg, "-L")) { - const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_libs.appendSlice(arena, &.{ "-L", dir }); - } else if (mem.startsWith(u8, arg, "-L")) { - try zig_libs.append(arena, arg); - } else if (mem.eql(u8, arg, "-l")) { - const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_libs.appendSlice(arena, &.{ "-l", lib }); - } else if (mem.startsWith(u8, arg, "-l")) { - try zig_libs.append(arena, arg); - } else if (mem.eql(u8, arg, "-D")) { - const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_cflags.appendSlice(arena, &.{ "-D", macro }); - } else if (mem.startsWith(u8, arg, "-D")) { - try zig_cflags.append(arena, arg); - } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { - try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); - } else if (b.debug_pkg_config) { - return compile.step.fail(maker, "unknown pkg-config flag '{s}'", .{arg}); - } - } - - try zig_cflags.shrinkToLen(arena); - try zig_libs.shrinkToLen(arena); - - return .{ - .cflags = zig_cflags.toOwnedSliceAssert(), - .libs = zig_libs.toOwnedSliceAssert(), - }; -} - fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { if (true) @panic("TODO checkCompileErrors"); // Clear this field so that it does not get printed by the build runner. diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 18d5db1c1d9853d8aee522e24440fbcd2dd0fa22..faa816c575337ae1a08d8967244502621cf40e88 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -772,6 +772,7 @@ pub const EnvVar = enum { CPLUS_INCLUDE_PATH, LIBRARY_PATH, CC, + PKG_CONFIG, // Terminal integration NO_COLOR, -- 2.54.0 From e2dbf6f48ffe09b01c701887b8eb77b326cfe7b2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 16:48:18 -0700 Subject: [PATCH 102/179] Maker.PkgConfig: use mem.cutPrefix --- lib/compiler/Maker/PkgConfig.zig | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/compiler/Maker/PkgConfig.zig b/lib/compiler/Maker/PkgConfig.zig index 1e2837e91c9c0a2de4377faff2dd40d5fb0efc78..567bad7061e92d1237f06f292e48b3965c01cc01 100644 --- a/lib/compiler/Maker/PkgConfig.zig +++ b/lib/compiler/Maker/PkgConfig.zig @@ -39,7 +39,6 @@ pub fn run( const pc = &maker.pkg_config; const graph = maker.graph; const arena = graph.arena; // TODO don't leak into process arena - const wl_rpath_prefix = "-Wl,-rpath,"; const pkg_name = match: { // First we have to map the library name to pkg config name. Unfortunately, @@ -123,8 +122,8 @@ pub fn run( try zig_cflags.appendSlice(arena, &.{ "-D", macro }); } else if (mem.startsWith(u8, arg, "-D")) { try zig_cflags.append(arena, arg); - } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { - try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); + } else if (mem.cutPrefix(u8, arg, "-Wl,-rpath,")) |rest| { + try zig_cflags.appendSlice(arena, &.{ "-rpath", rest }); } else if (force or pc.debug) { return step.fail(maker, "{s} package {s} unknown flag: {s}", .{ pkg_config_exe, lib_name, arg }); } -- 2.54.0 From 9709efce98289f393ae496921ba30123ea6123f5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 18:17:49 -0700 Subject: [PATCH 103/179] std.Build.Step.Run: introduce Arg.cc_args provides a way for the build system to append -target and -isystem/-I flags to a Run step. needed by translate-c package to avoid doing naughty stuff in the configure phase. --- BRANCH_TODO | 3 ++ lib/compiler/Maker/Step/Run.zig | 3 ++ lib/compiler/configurer.zig | 47 ++++++++++++++++++++- lib/std/Build.zig | 75 ++++++++++++++++++++++----------- lib/std/Build/Configuration.zig | 10 +++-- lib/std/Build/Step/Run.zig | 23 ++++++++++ 6 files changed, 132 insertions(+), 29 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index acecc8d1e583a2196780ee842adbc9d81d9ffd5c..ebc00a3d51d24216397354bb245fe6ef76421f0b 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,3 +1,4 @@ +* double check when targets get resolved (should be at configure time) * pass overridden pkg-dir to maker * finish migrating the rest of the build steps * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) @@ -7,6 +8,7 @@ * solve the TODOs added in this branch * get zig tests passing * test a bunch of third party projects / help people migrate + * tetris * get the target from the parent process instead * [handle missing cache hits when chaining two run steps](https://codeberg.org/ziglang/zig/pulls/30762) @@ -80,6 +82,7 @@ closes #31397 * `b.build_root` (Directory) -> `b.root` (Path) * `ConfigHeader.Options`: `include_guard_override` -> `include_guard` +* `LazyPath`: `getDisplayName` -> `format` or `fmt` ### Perf Data Point: `zig build -h` (cached) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 2bc3ff73c6d9685a976878f2a50d1ab917ab788a..2cc2972bcb9a126d9895fcfdfe792dc033ab6543 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -184,6 +184,9 @@ pub fn make( man.hash.addListOfBytes(run_args); } }, + .cc_args => { + @panic("TODO Run make cc_args"); + }, } } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 214dd4cc61b2e1fee76584cc71f83c4a701b6991..a5a35a2d4b83bfc5b317392aa0a88dad9fa4d33a 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -312,6 +312,8 @@ const Serialize = struct { .producer = true, .generated = false, .dep_file = false, + .target_query = false, + .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -319,6 +321,7 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = stepIndex(s, &a.artifact.step) }, .generated = .{ .value = null }, + .target_query = .{ .value = null }, }, .lazy_path => |a| .{ .flags = .{ @@ -330,6 +333,8 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, + .target_query = false, + .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -337,6 +342,7 @@ const Serialize = struct { .path = .{ .value = try addLazyPath(s, a.lazy_path) }, .producer = .{ .value = null }, .generated = .{ .value = null }, + .target_query = .{ .value = null }, }, .decorated_directory => |a| .{ .flags = .{ @@ -348,6 +354,8 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, + .target_query = false, + .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = try addOptionalString(s, a.suffix) }, @@ -355,6 +363,7 @@ const Serialize = struct { .path = .{ .value = try addLazyPath(s, a.lazy_path) }, .producer = .{ .value = null }, .generated = .{ .value = null }, + .target_query = .{ .value = null }, }, .file_content => |a| .{ .flags = .{ @@ -366,6 +375,8 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, + .target_query = false, + .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -373,6 +384,7 @@ const Serialize = struct { .path = .{ .value = try addLazyPath(s, a.lazy_path) }, .producer = .{ .value = null }, .generated = .{ .value = null }, + .target_query = .{ .value = null }, }, .bytes => |a| .{ .flags = .{ @@ -384,6 +396,8 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, + .target_query = false, + .link_libc = false, }, .prefix = .{ .value = try wc.addString(a) }, .suffix = .{ .value = null }, @@ -391,6 +405,7 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = null }, + .target_query = .{ .value = null }, }, .output_file, .output_file_dep => |a, tag| .{ .flags = .{ @@ -402,6 +417,8 @@ const Serialize = struct { .producer = false, .generated = true, .dep_file = tag == .output_file_dep, + .target_query = false, + .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -409,6 +426,7 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, + .target_query = .{ .value = null }, }, .output_directory => |a| .{ .flags = .{ @@ -420,6 +438,8 @@ const Serialize = struct { .producer = false, .generated = true, .dep_file = false, + .target_query = false, + .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -427,6 +447,7 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, + .target_query = .{ .value = null }, }, .passthru => .{ .flags = .{ @@ -438,6 +459,8 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, + .target_query = false, + .link_libc = false, }, .prefix = .{ .value = null }, .suffix = .{ .value = null }, @@ -445,6 +468,28 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = null }, + .target_query = .{ .value = null }, + }, + .cc_args => |a| .{ + .flags = .{ + .tag = .cc_args, + .prefix = false, + .suffix = false, + .basename = false, + .path = false, + .producer = false, + .generated = false, + .dep_file = false, + .target_query = a.target_query != .none, + .link_libc = a.link_libc, + }, + .prefix = .{ .value = null }, + .suffix = .{ .value = null }, + .basename = .{ .value = null }, + .path = .{ .value = null }, + .producer = .{ .value = null }, + .generated = .{ .value = null }, + .target_query = .{ .value = a.target_query.unwrap() }, }, }))); } @@ -1234,7 +1279,7 @@ fn addOptionalResolvedTarget( ) !Configuration.ResolvedTarget.OptionalIndex { const resolved_target = optional_resolved_target orelse return .none; return @enumFromInt(try wc.addDeduped(@as(Configuration.ResolvedTarget, .{ - .query = try wc.addTargetQuery(resolved_target.query), + .query = try wc.addTargetQuery(&resolved_target.query), .result = try wc.addTarget(resolved_target.result), }))); } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index ad87bc246627a68fb3e769d9a8649d2dd3936edc..252d65f6cc3e0126657cec0001886a54a90668ac 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -129,13 +129,17 @@ pub const Graph = struct { /// /// Use of this function indicates a dependency on the host system. pub fn cwdRelativePath(graph: *Graph, sub_path: []const u8) LazyPath { + return @This().path(graph, .cwd, sub_path); + } + + /// A path whose components and contents are known at some point during + /// `Step` resolution, relative to the provided base directory. + pub fn path(graph: *Graph, base: Configuration.Path.Base, sub_path: []const u8) LazyPath { const wc = &graph.wip_configuration; - return .{ - .relative = .{ - .base = .cwd, - .sub_path = wc.addString(sub_path) catch @panic("OOM"), - }, - }; + return .{ .relative = .{ + .base = base, + .sub_path = wc.addString(sub_path) catch @panic("OOM"), + } }; } /// Allocates using the global process arena, failing the build on @@ -780,8 +784,10 @@ pub const AssemblyOptions = struct { /// it available to other packages which depend on this one. /// `createModule` can be used instead to create a private module. pub fn addModule(b: *Build, name: []const u8, options: Module.CreateOptions) *Module { + const graph = b.graph; + const arena = graph.arena; const module = Module.create(b, options); - b.modules.put(b.graph.arena, b.dupe(name), module) catch @panic("OOM"); + b.modules.put(arena, graph.dupeString(name), module) catch @panic("OOM"); return module; } @@ -913,13 +919,15 @@ pub fn addWriteFile(b: *Build, file_path: []const u8, data: []const u8) *Step.Wr } pub fn addNamedWriteFiles(b: *Build, name: []const u8) *Step.WriteFile { + const graph = b.graph; const wf = Step.WriteFile.create(b); - b.named_writefiles.put(b.graph.arena, b.dupe(name), wf) catch @panic("OOM"); + b.named_writefiles.put(graph.arena, graph.dupeString(name), wf) catch @panic("OOM"); return wf; } pub fn addNamedLazyPath(b: *Build, name: []const u8, lp: LazyPath) void { - b.named_lazy_paths.put(b.graph.arena, b.dupe(name), lp.dupe(b)) catch @panic("OOM"); + const graph = b.graph; + b.named_lazy_paths.put(graph.arena, graph.dupeString(name), lp.dupe(graph)) catch @panic("OOM"); } /// Creates a step for mutating files inside a temporary directory created lazily @@ -1183,16 +1191,18 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw } pub fn step(b: *Build, name: []const u8, description: []const u8) *Step { - const step_info = b.allocator.create(Step.TopLevel) catch @panic("OOM"); + const graph = b.graph; + const arena = graph.arena; + const step_info = arena.create(Step.TopLevel) catch @panic("OOM"); step_info.* = .{ .step = .init(.{ .tag = .top_level, .name = name, .owner = b, }), - .description = b.dupe(description), + .description = graph.dupeString(description), }; - const gop = b.top_level_steps.getOrPut(b.allocator, name) catch @panic("OOM"); + const gop = b.top_level_steps.getOrPut(arena, name) catch @panic("OOM"); if (gop.found_existing) panic("A top-level step with name \"{s}\" already exists", .{name}); gop.key_ptr.* = step_info.step.name; @@ -1302,6 +1312,9 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile /// Exposes standard `zig build` options for choosing a target. pub fn standardTargetOptionsQueryOnly(b: *Build, args: StandardTargetOptionsArgs) Target.Query { + const graph = b.graph; + const arena = graph.arena; + const maybe_triple = b.option( []const u8, "target", @@ -1350,20 +1363,22 @@ pub fn standardTargetOptionsQueryOnly(b: *Build, args: StandardTargetOptionsArgs for (whitelist) |q| { log.info("allowed target: -Dtarget={s} -Dcpu={s}", .{ - q.zigTriple(b.allocator) catch @panic("OOM"), - q.serializeCpuAlloc(b.allocator) catch @panic("OOM"), + q.zigTriple(arena) catch @panic("OOM"), + q.serializeCpuAlloc(arena) catch @panic("OOM"), }); } log.err("chosen target '{s}' does not match one of the allowed targets", .{ - selected_target.zigTriple(b.allocator) catch @panic("OOM"), + selected_target.zigTriple(arena) catch @panic("OOM"), }); b.markInvalidUserInput(); return args.default_target; } pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8) error{OutOfMemory}!bool { - const name = b.dupe(name_raw); - const value = b.dupe(value_raw); + const graph = b.graph; + const arena = graph.arena; + const name = graph.dupeString(name_raw); + const value = graph.dupeString(value_raw); const gop = try b.user_input_options.getOrPut(name); if (!gop.found_existing) { gop.value_ptr.* = UserInputOption{ @@ -1378,7 +1393,7 @@ pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8 switch (gop.value_ptr.value) { .scalar => |s| { // turn it into a list - var list = std.array_list.Managed([]const u8).init(b.allocator); + var list = std.array_list.Managed([]const u8).init(arena); try list.append(s); try list.append(value); try b.user_input_options.put(name, .{ @@ -1608,15 +1623,21 @@ pub fn pathList(b: *Build, sub_paths: []const []const u8) []const LazyPath { } pub fn pathJoin(b: *Build, paths: []const []const u8) []u8 { - return fs.path.join(b.allocator, paths) catch @panic("OOM"); + const graph = b.graph; + const arena = graph.arena; + return fs.path.join(arena, paths) catch @panic("OOM"); } pub fn pathResolve(b: *Build, paths: []const []const u8) []u8 { - return fs.path.resolve(b.allocator, paths) catch @panic("OOM"); + const graph = b.graph; + const arena = graph.arena; + return fs.path.resolve(arena, paths) catch @panic("OOM"); } pub fn fmt(b: *Build, comptime format: []const u8, args: anytype) []u8 { - return std.fmt.allocPrint(b.allocator, format, args) catch @panic("OOM"); + const graph = b.graph; + const arena = graph.arena; + return std.fmt.allocPrint(arena, format, args) catch @panic("OOM"); } /// Creates an anonymous `Step` that searches for an executable on the host that @@ -2264,7 +2285,9 @@ pub const LazyPath = union(enum) { } pub fn path(lazy_path: LazyPath, b: *Build, sub_path: []const u8) LazyPath { - return lazy_path.join(b.allocator, sub_path) catch @panic("OOM"); + const graph = b.graph; + const arena = graph.arena; + return lazy_path.join(arena, sub_path) catch @panic("OOM"); } pub fn join(lazy_path: LazyPath, arena: Allocator, sub_path: []const u8) Allocator.Error!LazyPath { @@ -2460,7 +2483,9 @@ pub fn systemIntegrationOption( name: []const u8, config: SystemIntegrationOptionConfig, ) bool { - const gop = b.graph.system_integration_options.getOrPut(b.allocator, name) catch @panic("OOM"); + const graph = b.graph; + const arena = graph.arena; + const gop = graph.system_integration_options.getOrPut(arena, name) catch @panic("OOM"); if (gop.found_existing) switch (gop.value_ptr.*) { .user_disabled => { gop.value_ptr.* = .declared_disabled; @@ -2473,8 +2498,8 @@ pub fn systemIntegrationOption( .declared_disabled => return false, .declared_enabled => return true, } else { - gop.key_ptr.* = b.dupe(name); - if (config.default orelse b.graph.system_package_mode) { + gop.key_ptr.* = graph.dupeString(name); + if (config.default orelse graph.system_package_mode) { gop.value_ptr.* = .declared_enabled; return true; } else { diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index c008dcc90f31e445ea8b4701edf7ac0507e8e460..e79d9c58dc54376a649864e13e0e699a38c80723 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -230,7 +230,7 @@ pub const Wip = struct { return addString(wip, writer.buffered()); } - pub fn addTargetQuery(wip: *Wip, q: std.Target.Query) !TargetQuery.OptionalIndex { + pub fn addTargetQuery(wip: *Wip, q: *const std.Target.Query) !TargetQuery.OptionalIndex { if (q.isNative()) return .none; const gpa = wip.gpa; const cpu_name: ?String = switch (q.cpu_model) { @@ -575,6 +575,7 @@ pub const Step = extern struct { /// Always a compile step. producer: Storage.FlagOptional(.flags, .producer, Step.Index), generated: Storage.FlagOptional(.flags, .generated, GeneratedFileIndex), + target_query: Storage.FlagOptional(.flags, .target_query, TargetQuery.Index), pub const Flags = packed struct(u32) { tag: Arg.Tag, @@ -585,10 +586,12 @@ pub const Step = extern struct { producer: bool, generated: bool, dep_file: bool, - _: u22 = 0, + target_query: bool, + link_libc: bool, + _: u19 = 0, }; - pub const Tag = enum(u3) { + pub const Tag = enum(u4) { artifact, /// `path` contains the file. path_file, @@ -599,6 +602,7 @@ pub const Step = extern struct { output_file, output_directory, passthru, + cc_args, }; pub const Index = IndexType(@This()); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 35699051fbfd511c399cb7033e613abebd9cf9d5..c2c5a4f7a38e8217d7e79e4414abbab5d3451b4f 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -143,6 +143,13 @@ pub const Arg = union(enum) { output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. passthru, + /// Adds standard "-isystem" and "-iframework" arguments corresponding to the libc of the target. + cc_args: CcArgs, +}; + +pub const CcArgs = struct { + link_libc: bool, + target_query: Configuration.TargetQuery.OptionalIndex, }; pub const PrefixedArtifact = struct { @@ -529,6 +536,22 @@ pub fn addPassthruArgs(run: *Run) void { run.argv.append(arena, .passthru) catch @panic("OOM"); } +pub const AddCcArgs = struct { + link_libc: bool = false, + target_query: ?*const std.Target.Query = null, +}; + +/// Appends C compiler flags for the target and for including libc. +pub fn addCcArgs(run: *Run, options: AddCcArgs) void { + const graph = run.step.owner.graph; + const arena = graph.arena; + const wc = &graph.wip_configuration; + run.argv.append(arena, .{ .cc_args = .{ + .link_libc = options.link_libc, + .target_query = if (options.target_query) |q| wc.addTargetQuery(q) catch @panic("OOM") else .none, + } }) catch @panic("OOM"); +} + pub fn setStdIn(run: *Run, stdin: StdIn) void { switch (stdin) { .lazy_path => |lazy_path| lazy_path.addStepDependencies(&run.step), -- 2.54.0 From 5fb120a3c0e1b252fb85e8259b481c28b9dba465 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 20:16:56 -0700 Subject: [PATCH 104/179] maker: implement cleanTmpFiles --- lib/compiler/Maker.zig | 23 +++++++++++++---------- lib/std/Build/Configuration.zig | 8 ++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index dcc8cb21f72b8b572cac85e45aefea69b7893a96..e8dff14a4611f23a80637129c6e967addccf5453 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -833,7 +833,7 @@ fn makeStepNames( var pending_count: usize = 0; var total_compile_errors: usize = 0; - var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); + var cleanup_task = io.async(cleanTmpFiles, .{ maker, step_stack.keys() }); defer cleanup_task.await(io); for (step_stack.keys()) |step_index| { @@ -1677,17 +1677,20 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { fatal(f, args); } -fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { - std.log.err("TODO implement cleanTmpFiles", .{}); - if (true) return; +fn cleanTmpFiles(maker: *Maker, steps: []const Configuration.Step.Index) void { + const graph = maker.graph; + const io = graph.io; + const conf = &maker.scanned_config.configuration; for (steps) |step_index| { - const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; - if (wf.mode != .tmp) continue; - const path = wf.generated_directory.path orelse continue; - Dir.cwd().deleteTree(io, path) catch |err| { - log.warn("failed to delete {s}: {t}", .{ path, err }); - }; + const conf_step = step_index.ptr(conf); + const wf = conf_step.extended.cast(conf, Configuration.Step.WriteFile) orelse continue; + if (wf.flags.mode != .tmp) continue; + const step = maker.stepByIndex(step_index); + if (step.state != .success) continue; + const tmp_path = generatedPath(maker, wf.generated_directory).*; + tmp_path.root_dir.handle.deleteTree(io, tmp_path.subPathOrDot()) catch |err| + log.warn("failed to delete temporary path {f}: {t}", .{ tmp_path, err }); } } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index e79d9c58dc54376a649864e13e0e699a38c80723..34cd8c62e7940f616637d37074cb28c6fe3dd029 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -2513,6 +2513,14 @@ pub const Storage = enum { return base_flags.tag; } + pub fn cast(this: @This(), c: *const Configuration, comptime S: type) ?S { + const wanted_tag = @typeInfo(S.Flags).@"struct".fields[0].defaultValue().?; + const base_flags: BaseFlags = @bitCast(c.extra[@intFromEnum(this)]); + if (base_flags.tag != wanted_tag) return null; + var i: usize = @intFromEnum(this); + return data(c.extra, &i, S); + } + pub fn get(this: @This(), buffer: []const u32) U { var i: usize = @intFromEnum(this); const base_flags: BaseFlags = @bitCast(buffer[i]); -- 2.54.0 From d9881466382093f3c29ebe3f36a7d09d72407837 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 20:59:36 -0700 Subject: [PATCH 105/179] std.process.Child: add format and success methods --- lib/std/process.zig | 7 +++++++ lib/std/process/Child.zig | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/lib/std/process.zig b/lib/std/process.zig index 9869bb2593ff21c864f599f95e9c2ae709ab4bab..bcee558a65eef07afab17b484cc012326bb5004c 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -1113,3 +1113,10 @@ test protectMemory { protectMemory(&test_page, .{}) catch return error.SkipZigTest; protectMemory(&test_page, .{ .read = true, .write = true }) catch return error.SkipZigTest; } + +test { + _ = Child; + _ = Args; + _ = Environ; + _ = Preopens; +} diff --git a/lib/std/process/Child.zig b/lib/std/process/Child.zig index 65f6429e59f399fcb34137587d3f73ac9a35c24b..a3cba3e7ece8ff9e34c9998bb47ff816e2586e5e 100644 --- a/lib/std/process/Child.zig +++ b/lib/std/process/Child.zig @@ -96,6 +96,22 @@ pub const Term = union(enum) { signal: std.posix.SIG, stopped: std.posix.SIG, unknown: u32, + + pub fn success(t: Term) bool { + return switch (t) { + .exited => |code| code == 0, + else => false, + }; + } + + pub fn format(t: Term, w: *Io.Writer) Io.Writer.Error!void { + switch (t) { + .exited => |code| return w.print("exited with code {d}", .{code}), + .signal => |sig| return w.print("terminated with signal {t}", .{sig}), + .stopped => |sig| return w.print("stopped with signal {t}", .{sig}), + .unknown => return w.writeAll("terminated unexpectedly"), + } + } }; pub const Cwd = union(enum) { @@ -135,3 +151,7 @@ pub fn wait(child: *Child, io: Io) WaitError!Term { assert(child.id != null); return io.vtable.childWait(io.userdata, child); } + +test { + _ = Term; +} -- 2.54.0 From 4aa8fa898de3847f3a0469edf4484f60c3ecf051 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 21:00:11 -0700 Subject: [PATCH 106/179] Maker.PkgConfig: fix regression when pkg-config not found unless pkg_config == .force, this is supposed to be allowed --- lib/compiler/Maker/PkgConfig.zig | 38 +++++++++++++++---------- lib/compiler/Maker/Step.zig | 48 ++++++++++++++++++-------------- lib/compiler/Maker/Step/Fmt.zig | 10 ++++++- 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/lib/compiler/Maker/PkgConfig.zig b/lib/compiler/Maker/PkgConfig.zig index 567bad7061e92d1237f06f292e48b3965c01cc01..cafcf7c492e3be5986beb62e4b00759a6b01aa83 100644 --- a/lib/compiler/Maker/PkgConfig.zig +++ b/lib/compiler/Maker/PkgConfig.zig @@ -1,6 +1,7 @@ const std = @import("std"); const Io = std.Io; const mem = std.mem; +const assert = std.debug.assert; const Maker = @import("../Maker.zig"); const Step = @import("Step.zig"); @@ -92,14 +93,15 @@ pub fn run( }; const pkg_config_exe = getExe(graph); - const captured = try step.captureChildProcess(maker, progress_node, &.{ - pkg_config_exe, pkg_name, "--cflags", "--libs", + const stdout = try captureChildProcess(maker, step, .{ + .argv = &.{ pkg_config_exe, pkg_name, "--cflags", "--libs" }, + .progress_node = progress_node, + .allow_failure = !force, }); - try step.handleChildProcessTerm(maker, captured.term); var zig_cflags: std.ArrayList([]const u8) = .empty; var zig_libs: std.ArrayList([]const u8) = .empty; - var arg_it = mem.tokenizeAny(u8, captured.stdout, " \r\n\t"); + var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); while (arg_it.next()) |arg| { if (mem.eql(u8, arg, "-I")) { @@ -168,19 +170,14 @@ fn getList(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force: if (pc.list) |list| return list; const pkg_config_exe = getExe(graph); - const captured = try step.captureChildProcess(maker, progress_node, &.{ pkg_config_exe, "--list-all" }); - if (force) { - try step.handleChildProcessTerm(maker, captured.term); - } else switch (captured.term) { - .exited => |code| if (code != 0) return error.PkgConfigUnavailable, - else => { - try step.handleChildProcessTerm(maker, captured.term); - unreachable; - }, - } + const stdout = try captureChildProcess(maker, step, .{ + .argv = &.{ pkg_config_exe, "--list-all" }, + .progress_node = progress_node, + .allow_failure = !force, + }); var list: std.ArrayList(Pkg) = .empty; - var line_it = mem.tokenizeAny(u8, captured.stdout, "\r\n"); + var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); while (line_it.next()) |line| { if (mem.trim(u8, line, " \t").len == 0) continue; var tok_it = mem.tokenizeAny(u8, line, " \t"); @@ -200,3 +197,14 @@ fn getList(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force: pc.list = result; return result; } + +fn captureChildProcess(maker: *Maker, step: *Step, options: Step.CaptureChildProcessOptions) ![]const u8 { + const captured = step.captureChildProcess(maker, options) catch |err| switch (err) { + error.FileNotFound => return error.PkgConfigUnavailable, + else => |e| return e, + }; + assert(step.result_failed_command != null); + if (captured.term.success()) return captured.stdout; + if (!options.allow_failure) return step.fail(maker, "{s} {f}", .{ options.argv[0], captured.term }); + return error.PkgConfigUnavailable; +} diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 59fdabcfaa90ee2b5f78bc9118cdf4a980468df9..ddf60275d2c5c8cbc127d9f1a7c86c416d1b27af 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -329,13 +329,19 @@ pub fn reset(step: *Step, maker: *Maker) void { step.result_error_bundle = std.zig.ErrorBundle.empty; } -/// Populates `s.result_failed_command`. -pub fn captureChildProcess( - s: *Step, - maker: *Maker, - progress_node: std.Progress.Node, +pub const CaptureChildProcessError = error{ + FileNotFound, +} || ExtendedMakeError; + +pub const CaptureChildProcessOptions = struct { argv: []const []const u8, -) !std.process.RunResult { + progress_node: std.Progress.Node = .none, + environ_map: ?*const std.process.Environ.Map = null, + allow_failure: bool = false, +}; + +/// Populates `s.result_failed_command`. +pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcessOptions) !std.process.RunResult { const gpa = maker.gpa; const graph = maker.graph; const arena = graph.arena; // TODO stop leaking into process arena @@ -343,20 +349,25 @@ pub fn captureChildProcess( // If an error occurs, it's happened in this command: assert(s.result_failed_command == null); - s.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{}); + s.result_failed_command = try std.zig.allocPrintCmd(gpa, options.argv, .{}); try handleChildProcUnsupported(s, maker); - try graph.handleVerbose(.inherit, null, argv); + try graph.handleVerbose(.inherit, null, options.argv); const result = std.process.run(arena, io, .{ - .argv = argv, - .environ_map = &graph.environ_map, - .progress_node = progress_node, - }) catch |err| return s.fail(maker, "failed to run {s}: {t}", .{ argv[0], err }); + .argv = options.argv, + .environ_map = options.environ_map orelse &graph.environ_map, + .progress_node = options.progress_node, + }) catch |err| { + switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.FileNotFound => |e| if (options.allow_failure) return e, + else => {}, + } + return s.fail(maker, "failed to run {s}: {t}", .{ options.argv[0], err }); + }; - if (result.stderr.len > 0) { - try s.result_error_msgs.append(arena, result.stderr); - } + if (result.stderr.len > 0) try s.result_error_msgs.append(arena, result.stderr); return result; } @@ -679,12 +690,7 @@ pub inline fn handleChildProcUnsupported(s: *Step, maker: *Maker) FailError!void /// Asserts that the caller has already populated `s.result_failed_command`. pub fn handleChildProcessTerm(s: *Step, maker: *Maker, term: std.process.Child.Term) FailError!void { assert(s.result_failed_command != null); - return switch (term) { - .exited => |code| if (code != 0) s.fail(maker, "process exited with error code {d}", .{code}), - .signal => |sig| s.fail(maker, "process terminated with signal {t}", .{sig}), - .stopped => |sig| s.fail(maker, "process stopped with signal {t}", .{sig}), - .unknown => s.fail(maker, "process terminated unexpectedly", .{}), - }; + if (!term.success()) return s.fail(maker, "process {f}", .{term}); } /// Prefer `cacheHitAndWatch` unless you already added watch inputs diff --git a/lib/compiler/Maker/Step/Fmt.zig b/lib/compiler/Maker/Step/Fmt.zig index addd39a969491bf74c51b0db20235cfa26b18196..281686bed57d488d5db088d9f1c2593edf4f6371 100644 --- a/lib/compiler/Maker/Step/Fmt.zig +++ b/lib/compiler/Maker/Step/Fmt.zig @@ -43,7 +43,15 @@ pub fn make( argv.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, step_index)); } - const run_result = try step.captureChildProcess(maker, progress_node, argv.items); + const run_result = step.captureChildProcess(maker, .{ + .progress_node = progress_node, + .argv = argv.items, + .allow_failure = false, + }) catch |err| switch (err) { + error.FileNotFound => unreachable, + else => |e| return e, + }; + if (conf_fmt.flags.check) switch (run_result.term) { .exited => |code| if (code != 0 and run_result.stdout.len != 0) { var it = std.mem.tokenizeScalar(u8, run_result.stdout, '\n'); -- 2.54.0 From 398ea7e492b7016dcb6cebfe6656876002f5bf73 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 17 May 2026 21:30:08 -0700 Subject: [PATCH 107/179] Maker: handle fallible child proc capture gracefully --- lib/compiler/Maker/Step.zig | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index ddf60275d2c5c8cbc127d9f1a7c86c416d1b27af..10de4ccb378f9648dbfab5f9bc6d523cbc6905f3 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -59,6 +59,7 @@ result_duration_ns: ?u64 = null, result_peak_rss: usize = 0, /// If the step is failed and this field is populated, this is the command which failed. /// This field may be populated even if the step succeeded. +/// Memory owned by `Maker.gpa`. result_failed_command: ?[]const u8 = null, test_results: TestResults = .{}, @@ -313,14 +314,13 @@ pub fn reset(step: *Step, maker: *Maker) void { assert(step.state == .precheck_done); const gpa = maker.gpa; - if (step.result_failed_command) |cmd| gpa.free(cmd); + clearFailedCommand(step, gpa); step.result_error_msgs.clearRetainingCapacity(); step.result_stderr = ""; step.result_cached = false; step.result_duration_ns = null; step.result_peak_rss = 0; - step.result_failed_command = null; step.test_results = .{}; // We do not clearWatchInputs here because each step manages that choice // independently. @@ -347,8 +347,7 @@ pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcess const arena = graph.arena; // TODO stop leaking into process arena const io = graph.io; - // If an error occurs, it's happened in this command: - assert(s.result_failed_command == null); + clearFailedCommand(s, gpa); s.result_failed_command = try std.zig.allocPrintCmd(gpa, options.argv, .{}); try handleChildProcUnsupported(s, maker); @@ -372,6 +371,11 @@ pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcess return result; } +fn clearFailedCommand(s: *Step, gpa: Allocator) void { + if (s.result_failed_command) |cmd| gpa.free(cmd); + s.result_failed_command = null; +} + pub const FailError = error{ OutOfMemory, MakeFailed }; pub fn fail(step: *Step, maker: *const Maker, comptime fmt: []const u8, args: anytype) FailError { @@ -421,7 +425,7 @@ pub fn evalZigProcess( const io = graph.io; // If an error occurs, it's happened in this command: - assert(s.result_failed_command == null); + clearFailedCommand(s, gpa); s.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{}); if (s.getZigProcess()) |zp| update: { -- 2.54.0 From db7ceada15b007747bfd433e6635324edcec918e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 18 May 2026 20:34:51 -0700 Subject: [PATCH 108/179] Maker: implement Step.WriteFile --- lib/compiler/Maker/Step.zig | 35 +- lib/compiler/Maker/Step/UpdateSourceFiles.zig | 12 +- lib/compiler/Maker/Step/WriteFile.zig | 333 +++++++++++------- 3 files changed, 232 insertions(+), 148 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 10de4ccb378f9648dbfab5f9bc6d523cbc6905f3..92cdfdb957a2729b3b4817335dd0588de39e5172 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -26,6 +26,7 @@ pub const ObjCopy = @import("Step/ObjCopy.zig"); pub const Options = @import("Step/Options.zig"); pub const Run = @import("Step/Run.zig"); pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); +pub const WriteFile = @import("Step/WriteFile.zig"); /// Avoid false sharing. _: void align(std.atomic.cache_line) = {}, @@ -87,7 +88,7 @@ pub const Extended = union(enum) { top_level: TopLevel, translate_c: Todo, update_source_files: UpdateSourceFiles, - write_file: Todo, + write_file: WriteFile, pub fn init(tag: Configuration.Step.Tag) Extended { return switch (tag) { @@ -119,9 +120,10 @@ pub const Extended = union(enum) { progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { _ = todo; - _ = maker; _ = progress_node; - std.debug.panic("TODO implement another step type (index {d})", .{step_index}); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + std.debug.panic("TODO implement another step type: {s}", .{conf_step.name.slice(conf)}); } }; @@ -807,19 +809,17 @@ pub fn addWatchInput(step: *Step, maker: *Maker, arena: Allocator, lazy_file: La /// Paths derived from this directory should also be manually added via /// `addDirectoryWatchInputFromPath` if and only if this function returns /// `true`. -pub fn addDirectoryWatchInput(step: *Step, lazy_directory: LazyPath) Allocator.Error!bool { +pub fn addDirectoryWatchInput(step: *Step, maker: *Maker, lazy_directory: LazyPath) Allocator.Error!bool { switch (lazy_directory) { - .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), - .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), - .cwd_relative => |path_string| { - try addDirectoryWatchInputFromPath(step, .{ - .root_dir = .{ - .path = null, - .handle = .cwd(), - }, - .sub_path = path_string, - }); + .source_path => |source_path| { + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + const sub_path = source_path.sub_path.slice(conf); + const pkg_path = try maker.packagePath(arena, source_path.owner, sub_path); + try addDirectoryWatchInputFromPath(step, maker, pkg_path); }, + .relative => |relative| try addDirectoryWatchInputFromPath(step, maker, maker.relativePath(relative)), // Nothing to watch because this dependency edge is modeled instead via `dependants`. .generated => return false, } @@ -838,13 +838,6 @@ pub fn addDirectoryWatchInputFromPath(step: *Step, maker: *Maker, path: Path) !v return addWatchInputFromPath(step, maker, path, "."); } -fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { - return addDirectoryWatchInputFromPath(step, .{ - .root_dir = package.build_root, - .sub_path = sub_path, - }); -} - fn addWatchInputPath(step: *Step, maker: *Maker, path: Path) Allocator.Error!void { return addWatchInputFromPath(step, maker, .{ .root_dir = path.root_dir, diff --git a/lib/compiler/Maker/Step/UpdateSourceFiles.zig b/lib/compiler/Maker/Step/UpdateSourceFiles.zig index 744fd35341e3c9d31fa9b5d4db2782011ddc2fbf..d33ff091d157bfcc601031253e6310f04c18a38a 100644 --- a/lib/compiler/Maker/Step/UpdateSourceFiles.zig +++ b/lib/compiler/Maker/Step/UpdateSourceFiles.zig @@ -32,6 +32,8 @@ pub fn make( progress_node.setEstimatedTotalItems(conf_usf.embeds.slice.len + conf_usf.copies.slice.len); + step.clearWatchInputs(maker); + for (conf_usf.embeds.slice) |*embed| { const dest_path: Path = .{ .root_dir = build_root, @@ -43,12 +45,12 @@ pub fn make( .sub_path = dirname, }; dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| - return step.fail(maker, "failed to create path {f}: {t}", .{ dirname_path, err }); + return step.fail(maker, "failed creating path {f}: {t}", .{ dirname_path, err }); } dest_path.root_dir.handle.writeFile(io, .{ .sub_path = dest_path.sub_path, .data = embed.contents.slice(conf), - }) catch |err| return step.fail(maker, "failed to write file {f}: {t}", .{ dest_path, err }); + }) catch |err| return step.fail(maker, "failed writing file {f}: {t}", .{ dest_path, err }); any_miss = true; progress_node.completeOne(); } @@ -64,11 +66,11 @@ pub fn make( .sub_path = dirname, }; dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| - return step.fail(maker, "failed to create path {f}: {t}", .{ dirname_path, err }); + return step.fail(maker, "failed creating path {f}: {t}", .{ dirname_path, err }); } const src_lazy_path = copy.src_file.get(conf); const source_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); - if (!step.inputs.populated()) try step.addWatchInput(maker, arena, src_lazy_path); + try step.addWatchInput(maker, arena, src_lazy_path); const prev_status = source_path.root_dir.handle.updateFile( io, @@ -76,7 +78,7 @@ pub fn make( dest_path.root_dir.handle, dest_path.sub_path, .{}, - ) catch |err| return step.fail(maker, "unable to update file from {f} to {f}: {t}", .{ + ) catch |err| return step.fail(maker, "failed updating file from {f} to {f}: {t}", .{ source_path, dest_path, err, }); any_miss = any_miss or prev_status == .stale; diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index e392a56b01cf196db1a6cf11a45e419bb9470e53..4be41d526b16d2cff25ef40fbc825a5f2f3724ed 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -1,205 +1,294 @@ -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const graph = b.graph; +const WriteFile = @This(); + +const std = @import("std"); +const Io = std.Io; +const assert = std.debug.assert; +const Path = std.Build.Cache.Path; +const allocPrint = std.fmt.allocPrint; +const Configuration = std.Build.Configuration; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + wf: *WriteFile, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + _ = wf; + const graph = maker.graph; + const gpa = maker.gpa; + const arena = maker.graph.arena; // TODO don't leak into process arena const io = graph.io; - const arena = b.allocator; - const gpa = graph.cache.gpa; - const write_file: *WriteFile = @fieldParentPtr("step", step); + const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_wf = conf_step.extended.get(conf.extra).write_file; + const cache_root = graph.local_cache_root; + const directories = conf_wf.directories.slice; - const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); - var open_dirs_count: usize = 0; + const open_dir_cache = try arena.alloc(Io.Dir, directories.len); + var open_dirs_count: u32 = 0; defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); - switch (write_file.mode) { + // Doesn't yet include contents of directories. + var total_items: usize = conf_wf.embeds.slice.len + conf_wf.copies.slice.len + conf_wf.directories.slice.len; + progress_node.setEstimatedTotalItems(total_items); + + switch (conf_wf.flags.mode) { .whole_cached => { - step.clearWatchInputs(); + step.clearWatchInputs(maker); - // The cache is used here not really as a way to speed things up - because writing - // the data to a file would probably be very fast - but as a way to find a canonical - // location to put build artifacts. + // The cache is used here primarily as a way to find a canonical + // location to put build artifacts without parallel step execution + // clobbering each other. - // If, for example, a hard-coded path was used as the location to put WriteFile - // files, then two WriteFiles executing in parallel might clobber each other. - - var man = b.graph.cache.obtain(); + var man = graph.cache.obtain(); defer man.deinit(); - for (write_file.files.items) |file| { - man.hash.addBytes(file.sub_path); + for (conf_wf.embeds.slice) |*embed| { + man.hash.addBytes(embed.sub_path.slice(conf)); + man.hash.addBytes(embed.contents.slice(conf)); + } - switch (file.contents) { - .bytes => |bytes| { - man.hash.addBytes(bytes); - }, - .copy => |lazy_path| { - const path = lazy_path.getPath3(b, step); - _ = try man.addFilePath(path, null); - try step.addWatchInput(lazy_path); - }, - } + for (conf_wf.copies.slice) |*copy| { + man.hash.addBytes(copy.sub_path.slice(conf)); + const src_lazy_path = copy.src_file.get(conf); + const source_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); + _ = try man.addFilePath(source_path, null); + try step.addWatchInput(maker, arena, src_lazy_path); } - for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { - man.hash.addBytes(dir.sub_path); - for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); - if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); + for (directories, open_dir_cache) |conf_dir, *opened_dir| { + const exclude_extensions = conf_dir.exclude_extensions.slice(conf) orelse &.{}; + const include_extensions = conf_dir.include_extensions.slice(conf); + + man.hash.addBytes(conf_dir.sub_path.slice(conf)); + for (exclude_extensions) |ext| man.hash.addBytes(ext.slice(conf)); + if (include_extensions) |includes| for (includes) |inc| { + man.hash.addBytes(inc.slice(conf)); + }; - const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); - const src_dir_path = dir.source.getPath3(b, step); + const src_lazy_path = conf_dir.src_path.get(conf); + const need_derived_inputs = try step.addDirectoryWatchInput(maker, src_lazy_path); + const src_dir_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index); var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {s}", .{ - src_dir_path, @errorName(err), - }); + return step.fail(maker, "failed opening source directory {f}: {t}", .{ src_dir_path, err }); }; - open_dir_cache_elem.* = src_dir; + opened_dir.* = src_dir; open_dirs_count += 1; var it = try src_dir.walk(gpa); defer it.deinit(); - while (try it.next(io)) |entry| { - if (!dir.options.pathIncluded(entry.path)) continue; + while (it.next(io) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return step.fail(maker, "failed iterating dir {f}: {t}", .{ src_dir_path, e }), + }) |entry| { + if (!pathIncluded(conf, exclude_extensions, include_extensions, entry.path)) continue; switch (entry.kind) { .directory => { if (need_derived_inputs) { const entry_path = try src_dir_path.join(arena, entry.path); - try step.addDirectoryWatchInputFromPath(entry_path); + try step.addDirectoryWatchInputFromPath(maker, entry_path); } }, .file => { const entry_path = try src_dir_path.join(arena, entry.path); _ = try man.addFilePath(entry_path, null); + total_items += 1; }, else => continue, } } } - if (try step.cacheHit(&man)) { + if (try step.cacheHit(maker, &man)) { const digest = man.final(); - write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); + maker.generatedPath(conf_wf.generated_directory).* = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), + }; assert(step.result_cached); return; } const digest = man.final(); - const cache_path = "o" ++ Dir.path.sep_str ++ digest; + const out_path: Path = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), + }; - write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); + progress_node.setEstimatedTotalItems(total_items); + try operate(maker, step_index, open_dir_cache, out_path, progress_node); + try step.writeManifest(maker, &man); - try operate(write_file, open_dir_cache, .{ - .root_dir = b.cache_root, - .sub_path = cache_path, - }); - - try step.writeManifest(&man); + maker.generatedPath(conf_wf.generated_directory).* = out_path; }, .tmp => { step.result_cached = false; var rand_int: u64 = undefined; io.random(@ptrCast(&rand_int)); - const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + const hex_digest = std.fmt.hex(rand_int); - write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); + const out_path: Path = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "tmp", &hex_digest }), + }; - try operate(write_file, open_dir_cache, .{ - .root_dir = b.cache_root, - .sub_path = tmp_dir_sub_path, - }); + try operate(maker, step_index, open_dir_cache, out_path, progress_node); + + maker.generatedPath(conf_wf.generated_directory).* = out_path; }, - .mutate => |lp| { + .mutate => { step.result_cached = false; - const root_path = try lp.getPath4(b, step); - write_file.generated_directory.path = try root_path.toString(arena); - try operate(write_file, open_dir_cache, root_path); + const root_path = try maker.resolveLazyPathIndex(arena, conf_wf.mutate_path.value.?, step_index); + try operate(maker, step_index, open_dir_cache, root_path, progress_node); + maker.generatedPath(conf_wf.generated_directory).* = root_path; }, } } -fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { - const step = &write_file.step; - const b = step.owner; - const io = b.graph.io; - const gpa = b.graph.cache.gpa; - const arena = b.allocator; +fn operate( + maker: *Maker, + step_index: Configuration.Step.Index, + open_dir_cache: []const Io.Dir, + root_path: std.Build.Cache.Path, + progress_node: std.Progress.Node, +) !void { + const graph = maker.graph; + const gpa = maker.gpa; + const arena = maker.graph.arena; // TODO don't leak into process arena + const io = graph.io; + const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_wf = conf_step.extended.get(conf.extra).write_file; - var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| - return step.fail("unable to make path {f}: {t}", .{ root_path, err }); - defer cache_dir.close(io); + const root_directory: std.Build.Cache.Directory = .{ + .handle = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| + return step.fail(maker, "failed creating path {f}: {t}", .{ root_path, err }), + .path = try root_path.toString(arena), + }; + defer root_directory.handle.close(io); - for (write_file.files.items) |file| { - if (Dir.path.dirname(file.sub_path)) |dirname| { - cache_dir.createDirPath(io, dirname) catch |err| { - return step.fail("unable to make path '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, dirname, err, - }); + for (conf_wf.embeds.slice) |*embed| { + const dest_path: Path = .{ + .root_dir = root_directory, + .sub_path = embed.sub_path.slice(conf), + }; + if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| { + const dirname_path: Path = .{ + .root_dir = root_directory, + .sub_path = dirname, }; + dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| + return step.fail(maker, "failed creating path {f}: {t}", .{ dirname_path, err }); } - switch (file.contents) { - .bytes => |bytes| { - cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, file.sub_path, err, - }); - }; - }, - .copy => |file_source| { - const source_path = file_source.getPath2(b, step); - const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { - return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ - source_path, root_path, Dir.path.sep, file.sub_path, err, - }); - }; - // At this point we already will mark the step as a cache miss. - // But this is kind of a partial cache hit since individual - // file copies may be avoided. Oh well, this information is - // discarded. - _ = prev_status; - }, + dest_path.root_dir.handle.writeFile(io, .{ + .sub_path = dest_path.sub_path, + .data = embed.contents.slice(conf), + }) catch |err| return step.fail(maker, "failed writing contents to file {f}: {t}", .{ dest_path, err }); + progress_node.completeOne(); + } + + for (conf_wf.copies.slice) |*copy| { + const dest_path: Path = .{ + .root_dir = root_directory, + .sub_path = copy.sub_path.slice(conf), + }; + // Rather than passing make_path = true below, this optimizes for the + // more common case where the directory does not exist. + if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| { + const dirname_path: Path = .{ + .root_dir = root_directory, + .sub_path = dirname, + }; + dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err| + return step.fail(maker, "failed creating path {f}: {t}", .{ dirname_path, err }); } + const source_path = try maker.resolveLazyPathIndex(arena, copy.src_file, step_index); + Io.Dir.copyFile( + source_path.root_dir.handle, + source_path.sub_path, + dest_path.root_dir.handle, + dest_path.sub_path, + io, + .{}, + ) catch |err| return step.fail(maker, "failed copying file from {f} to {f}: {t}", .{ + source_path, dest_path, err, + }); + progress_node.completeOne(); } - for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { - const src_dir_path = dir.source.getPath3(b, step); - const dest_dirname = dir.sub_path; + for (conf_wf.directories.slice, open_dir_cache) |conf_dir, already_open_dir| { + const exclude_extensions = conf_dir.exclude_extensions.slice(conf) orelse &.{}; + const include_extensions = conf_dir.include_extensions.slice(conf); - if (dest_dirname.len != 0) { - cache_dir.createDirPath(io, dest_dirname) catch |err| { - return step.fail("unable to make path '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, dest_dirname, err, - }); - }; + const src_dir_path = try maker.resolveLazyPathIndex(arena, conf_dir.src_path, step_index); + const dest_dir_path: Path = .{ + .root_dir = root_directory, + .sub_path = conf_dir.sub_path.slice(conf), + }; + + if (dest_dir_path.sub_path.len != 0) { + dest_dir_path.root_dir.handle.createDirPath(io, dest_dir_path.sub_path) catch |err| + return step.fail(maker, "failed creating path {f}: {t}", .{ dest_dir_path, err }); } var it = try already_open_dir.walk(gpa); defer it.deinit(); - while (try it.next(io)) |entry| { - if (!dir.options.pathIncluded(entry.path)) continue; + while (it.next(io) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return step.fail(maker, "failed iterating dir {f}: {t}", .{ src_dir_path, e }), + }) |entry| { + if (!pathIncluded(conf, exclude_extensions, include_extensions, entry.path)) continue; const src_entry_path = try src_dir_path.join(arena, entry.path); - const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); + const dest_path = try dest_dir_path.join(arena, entry.path); switch (entry.kind) { - .directory => try cache_dir.createDirPath(io, dest_path), + .directory => dest_path.root_dir.handle.createDirPath(io, dest_path.sub_path) catch |err| { + return step.fail(maker, "failed creating path {f}: {t}", .{ dest_path, err }); + }, .file => { - const prev_status = Io.Dir.updateFile( + Io.Dir.copyFile( src_entry_path.root_dir.handle, - io, src_entry_path.sub_path, - cache_dir, - dest_path, + dest_path.root_dir.handle, + dest_path.sub_path, + io, .{}, - ) catch |err| { - return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ - src_entry_path, root_path, Dir.path.sep, dest_path, err, - }); - }; - _ = prev_status; + ) catch |err| return step.fail(maker, "failed copying file from {f} to {f}: {t}", .{ + src_entry_path, dest_path, err, + }); + progress_node.completeOne(); }, else => continue, } } } } + +fn pathIncluded( + conf: *const Configuration, + exclude_extensions: []const Configuration.String, + include_extensions: ?[]const Configuration.String, + path: []const u8, +) bool { + for (exclude_extensions) |ext| { + if (std.mem.endsWith(u8, path, ext.slice(conf))) + return false; + } + if (include_extensions) |incs| { + for (incs) |inc| { + if (std.mem.endsWith(u8, path, inc.slice(conf))) + return true; + } else { + return false; + } + } + return true; +} -- 2.54.0 From bb1b59ee1feecd5500847d394fc90842efe138ed Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 18 May 2026 21:58:19 -0700 Subject: [PATCH 109/179] Maker: implement Step.InstallDir --- BRANCH_TODO | 1 + lib/compiler/Maker.zig | 45 +++++++++++++-- lib/compiler/Maker/Step.zig | 5 +- lib/compiler/Maker/Step/InstallDir.zig | 78 ++++++++++++++++++-------- lib/compiler/configurer.zig | 2 +- lib/std/Build.zig | 17 ------ lib/std/Build/Configuration.zig | 14 +---- 7 files changed, 103 insertions(+), 59 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index ebc00a3d51d24216397354bb245fe6ef76421f0b..aa3f095f985d70774347a7ca8d4a20ea402d0a3d 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -41,6 +41,7 @@ ## Already Filed Followup Issues * build system fmt step with check=false does not acquire a write lock on source files #35204 * enhance CheckFile step output when there is not a match #35208 +* missing truncate functionality #35353 ## Release Notes diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index e8dff14a4611f23a80637129c6e967addccf5453..52815f794505d7538c628d15fd2503d9540a9716 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1879,13 +1879,50 @@ pub fn installGenerated( return installPath(maker, arena, src_path, dest_path, asking_step_index); } +pub fn truncatePath( + maker: *Maker, + arena: Allocator, + dest_path: Path, + asking_step_index: Configuration.Step.Index, +) Step.ExtendedMakeError!void { + const graph = maker.graph; + const io = graph.io; + if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ + "truncate", try dest_path.toString(arena), + }); + // https://codeberg.org/ziglang/zig/issues/35353 + const err = e: { + var file = f: { + break :f dest_path.root_dir.handle.createFile(io, dest_path.sub_path, .{}) catch |err| switch (err) { + error.FileNotFound => { + const parent_path = dest_path.dirname() orelse break :e err; + parent_path.root_dir.handle.createDirPath(io, parent_path.sub_path) catch |in| switch (in) { + error.Canceled => |e| return e, + else => |e| { + const s = stepByIndex(maker, asking_step_index); + return s.fail(maker, "failed creating directory {f}: {t}", .{ parent_path, e }); + }, + }; + break :f dest_path.root_dir.handle.createFile(io, dest_path.sub_path, .{}) catch |in| break :e in; + }, + error.Canceled => |e| return e, + else => |e| break :e e, + }; + }; + file.close(io); + return; + }; + const s = stepByIndex(maker, asking_step_index); + return s.fail(maker, "failed truncating file {f}: {t}", .{ dest_path, err }); +} + pub fn installPath( maker: *Maker, arena: Allocator, src_path: Path, dest_path: Path, asking_step_index: Configuration.Step.Index, -) !Dir.PrevStatus { +) Step.ExtendedMakeError!Dir.PrevStatus { const graph = maker.graph; const io = graph.io; if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ @@ -1900,7 +1937,7 @@ pub fn installPath( .{}, ) catch |err| { const s = stepByIndex(maker, asking_step_index); - return s.fail(maker, "unable to update file from {f} to {f}: {t}", .{ src_path, dest_path, err }); + return s.fail(maker, "failed updating file from {f} to {f}: {t}", .{ src_path, dest_path, err }); }; } @@ -1910,7 +1947,7 @@ pub fn installDir( arena: Allocator, dest_path: Path, asking_step_index: Configuration.Step.Index, -) !Dir.CreatePathStatus { +) Step.ExtendedMakeError!Dir.CreatePathStatus { const graph = maker.graph; const io = graph.io; if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ @@ -1918,7 +1955,7 @@ pub fn installDir( }); return dest_path.root_dir.handle.createDirPathStatus(io, dest_path.sub_path, .default_dir) catch |err| { const s = stepByIndex(maker, asking_step_index); - return s.fail(maker, "unable to create dir {f}: {t}", .{ dest_path, err }); + return s.fail(maker, "failed creating dir {f}: {t}", .{ dest_path, err }); }; } diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 92cdfdb957a2729b3b4817335dd0588de39e5172..6b74c1469adf77a577dabfc102d985750349c563 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -21,6 +21,7 @@ const Maker = @import("../Maker.zig"); pub const Compile = @import("Step/Compile.zig"); pub const Fmt = @import("Step/Fmt.zig"); pub const InstallArtifact = @import("Step/InstallArtifact.zig"); +pub const InstallDir = @import("Step/InstallDir.zig"); pub const InstallFile = @import("Step/InstallFile.zig"); pub const ObjCopy = @import("Step/ObjCopy.zig"); pub const Options = @import("Step/Options.zig"); @@ -79,11 +80,10 @@ pub const Extended = union(enum) { find_program: Todo, fmt: Fmt, install_artifact: InstallArtifact, - install_dir: Todo, + install_dir: InstallDir, install_file: InstallFile, obj_copy: ObjCopy, options: Options, - remove_dir: Todo, run: Run, top_level: TopLevel, translate_c: Todo, @@ -103,7 +103,6 @@ pub const Extended = union(enum) { .install_file => .{ .install_file = .{} }, .obj_copy => .{ .obj_copy = .{} }, .options => .{ .options = .{} }, - .remove_dir => .{ .remove_dir = .{} }, .run => .{ .run = .{} }, .top_level => .{ .top_level = .{} }, .translate_c => .{ .translate_c = .{} }, diff --git a/lib/compiler/Maker/Step/InstallDir.zig b/lib/compiler/Maker/Step/InstallDir.zig index 7d079380dd9158a79d4e194490f168a5512e7621..484bebf329c791d65e121bcfa2297d8c79843283 100644 --- a/lib/compiler/Maker/Step/InstallDir.zig +++ b/lib/compiler/Maker/Step/InstallDir.zig @@ -1,7 +1,10 @@ const InstallDir = @This(); const std = @import("std"); +const Io = std.Io; +const log = std.log; const Configuration = std.Build.Configuration; +const endsWith = std.mem.endsWith; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); @@ -12,51 +15,82 @@ pub fn make( maker: *Maker, progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { + _ = install_dir; const graph = maker.graph; + const gpa = maker.gpa; const arena = maker.graph.arena; // TODO don't leak into process arena const io = graph.io; const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_id = conf_step.extended.get(conf.extra).install_dir; - step.clearWatchInputs(); - const dest_prefix = b.getInstallPath(install_dir.options.install_dir, install_dir.options.install_subdir); - const src_dir_path = install_dir.options.source_dir.getPath3(b, step); - const need_derived_inputs = try step.addDirectoryWatchInput(install_dir.options.source_dir); - var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {t}", .{ src_dir_path, err }); - }; + step.clearWatchInputs(maker); + + const dest_parent_path = try maker.resolveInstallDir(arena, conf_id.dest_dir); + const dest_prefix = if (conf_id.dest_sub_path.value) |s| + try dest_parent_path.join(arena, s.slice(conf)) + else + dest_parent_path; + const src_dir_lazy_path = conf_id.source_dir.get(conf); + const src_dir_path = try maker.resolveLazyPath(arena, src_dir_lazy_path, step_index); + const need_derived_inputs = try step.addDirectoryWatchInput(maker, src_dir_lazy_path); + + var src_dir = src_dir_path.root_dir.handle.openDir( + io, + src_dir_path.subPathOrDot(), + .{ .iterate = true }, + ) catch |err| return step.fail(maker, "failed opening source directory {f}: {t}", .{ src_dir_path, err }); defer src_dir.close(io); - var it = try src_dir.walk(arena); + + const exclude_extensions = conf_id.exclude_extensions.slice; + const include_extensions: ?[]const Configuration.String = if (conf_id.flags.include_extensions_active) + conf_id.include_extensions.slice + else + null; + const blank_extensions = conf_id.blank_extensions.slice; + var all_cached = true; - next_entry: while (try it.next(io)) |entry| { - for (install_dir.options.exclude_extensions) |ext| { - if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; + var it = try src_dir.walk(gpa); + defer it.deinit(); + next_entry: while (it.next(io) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| return step.fail(maker, "failed iterating dir {f}: {t}", .{ src_dir_path, e }), + }) |entry| { + for (exclude_extensions) |ext| { + if (endsWith(u8, entry.path, ext.slice(conf))) continue :next_entry; } - if (install_dir.options.include_extensions) |incs| { - for (incs) |inc| { - if (std.mem.endsWith(u8, entry.path, inc)) break; + if (include_extensions) |includes| { + for (includes) |inc| { + if (endsWith(u8, entry.path, inc.slice(conf))) break; } else { continue :next_entry; } } - const src_path = try install_dir.options.source_dir.join(arena, entry.path); - const dest_path = b.pathJoin(&.{ dest_prefix, entry.path }); + const dest_path = try dest_prefix.join(arena, entry.path); switch (entry.kind) { .directory => { - if (need_derived_inputs) _ = try step.addDirectoryWatchInput(src_path); - const p = try step.installDir(dest_path); + if (need_derived_inputs) { + const entry_path = try src_dir_path.join(arena, entry.path); + try step.addDirectoryWatchInputFromPath(maker, entry_path); + } + const p = try maker.installDir(arena, dest_path, step_index); all_cached = all_cached and p == .existed; }, .file => { - for (install_dir.options.blank_extensions) |ext| { - if (std.mem.endsWith(u8, entry.path, ext)) { - try b.truncateFile(dest_path); + for (blank_extensions) |ext| { + if (endsWith(u8, entry.path, ext.slice(conf))) { + // TODO check if the file was already there and length 0 + try maker.truncatePath(arena, dest_path, step_index); continue :next_entry; } } - const p = try step.installFile(src_path, dest_path); + const entry_path = try src_dir_path.join(arena, entry.path); + const p = try maker.installPath(arena, entry_path, dest_path, step_index); all_cached = all_cached and p == .fresh; + progress_node.completeOne(); }, else => continue, } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index a5a35a2d4b83bfc5b317392aa0a88dad9fa4d33a..d07f77b118aeb4e15978d05873188724ba595b3b 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -931,6 +931,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .dest_sub_path = dest_sub_path != null, .exclude_extensions = sid.options.exclude_extensions.len != 0, .include_extensions = include_extensions.len != 0, + .include_extensions_active = sid.options.include_extensions != null, .blank_extensions = sid.options.blank_extensions.len != 0, }, .source_dir = try s.addLazyPath(sid.options.source_dir), @@ -941,7 +942,6 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .blank_extensions = .{ .slice = try s.initStringList(sid.options.blank_extensions) }, }))); }, - .remove_dir => @panic("TODO"), .fail => e: { const sf: *Step.Fail = @fieldParentPtr("step", step); break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.Fail, .{ diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 252d65f6cc3e0126657cec0001886a54a90668ac..92474787c6a2b7628f4d3e2a1c44a348e9a6321e 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1584,23 +1584,6 @@ pub fn addCheckFile( return Step.CheckFile.create(b, file_source, options); } -pub fn truncateFile(b: *Build, dest_path: []const u8) (Io.Dir.CreateDirError || Io.Dir.StatFileError)!void { - const graph = b.graph; - const io = graph.io; - if (graph.verbose) log.info("truncate {s}", .{dest_path}); - const cwd = Io.Dir.cwd(); - var src_file = cwd.createFile(io, dest_path, .{}) catch |err| switch (err) { - error.FileNotFound => blk: { - if (fs.path.dirname(dest_path)) |dirname| { - try cwd.createDirPath(io, dirname); - } - break :blk try cwd.createFile(io, dest_path, .{}); - }, - else => |e| return e, - }; - src_file.close(io); -} - /// References a file or directory relative to the source root. pub fn path(b: *Build, sub_path: []const u8) LazyPath { if (fs.path.isAbsolute(sub_path)) { diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 34cd8c62e7940f616637d37074cb28c6fe3dd029..5888aa8abff79acb414d20537168eff36b3a6d6c 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -466,7 +466,6 @@ pub const Step = extern struct { install_file: InstallFile, obj_copy: ObjCopy, options: Options, - remove_dir: RemoveDir, run: Run, top_level: TopLevel, translate_c: TranslateC, @@ -501,7 +500,6 @@ pub const Step = extern struct { install_file, obj_copy, options, - remove_dir, run, top_level, translate_c, @@ -1197,8 +1195,9 @@ pub const Step = extern struct { dest_sub_path: bool, exclude_extensions: bool, include_extensions: bool, + include_extensions_active: bool, blank_extensions: bool, - _: u23 = 0, + _: u22 = 0, }; }; @@ -1320,15 +1319,6 @@ pub const Step = extern struct { }; }; - pub const RemoveDir = struct { - flags: @This().Flags, - - pub const Flags = packed struct(u32) { - tag: Tag = .remove_dir, - _: u27 = 0, - }; - }; - pub const TranslateC = struct { flags: @This().Flags, src_path: LazyPath.Index, -- 2.54.0 From 6327741b73e00e08880b619ef4b10ad2a0a217d3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 18 May 2026 22:02:40 -0700 Subject: [PATCH 110/179] doctest: update to newer std.zig API --- tools/doctest.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/doctest.zig b/tools/doctest.zig index 90c7108d297acaf6db6da02eb29b87af2ac151a9..d1d7919a51f299ecea3914ac8307d6b6741723a6 100644 --- a/tools/doctest.zig +++ b/tools/doctest.zig @@ -311,7 +311,9 @@ fn printOutput( .arch_os_abi = triple, }); const target = try std.zig.system.resolveTargetQuery(io, target_query); - switch (getExternalExecutor(io, &host, &target, .{ + switch (getExternalExecutor(io, &target, .{ + .host_cpu_arch = host.cpu.arch, + .host_os_tag = host.os.tag, .link_libc = code.link_libc, })) { .native => {}, @@ -526,7 +528,10 @@ fn printOutput( .lib => { const bin_basename = try std.zig.binNameAlloc(arena, .{ .root_name = code_name, - .target = &builtin.target, + .cpu_arch = builtin.target.cpu.arch, + .os_tag = builtin.target.os.tag, + .ofmt = builtin.target.ofmt, + .abi = builtin.target.abi, .output_mode = .Lib, }); -- 2.54.0 From eb80aa00608a6ada01b83cc7d8d4995c9636cfff Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 18 May 2026 22:10:25 -0700 Subject: [PATCH 111/179] std.Build.LazyPath.basename: fix impl * no more parameters * don't call getPath2, that was never valid to call in the configure phase... --- BRANCH_TODO | 16 ++++++++++------ lib/std/Build.zig | 7 ++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index aa3f095f985d70774347a7ca8d4a20ea402d0a3d..6c4920c58e8fb546acb2b3b92a216cdfbd621173 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,22 +1,19 @@ +* WriteFile step failing when making langref * double check when targets get resolved (should be at configure time) -* pass overridden pkg-dir to maker * finish migrating the rest of the build steps +* pass overridden pkg-dir to maker * inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) * make zig-pkg path root configurable in maker (make sure --system still works) * eliminate calls to getPath, getPath2, getPath3 * test lazyImport -* solve the TODOs added in this branch * get zig tests passing +* solve the TODOs added in this branch * test a bunch of third party projects / help people migrate * tetris -* get the target from the parent process instead * [handle missing cache hits when chaining two run steps](https://codeberg.org/ziglang/zig/pulls/30762) * [Absolute and cwd-relative paths in build cache](https://codeberg.org/ziglang/zig/issues/32097) -* make more stuff use IndexType -* make addExtra return Index using reflection -* refactor with DefaultingEnum * implement {q} or delete {q} uses * make the generated dependencies.zig be dependencies.zon and don't put absolute paths in there @@ -25,6 +22,10 @@ * re-evaluate https://codeberg.org/ziglang/zig/pulls/35224 ## Followup Issues +* make more stuff use IndexType +* make addExtra return Index using reflection +* refactor with DefaultingEnum +* get the target from the parent process instead * stop leaking into global process arena * reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make * link_eh_frame_hdr should be DefaultingBool @@ -37,6 +38,7 @@ * UpdateSourceFiles: introduce Group * WriteFiles: introduce Group * re-examine the use case of adding file paths to Options steps +* extract the reusable Configure abstractions and reuse it for Zir etc ## Already Filed Followup Issues * build system fmt step with check=false does not acquire a write lock on source files #35204 @@ -84,6 +86,8 @@ closes #31397 * `b.build_root` (Directory) -> `b.root` (Path) * `ConfigHeader.Options`: `include_guard_override` -> `include_guard` * `LazyPath`: `getDisplayName` -> `format` or `fmt` +* `LazyPath.basename` no longer takes parameters. The returned basename might + be unknown until make phase in which case returned string is length zero. ### Perf Data Point: `zig build -h` (cached) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 92474787c6a2b7628f4d3e2a1c44a348e9a6321e..f2b0ed84d9dc9a38777e734d9d308edd99ad0343 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -2332,14 +2332,11 @@ pub const LazyPath = union(enum) { } } - pub fn basename(lazy_path: LazyPath, src_builder: *Build, asking_step: ?*Step) []const u8 { + pub fn basename(lazy_path: LazyPath) []const u8 { return fs.path.basename(switch (lazy_path) { .src_path => |sp| sp.sub_path, .cwd_relative => |sub_path| sub_path, - .generated => |gen| if (gen.sub_path.len > 0) - gen.sub_path - else - gen.file.getPath2(src_builder, asking_step), + .generated => |gen| gen.sub_path, .dependency => |dep| dep.sub_path, }); } -- 2.54.0 From c56f3eae689012ca416e66e7bb049ebaad8ffeb7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2026 15:37:00 -0700 Subject: [PATCH 112/179] build.zig: use addDirectoryArg rather than addFileArg for .zig_lib and .zig_exe --- build.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.zig b/build.zig index f539f8e0d9fa0b75b6e53e1664e992bb448784bc..bbfb24c4b60579b4a0f78cb572b16a488060b530 100644 --- a/build.zig +++ b/build.zig @@ -1528,10 +1528,10 @@ fn generateLangRef(b: *std.Build) !std.Build.LazyPath { // TODO: enhance doctest to use "--listen=-" rather than operating in a // temporary directory cmd.addArg("--cache-root"); - cmd.addFileArg(.cache_root); + cmd.addDirectoryArg(.cache_root); cmd.addArg("--zig-lib-dir"); - cmd.addFileArg(.zig_lib); + cmd.addDirectoryArg(.zig_lib); cmd.addArg("-i"); cmd.addFileArg(b.path(b.fmt("doc/langref/{s}", .{entry.name}))); -- 2.54.0 From 2c70c40499d260804d3e6441f6e8ff8b88378a8c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2026 15:37:32 -0700 Subject: [PATCH 113/179] configurer: fix serialization of Run path_directory args --- lib/compiler/configurer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index d07f77b118aeb4e15978d05873188724ba595b3b..a4c661b1f4431b2cf5bcea46019d3afdae6b4b16 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -358,7 +358,7 @@ const Serialize = struct { .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, - .suffix = .{ .value = try addOptionalString(s, a.suffix) }, + .suffix = .{ .value = if (a.suffix.len != 0) try wc.addString(a.suffix) else null }, .basename = .{ .value = null }, .path = .{ .value = try addLazyPath(s, a.lazy_path) }, .producer = .{ .value = null }, -- 2.54.0 From 58f8dcd15e04a02185f4e18dbf76c54942e05350 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2026 15:41:58 -0700 Subject: [PATCH 114/179] std.Build: improve documentation for UpdateSourceFiles step --- lib/std/Build.zig | 10 ++++++++++ lib/std/Build/Step/UpdateSourceFiles.zig | 5 ----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index f2b0ed84d9dc9a38777e734d9d308edd99ad0343..39b4b8993f65fee3ef842cf0ec4139d463e2ae9b 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -966,6 +966,16 @@ pub fn addWriteFiles(b: *Build) *Step.WriteFile { return Step.WriteFile.create(b); } +/// Creates a step for writing data to paths relative to the build root, +/// mutating the project's source files. +/// +/// This build step was designed not to be used during the normal build +/// process, but rather as a utility run by a developer with intention to +/// update source files, which will then be committed to version control. +/// +/// Example use cases: +/// * precompiling assets which are tracked by version control +/// * snapshot testing pub fn addUpdateSourceFiles(b: *Build) *Step.UpdateSourceFiles { return Step.UpdateSourceFiles.create(b); } diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig index cc5b114b8c2920303ccb90e029a00c691c90587d..11391ea7d27729233ff0d89cf706e7f10ce8af7a 100644 --- a/lib/std/Build/Step/UpdateSourceFiles.zig +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -1,8 +1,3 @@ -//! Writes data to paths relative to the package root, effectively mutating the -//! package's source files. Be careful with the latter functionality; it should -//! not be used during the normal build process, but as a utility run by a -//! developer with intention to update source files, which will then be -//! committed to version control. const UpdateSourceFiles = @This(); const std = @import("std"); -- 2.54.0 From f414370fba47e375cc240f3fcaa8fa36c7c0fed3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2026 15:42:18 -0700 Subject: [PATCH 115/179] docgen: better error message on failure to read output --- tools/docgen.zig | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/docgen.zig b/tools/docgen.zig index ab9419d55ab98eff79e07c01a831f6ec07168728..8d4324a0d4301c2b8a401b49aa35e900637b83a2 100644 --- a/tools/docgen.zig +++ b/tools/docgen.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const std = @import("std"); const Io = std.Io; const Dir = std.Io.Dir; +const Path = std.Build.Cache.Path; const process = std.process; const Progress = std.Progress; const print = std.debug.print; @@ -73,8 +74,13 @@ pub fn main(init: std.process.Init) !void { var out_file_buffer: [4096]u8 = undefined; var out_file_writer = out_file.writer(io, &out_file_buffer); - var code_dir = try Dir.cwd().openDir(io, code_dir_path, .{}); - defer code_dir.close(io); + var code_dir: Path = .{ + .root_dir = .{ + .handle = try Dir.cwd().openDir(io, code_dir_path, .{}), + .path = code_dir_path, + }, + }; + defer code_dir.root_dir.handle.close(io); var in_file_reader = in_file.reader(io, &.{}); const input_file_bytes = try in_file_reader.interface.allocRemaining(arena, .limited(max_doc_file_size)); @@ -988,7 +994,7 @@ fn genHtml( io: Io, tokenizer: *Tokenizer, toc: *Toc, - code_dir: Dir, + code_dir: Path, out: *Writer, ) !void { for (toc.nodes) |node| { @@ -1044,8 +1050,13 @@ fn genHtml( }); defer allocator.free(out_basename); - const contents = code_dir.readFileAlloc(io, out_basename, allocator, .limited(std.math.maxInt(u32))) catch |err| { - return parseError(tokenizer, code.token, "unable to open '{s}': {t}", .{ out_basename, err }); + const out_path: Path = .{ + .root_dir = code_dir.root_dir, + .sub_path = out_basename, + }; + + const contents = out_path.root_dir.handle.readFileAlloc(io, out_path.sub_path, allocator, .unlimited) catch |err| { + return parseError(tokenizer, code.token, "failed opening {f}: {t}", .{ out_path, err }); }; defer allocator.free(contents); -- 2.54.0 From 5c133c5765f3c4e80d46c35c104edc23d883c60e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2026 16:29:49 -0700 Subject: [PATCH 116/179] Configuration: fix deserialization of LengthPrefixedList don't assume only 1 field --- BRANCH_TODO | 3 ++- lib/std/Build/Configuration.zig | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index 6c4920c58e8fb546acb2b3b92a216cdfbd621173..da43728d8dee952202828bc0edb58a3ccd76cac8 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,4 +1,4 @@ -* WriteFile step failing when making langref +* Run step getting wrong path_directory values * double check when targets get resolved (should be at configure time) * finish migrating the rest of the build steps * pass overridden pkg-dir to maker @@ -39,6 +39,7 @@ * WriteFiles: introduce Group * re-examine the use case of adding file paths to Options steps * extract the reusable Configure abstractions and reuse it for Zir etc +* add the ability to delete files to UpdateSourceFiles ## Already Filed Followup Issues * build system fmt step with check=false does not acquire a write lock on source files #35204 diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 5888aa8abff79acb414d20537168eff36b3a6d6c..612c3256bb4010ad2dc2458f470d85ff2bcfa86b 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -2545,8 +2545,10 @@ pub const Storage = enum { }; } - /// The field contains a u32 length followed by that many items, each - /// element bitcastable to u32. + /// The field contains a u32 length followed by that many items. Each + /// element needs well-defined memory layout but can otherwise be any + /// multiple of u32 length. The length is number of elements, not the + /// number of u32s. pub fn LengthPrefixedList(comptime ElemArg: type) type { return struct { slice: []const Elem, @@ -2759,19 +2761,21 @@ pub const Storage = enum { }, .extended => @compileError("TODO"), .length_prefixed_list => { + const n = @divExact(@sizeOf(Field.Elem), @sizeOf(u32)); const data_start = i.* + 1; - const len = buffer[data_start - 1]; - defer i.* = data_start + len; - return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; + const buf_len = buffer[data_start - 1] * n; + defer i.* = data_start + buf_len; + return .{ .slice = @ptrCast(buffer[data_start..][0..buf_len]) }; }, .flag_length_prefixed_list => { const flags = @field(container, @tagName(Field.flags)); const flag = @field(flags, @tagName(Field.flag)); if (!flag) return .{ .slice = &.{} }; + const n = @divExact(@sizeOf(Field.Elem), @sizeOf(u32)); const data_start = i.* + 1; - const len = buffer[data_start - 1]; - defer i.* = data_start + len; - return .{ .slice = @ptrCast(buffer[data_start..][0..len]) }; + const buf_len = buffer[data_start - 1] * n; + defer i.* = data_start + buf_len; + return .{ .slice = @ptrCast(buffer[data_start..][0..buf_len]) }; }, .flag_list => { const flags = @field(container, @tagName(Field.flags)); -- 2.54.0 From ec65f129d8cfdba6c15f0ea5cda5acea5c8d3ada Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2026 16:58:07 -0700 Subject: [PATCH 117/179] Maker: lower -target arguments --- BRANCH_TODO | 114 ---------------------------- lib/compiler/Maker/Step/Compile.zig | 12 +-- 2 files changed, 6 insertions(+), 120 deletions(-) delete mode 100644 BRANCH_TODO diff --git a/BRANCH_TODO b/BRANCH_TODO deleted file mode 100644 index da43728d8dee952202828bc0edb58a3ccd76cac8..0000000000000000000000000000000000000000 --- a/BRANCH_TODO +++ /dev/null @@ -1,114 +0,0 @@ -* Run step getting wrong path_directory values -* double check when targets get resolved (should be at configure time) -* finish migrating the rest of the build steps -* pass overridden pkg-dir to maker -* inspect b4ffb402c082605c4b324e88120306fc8fb3cf32 diff and apply changes as needed (merge conflict) -* make zig-pkg path root configurable in maker (make sure --system still works) -* eliminate calls to getPath, getPath2, getPath3 -* test lazyImport -* get zig tests passing -* solve the TODOs added in this branch -* test a bunch of third party projects / help people migrate - * tetris - -* [handle missing cache hits when chaining two run steps](https://codeberg.org/ziglang/zig/pulls/30762) -* [Absolute and cwd-relative paths in build cache](https://codeberg.org/ziglang/zig/issues/32097) - - -* implement {q} or delete {q} uses -* make the generated dependencies.zig be dependencies.zon and don't put absolute paths in there - - and adjust dependencyInner to not openDir() - -* re-evaluate https://codeberg.org/ziglang/zig/pulls/35224 - -## Followup Issues -* make more stuff use IndexType -* make addExtra return Index using reflection -* refactor with DefaultingEnum -* get the target from the parent process instead -* stop leaking into global process arena -* reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make -* link_eh_frame_hdr should be DefaultingBool -* make --foo, --no-foo CLI args uniform (make them -f args instead) -* install steps should provide generated files for installed things, then delete the run step hack - - but artifact install steps also add paths for dyn libs on windows -* no more "artifact arg" to run step. if you want to run the post-install binary, get the lazy path - from the install step. -* fmt step: import zig fmt code directly rather than child proc -* UpdateSourceFiles: introduce Group -* WriteFiles: introduce Group -* re-examine the use case of adding file paths to Options steps -* extract the reusable Configure abstractions and reuse it for Zir etc -* add the ability to delete files to UpdateSourceFiles - -## Already Filed Followup Issues -* build system fmt step with check=false does not acquire a write lock on source files #35204 -* enhance CheckFile step output when there is not a match #35208 -* missing truncate functionality #35353 - -## Release Notes - -### Run Step: Passthru Args - -In the Run step, passthru args are all together now, not observable in -configure phase whether run args are provided. - -```zig -if (b.args) |args| { - run_cmd.addArgs(args); -} -``` - -⬇️ - -```zig -run_cmd.addPassthruArgs(); -``` - -This removes a capability from build scripts since they can no longer observe -those arguments. In exchange, it means that when changing those arguments, -build scripts no longer must be rebuilt from source. - -closes #31397 - -### Fmt Step: Options - -`paths` and `exclude_paths` are now LazyPath lists. There is a convenience method to create them: `b.pathList`. - -```diff -- const fmt_include_paths = &.{ "lib", "src", "test", "tools", "build.zig", "build.zig.zon" }; -- const fmt_exclude_paths = &.{ "test/cases", "test/behavior/zon" }; -+ const fmt_include_paths = b.pathList(&.{ "lib", "src", "test", "tools", "build.zig", "build.zig.zon" }); -+ const fmt_exclude_paths = b.pathList(&.{ "test/cases", "test/behavior/zon" }); -``` - -### std.Build API - -* `b.build_root` (Directory) -> `b.root` (Path) -* `ConfigHeader.Options`: `include_guard_override` -> `include_guard` -* `LazyPath`: `getDisplayName` -> `format` or `fmt` -* `LazyPath.basename` no longer takes parameters. The returned basename might - be unknown until make phase in which case returned string is length zero. - -### Perf Data Point: `zig build -h` (cached) - -``` -Benchmark 1 (34 runs): master/zig build -h - measurement mean ± σ min … max outliers delta - wall_time 150ms ± 5.52ms 145ms … 165ms 4 (12%) 0% - peak_rss 84.8MB ± 275KB 84.2MB … 85.1MB 0 ( 0%) 0% - cpu_cycles 593M ± 4.01M 588M … 608M 2 ( 6%) 0% - instructions 995M ± 52.5K 995M … 995M 0 ( 0%) 0% - cache_references 25.8M ± 165K 25.4M … 26.1M 0 ( 0%) 0% - cache_misses 651K ± 20.1K 619K … 697K 0 ( 0%) 0% - branch_misses 918K ± 7.44K 906K … 935K 0 ( 0%) 0% -Benchmark 2 (348 runs): branch/zig build -h - measurement mean ± σ min … max outliers delta - wall_time 14.3ms ± 744us 13.2ms … 23.3ms 8 ( 2%) ⚡- 90.4% ± 0.4% - peak_rss 78.5MB ± 562KB 77.1MB … 81.4MB 7 ( 2%) ⚡- 7.4% ± 0.2% - cpu_cycles 24.1M ± 821K 22.8M … 27.1M 3 ( 1%) ⚡- 95.9% ± 0.1% - instructions 43.7M ± 23.8K 43.7M … 43.8M 56 (16%) ⚡- 95.6% ± 0.0% - cache_references 1.46M ± 14.6K 1.40M … 1.50M 19 ( 5%) ⚡- 94.3% ± 0.1% - cache_misses 142K ± 4.87K 127K … 157K 2 ( 1%) ⚡- 78.1% ± 0.4% - branch_misses 126K ± 1.37K 120K … 129K 12 ( 3%) ⚡- 86.3% ± 0.1% -``` diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index a01b548269dee83ef04a81a73524a3dbd1215328..f70e5eea593b738886aa8a1b0739a63b67b41d64 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -1274,21 +1274,21 @@ fn appendModuleFlags( if (m.resolved_target.get(conf)) |target| { // Communicate the query via CLI since it's more compact. - if (target.query.get(conf)) |query| { + if (target.query.get(conf)) |compact_query| { try zig_args.ensureUnusedCapacity(gpa, 6); - if (true) @panic("TODO appendModuleFlags"); + const query = compact_query.unwrap(conf); zig_args.appendAssumeCapacity("-target"); zig_args.appendAssumeCapacity(try query.zigTriple(arena)); + zig_args.appendAssumeCapacity("-mcpu"); zig_args.appendAssumeCapacity(try query.serializeCpuAlloc(arena)); - if (query.dynamic_linker) |dynamic_linker| { - const dynamic_linker_slice = dynamic_linker.slice(conf); - if (dynamic_linker_slice.len != 0) { + if (query.dynamic_linker) |*dynamic_linker| { + if (dynamic_linker.get()) |dynamic_linker_path| { zig_args.appendAssumeCapacity("--dynamic-linker"); - zig_args.appendAssumeCapacity(dynamic_linker_slice); + zig_args.appendAssumeCapacity(dynamic_linker_path); } else { zig_args.appendAssumeCapacity("--no-dynamic-linker"); } -- 2.54.0 From 3a259e2f0f56220ffa3019b009429e9adc4bc6d0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2026 17:17:10 -0700 Subject: [PATCH 118/179] Maker.Step.Compile: implement checkCompileErrors --- lib/compiler/Maker/Step/Compile.zig | 91 +++++++++++++++++------------ 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index f70e5eea593b738886aa8a1b0739a63b67b41d64..e8f632ed28f71fc525f39f7e15168838741d3df8 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -2,6 +2,7 @@ const Compile = @This(); const std = @import("std"); const Allocator = std.mem.Allocator; +const mem = std.mem; const Configuration = std.Build.Configuration; const Dir = std.Io.Dir; const Path = std.Build.Cache.Path; @@ -9,7 +10,6 @@ const Module = std.Build.Configuration.Module; const Io = std.Io; const Sha256 = std.crypto.hash.sha2.Sha256; const assert = std.debug.assert; -const mem = std.mem; const allocPrint = std.fmt.allocPrint; const Step = @import("../Step.zig"); @@ -51,7 +51,7 @@ pub fn make( (graph.incremental == true) and (maker.watch or maker.web_server != null), ) catch |err| switch (err) { error.NeedCompileErrorCheck => { - try checkCompileErrors(compile, maker); + try checkCompileErrors(maker, compile_index); return; }, else => |e| return e, @@ -518,7 +518,7 @@ fn lowerZigArgs( const import_cli_name = cli_named_modules.names.keys()[import_index]; zig_args.appendAssumeCapacity("--dep"); const name_slice = name.slice(conf); - if (std.mem.eql(u8, import_cli_name, name_slice)) { + if (mem.eql(u8, import_cli_name, name_slice)) { zig_args.appendAssumeCapacity(import_cli_name); } else { zig_args.appendAssumeCapacity(try allocPrint(arena, "{s}={s}", .{ @@ -898,8 +898,8 @@ fn lowerZigArgs( // Write the args to zig-cache/args/ to avoid conflicts with // other zig build commands running in parallel. - const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); - const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); + const partially_quoted = try mem.join(arena, "\" \"", escaped_args.items); + const args = try mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); var args_hash: [Sha256.digest_length]u8 = undefined; Sha256.hash(args, &args_hash, .{}); @@ -943,24 +943,30 @@ fn lowerZigArgs( } } -pub fn rebuildInFuzzMode(compile: *Compile, maker: *Maker, progress_node: std.Progress.Node) !Path { +pub fn rebuildInFuzzMode( + compile: *Compile, + maker: *Maker, + step_index: Configuration.Step.Index, + progress_node: std.Progress.Node, +) !Path { const gpa = maker.graph.gpa; + const step = maker.stepByIndex(step_index); - compile.step.result_error_msgs.clearRetainingCapacity(); - compile.step.result_stderr = ""; + step.result_error_msgs.clearRetainingCapacity(); + step.result_stderr = ""; - compile.step.result_error_bundle.deinit(gpa); - compile.step.result_error_bundle = std.zig.ErrorBundle.empty; + step.result_error_bundle.deinit(gpa); + step.result_error_bundle = std.zig.ErrorBundle.empty; - if (compile.step.result_failed_command) |cmd| { + if (step.result_failed_command) |cmd| { gpa.free(cmd); - compile.step.result_failed_command = null; + step.result_failed_command = null; } const zig_args = &compile.zig_args; zig_args.clearRetainingCapacity(); try lowerZigArgs(compile, maker, progress_node, zig_args, true); - const maybe_output_bin_path = try compile.step.evalZigProcess(zig_args.items, progress_node, false, maker); + const maybe_output_bin_path = try step.evalZigProcess(zig_args.items, progress_node, false, maker); return maybe_output_bin_path.?; } @@ -973,35 +979,43 @@ fn addFlag(gpa: Allocator, args: *std.ArrayList([]const u8), comptime name: []co try args.append(gpa, if (cond) "-f" ++ name else "-fno-" ++ name); } -fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { - if (true) @panic("TODO checkCompileErrors"); +fn checkCompileErrors( + maker: *Maker, + step_index: Configuration.Step.Index, +) Step.ExtendedMakeError!void { + const step = maker.stepByIndex(step_index); + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_comp = conf_step.extended.get(conf.extra).compile; + // Clear this field so that it does not get printed by the build runner. - const actual_eb = compile.step.result_error_bundle; - compile.step.result_error_bundle = .empty; - - const arena = compile.step.owner.allocator; + const actual_eb = step.result_error_bundle; + step.result_error_bundle = .empty; const actual_errors = ae: { var aw: std.Io.Writer.Allocating = .init(arena); defer aw.deinit(); - try actual_eb.renderToWriter(.{ + actual_eb.renderToWriter(.{ .include_reference_trace = false, .include_source_line = false, - }, &aw.writer); + }, &aw.writer) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + }; break :ae try aw.toOwnedSlice(); }; // Render the expected lines into a string that we can compare verbatim. var expected_generated: std.ArrayList(u8) = .empty; - const expect_errors = compile.expect_errors.?; - var actual_line_it = mem.splitScalar(u8, actual_errors, '\n'); - // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile - switch (expect_errors) { - .starts_with => |expect_starts_with| { - if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return; - return compile.step.fail(maker, + switch (conf_comp.expect_errors.u) { + .none => unreachable, + .starts_with => |expect_starts_with_string| { + const expect_starts_with = expect_starts_with_string.slice(conf); + if (mem.startsWith(u8, actual_errors, expect_starts_with)) return; + return step.fail(maker, \\ \\========= should start with: ============ \\{s} @@ -1010,13 +1024,14 @@ fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { \\========================================= , .{ expect_starts_with, actual_errors }); }, - .contains => |expect_line| { + .contains => |expect_line_string| { + const expect_line = expect_line_string.slice(conf); while (actual_line_it.next()) |actual_line| { if (!matchCompileError(actual_line, expect_line)) continue; return; } - return compile.step.fail(maker, + return step.fail(maker, \\ \\========= should contain: =============== \\{s} @@ -1025,12 +1040,13 @@ fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { \\========================================= , .{ expect_line, actual_errors }); }, - .stderr_contains => |expect_line| { - const actual_stderr: []const u8 = if (compile.step.result_error_msgs.items.len > 0) - compile.step.result_error_msgs.items[0] + .stderr_contains => |expect_line_string| { + const expect_line = expect_line_string.slice(conf); + const actual_stderr: []const u8 = if (step.result_error_msgs.items.len > 0) + step.result_error_msgs.items[0] else &.{}; - compile.step.result_error_msgs.clearRetainingCapacity(); + step.result_error_msgs.clearRetainingCapacity(); var stderr_line_it = mem.splitScalar(u8, actual_stderr, '\n'); @@ -1039,7 +1055,7 @@ fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { return; } - return compile.step.fail(maker, + return step.fail(maker, \\ \\========= should contain: =============== \\{s} @@ -1049,7 +1065,8 @@ fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { , .{ expect_line, actual_stderr }); }, .exact => |expect_lines| { - for (expect_lines) |expect_line| { + for (expect_lines.slice) |expect_line_string| { + const expect_line = expect_line_string.slice(conf); const actual_line = actual_line_it.next() orelse { try expected_generated.appendSlice(arena, expect_line); try expected_generated.append(arena, '\n'); @@ -1066,7 +1083,7 @@ fn checkCompileErrors(compile: *Compile, maker: *Maker) !void { if (mem.eql(u8, expected_generated.items, actual_errors)) return; - return compile.step.fail(maker, + return step.fail(maker, \\ \\========= expected: ===================== \\{s} -- 2.54.0 From d1204d410472786bfd831a6349437b7784273f42 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2026 18:01:40 -0700 Subject: [PATCH 119/179] Maker.Step.Run: implement addPathForDynLibs --- lib/compiler/Maker/Step/Compile.zig | 4 +-- lib/compiler/Maker/Step/Run.zig | 54 +++++++++++++++++++---------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index e8f632ed28f71fc525f39f7e15168838741d3df8..881546f5122b7cd3b4cfec9e555871fa7c6c96d4 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -119,7 +119,7 @@ fn updateGeneratedFile( /// the root module. The root module is guaranteed to be first. const ModuleList = std.AutoArrayHashMapUnmanaged(Configuration.Module.Index, Configuration.String); /// Keyed on the first key in the module list. -const ModuleGraph = std.ArrayHashMapUnmanaged(ModuleList, void, ModuleListContext, false); +pub const ModuleGraph = std.ArrayHashMapUnmanaged(ModuleList, void, ModuleListContext, false); const ModuleListContext = struct { pub fn eql(ctx: @This(), a: ModuleList, b: ModuleList) bool { @@ -1164,7 +1164,7 @@ const CliNamedModules = struct { } }; -fn getCompileDependencies( +pub fn getCompileDependencies( arena: Allocator, module_graph: *ModuleGraph, conf: *const Configuration, diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 2cc2972bcb9a126d9895fcfdfe792dc033ab6543..c927d6d4af90ed114483a0b53504cf3c33a5fd7b 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -137,16 +137,9 @@ pub fn make( const producer_index = arg.producer.value.?; const producer_step = producer_index.ptr(conf); const producer = producer_step.extended.get(conf.extra).compile; - const root_module = producer.root_module.get(conf); - const root_module_target = root_module.resolved_target.get(conf).?.result.get(conf); - const os_tag = root_module_target.flags.os_tag.unwrap().?; const producer_make_comp_step = maker.stepByIndex(producer_index); const producer_make_comp = &producer_make_comp_step.extended.compile; - if (os_tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - addPathForDynLibs(producer_index); - } const file_path = producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?).*; argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ @@ -953,7 +946,7 @@ const FuzzTestRunner = struct { if (i == std.math.maxInt(u32)) return; i += 1; }) { - const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; + const name_prefix = "f" ++ Dir.path.sep_str ++ "in"; in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; in_f = cache_root.handle.openFile(io, in_name, .{ .lock = .exclusive, @@ -990,7 +983,7 @@ const FuzzTestRunner = struct { defer in_f.close(io); // Save it to a seperate file - const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; + const crash_name = "f" ++ Dir.path.sep_str ++ "crash"; const out = cache_root.handle.createFile(io, crash_name, .{ .lock = .exclusive, // Multiple run steps could have found a crash at the same time }) catch |e| return step.fail(maker, "failed to create file '{f}{s}': {t}", .{ @@ -1922,7 +1915,7 @@ fn runCommand( if (root_target.os.tag == .windows) { // On Windows we don't have rpaths so we have to add .dll search paths to PATH - addPathForDynLibs(producer_index); + try addPathForDynLibs(maker, producer_index, environ_map, argv[0]); } gpa.free(step.result_failed_command.?); @@ -2298,14 +2291,39 @@ fn convertPathArg(run_index: Configuration.Step.Index, maker: *Maker, path: Path return Dir.path.join(arena, &.{ ".", child_cwd_rel }); } -fn addPathForDynLibs(artifact: Configuration.Step.Index) void { - if (true) @panic("TODO addPathForDynLibs"); - for (artifact.getCompileDependencies(true)) |compile| { - if (compile.root_module.resolved_target.?.result.os.tag == .windows and - compile.isDynamicLibrary()) - { - @panic("TODO addPathForDynLibs"); - //addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, step)).?); +fn addPathForDynLibs( + maker: *Maker, + artifact: Configuration.Step.Index, + environ_map: *process.Environ.Map, + argv0: []const u8, +) !void { + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into process arena + const use_wine = graph.enable_wine and builtin.os.tag != .windows and std.ascii.endsWithIgnoreCase(argv0, ".exe"); + const path_key = if (use_wine) "WINEPATH" else "PATH"; + const path_delimiter: u8 = if (builtin.os.tag == .windows or use_wine) + Dir.path.delimiter_windows + else + Dir.path.delimiter; + + var module_graph: Step.Compile.ModuleGraph = .empty; + const compile_deps = try Step.Compile.getCompileDependencies(arena, &module_graph, conf, artifact, true); + + for (compile_deps) |dep_index| { + const conf_comp_step = dep_index.ptr(conf); + const conf_comp = conf_comp_step.extended.get(conf.extra).compile; + const root_module = conf_comp.root_module.get(conf); + const target = root_module.resolved_target.get(conf).?.result.get(conf); + if (target.flags.os_tag == .windows and conf_comp.isDynamicLibrary()) { + const dll_path = try maker.generatedPath(conf_comp.generated_bin.value.?).toString(arena); + const search_path = Dir.path.dirname(dll_path).?; + if (environ_map.get(path_key)) |prev_path| { + const new_path = try allocPrint(arena, "{s}{c}{s}", .{ prev_path, path_delimiter, search_path }); + try environ_map.put(path_key, new_path); + } else { + try environ_map.put(path_key, search_path); + } } } } -- 2.54.0 From 88ae1d9aeca6441f2c905bbde61d399025c63242 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 12:24:33 -0700 Subject: [PATCH 120/179] Configuration: fix the target os enum mismatch --- lib/std/Build/Configuration.zig | 98 +++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 612c3256bb4010ad2dc2458f470d85ff2bcfa86b..2ad1cf4d03dd6d56a2c8f125a66dbf6703758915 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -2324,6 +2324,7 @@ pub const TargetQuery = struct { ps3, ps4, ps5, + psp, vita, emscripten, wasi, @@ -2335,18 +2336,105 @@ pub const TargetQuery = struct { opencl, opengl, vulkan, + tios, default, pub fn init(x: ?std.Target.Os.Tag) @This() { - // TODO comptime assert the enums match - return @enumFromInt(@intFromEnum(x orelse return .default)); + return switch (x orelse return .default) { + .freestanding => .freestanding, + .other => .other, + .contiki => .contiki, + .fuchsia => .fuchsia, + .hermit => .hermit, + .managarm => .managarm, + .haiku => .haiku, + .hurd => .hurd, + .illumos => .illumos, + .linux => .linux, + .plan9 => .plan9, + .rtems => .rtems, + .serenity => .serenity, + .dragonfly => .dragonfly, + .freebsd => .freebsd, + .netbsd => .netbsd, + .openbsd => .openbsd, + .driverkit => .driverkit, + .ios => .ios, + .maccatalyst => .maccatalyst, + .macos => .macos, + .tvos => .tvos, + .visionos => .visionos, + .watchos => .watchos, + .windows => .windows, + .uefi => .uefi, + .@"3ds" => .@"3ds", + .ps3 => .ps3, + .ps4 => .ps4, + .ps5 => .ps5, + .psp => .psp, + .vita => .vita, + .emscripten => .emscripten, + .wasi => .wasi, + .amdhsa => .amdhsa, + .amdpal => .amdpal, + .cuda => .cuda, + .mesa3d => .mesa3d, + .nvcl => .nvcl, + .opencl => .opencl, + .opengl => .opengl, + .vulkan => .vulkan, + .tios => .tios, + }; } pub fn unwrap(this: @This()) ?std.Target.Os.Tag { - // TODO comptime assert the enums match - if (this == .default) return null; - return @enumFromInt(@intFromEnum(this)); + return switch (this) { + .default => null, + .freestanding => .freestanding, + .other => .other, + .contiki => .contiki, + .fuchsia => .fuchsia, + .hermit => .hermit, + .managarm => .managarm, + .haiku => .haiku, + .hurd => .hurd, + .illumos => .illumos, + .linux => .linux, + .plan9 => .plan9, + .rtems => .rtems, + .serenity => .serenity, + .dragonfly => .dragonfly, + .freebsd => .freebsd, + .netbsd => .netbsd, + .openbsd => .openbsd, + .driverkit => .driverkit, + .ios => .ios, + .maccatalyst => .maccatalyst, + .macos => .macos, + .tvos => .tvos, + .visionos => .visionos, + .watchos => .watchos, + .windows => .windows, + .uefi => .uefi, + .@"3ds" => .@"3ds", + .ps3 => .ps3, + .ps4 => .ps4, + .ps5 => .ps5, + .psp => .psp, + .vita => .vita, + .emscripten => .emscripten, + .wasi => .wasi, + .amdhsa => .amdhsa, + .amdpal => .amdpal, + .cuda => .cuda, + .mesa3d => .mesa3d, + .nvcl => .nvcl, + .opencl => .opencl, + .opengl => .opengl, + .vulkan => .vulkan, + .tios => .tios, + }; } }; pub const ObjectFormat = enum(u4) { -- 2.54.0 From b991b868ae2d6719dbe2e36f95c6d2cb799d8810 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 12:26:44 -0700 Subject: [PATCH 121/179] Configuration: prevent ObjectFormat enum mismatch --- lib/std/Build/Configuration.zig | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 2ad1cf4d03dd6d56a2c8f125a66dbf6703758915..8a1100fa1d3cbce7f2db220165037c7a51a6d619 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -2437,6 +2437,7 @@ pub const TargetQuery = struct { }; } }; + pub const ObjectFormat = enum(u4) { c, coff, @@ -2451,8 +2452,17 @@ pub const TargetQuery = struct { default, pub fn init(x: ?std.Target.ObjectFormat) @This() { - // TODO comptime assert the enums match - return @enumFromInt(@intFromEnum(x orelse return .default)); + return switch (x orelse return .default) { + .c => .c, + .coff => .coff, + .elf => .elf, + .hex => .hex, + .macho => .macho, + .plan9 => .plan9, + .raw => .raw, + .spirv => .spirv, + .wasm => .wasm, + }; } pub fn unwrap(this: @This()) ?std.Target.ObjectFormat { -- 2.54.0 From 940b9bccc2cba5df617112415fa745fde7002891 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 12:41:33 -0700 Subject: [PATCH 122/179] Configuration: fix Cpu.Arch enum mismatch --- lib/compiler/Maker/Step/Compile.zig | 4 +- lib/std/Build/Configuration.zig | 200 ++++++++++++++++++++++++++-- 2 files changed, 191 insertions(+), 13 deletions(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 881546f5122b7cd3b4cfec9e555871fa7c6c96d4..33da0b7df8d3fa714d8f54728f0bea97140296d5 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -1289,9 +1289,9 @@ fn appendModuleFlags( } } - if (m.resolved_target.get(conf)) |target| { + if (m.resolved_target.get(conf)) |resolved_target| { // Communicate the query via CLI since it's more compact. - if (target.query.get(conf)) |compact_query| { + if (resolved_target.query.get(conf)) |compact_query| { try zig_args.ensureUnusedCapacity(gpa, 6); const query = compact_query.unwrap(conf); diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 8a1100fa1d3cbce7f2db220165037c7a51a6d619..c6a8e37aa4c321b3366b00c4e85f8d17b6d22f8d 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -2178,6 +2178,7 @@ pub const TargetQuery = struct { }; } }; + pub const Abi = enum(u5) { none, gnu, @@ -2210,16 +2211,71 @@ pub const TargetQuery = struct { default, pub fn init(x: ?std.Target.Abi) @This() { - // TODO comptime assert the enums match - return @enumFromInt(@intFromEnum(x orelse return .default)); + return switch (x orelse return .default) { + .none => .none, + .gnu => .gnu, + .gnuabin32 => .gnuabin32, + .gnuabi64 => .gnuabi64, + .gnueabi => .gnueabi, + .gnueabihf => .gnueabihf, + .gnuf32 => .gnuf32, + .gnusf => .gnusf, + .gnux32 => .gnux32, + .eabi => .eabi, + .eabihf => .eabihf, + .ilp32 => .ilp32, + .android => .android, + .androideabi => .androideabi, + .musl => .musl, + .muslabin32 => .muslabin32, + .muslabi64 => .muslabi64, + .musleabi => .musleabi, + .musleabihf => .musleabihf, + .muslf32 => .muslf32, + .muslsf => .muslsf, + .muslx32 => .muslx32, + .msvc => .msvc, + .itanium => .itanium, + .simulator => .simulator, + .ohos => .ohos, + .ohoseabi => .ohoseabi, + }; } pub fn unwrap(this: @This()) ?std.Target.Abi { - // TODO comptime assert the enums match - if (this == .default) return null; - return @enumFromInt(@intFromEnum(this)); + return switch (this) { + .none => .none, + .gnu => .gnu, + .gnuabin32 => .gnuabin32, + .gnuabi64 => .gnuabi64, + .gnueabi => .gnueabi, + .gnueabihf => .gnueabihf, + .gnuf32 => .gnuf32, + .gnusf => .gnusf, + .gnux32 => .gnux32, + .eabi => .eabi, + .eabihf => .eabihf, + .ilp32 => .ilp32, + .android => .android, + .androideabi => .androideabi, + .musl => .musl, + .muslabin32 => .muslabin32, + .muslabi64 => .muslabi64, + .musleabi => .musleabi, + .musleabihf => .musleabihf, + .muslf32 => .muslf32, + .muslsf => .muslsf, + .muslx32 => .muslx32, + .msvc => .msvc, + .itanium => .itanium, + .simulator => .simulator, + .ohos => .ohos, + .ohoseabi => .ohoseabi, + .default => null, + }; } }; + pub const CpuArch = enum(u6) { aarch64, aarch64_be, @@ -2233,6 +2289,7 @@ pub const TargetQuery = struct { bpfeb, bpfel, csky, + ez80, hexagon, hppa, hppa64, @@ -2283,16 +2340,136 @@ pub const TargetQuery = struct { default, pub fn init(x: ?std.Target.Cpu.Arch) @This() { - // TODO comptime assert the enums match - return @enumFromInt(@intFromEnum(x orelse return .default)); + return switch (x orelse return .default) { + .aarch64 => .aarch64, + .aarch64_be => .aarch64_be, + .alpha => .alpha, + .amdgcn => .amdgcn, + .arc => .arc, + .arceb => .arceb, + .arm => .arm, + .armeb => .armeb, + .avr => .avr, + .bpfeb => .bpfeb, + .bpfel => .bpfel, + .csky => .csky, + .ez80 => .ez80, + .hexagon => .hexagon, + .hppa => .hppa, + .hppa64 => .hppa64, + .kalimba => .kalimba, + .kvx => .kvx, + .lanai => .lanai, + .loongarch32 => .loongarch32, + .loongarch64 => .loongarch64, + .m68k => .m68k, + .microblaze => .microblaze, + .microblazeel => .microblazeel, + .mips => .mips, + .mipsel => .mipsel, + .mips64 => .mips64, + .mips64el => .mips64el, + .msp430 => .msp430, + .nvptx => .nvptx, + .nvptx64 => .nvptx64, + .or1k => .or1k, + .powerpc => .powerpc, + .powerpcle => .powerpcle, + .powerpc64 => .powerpc64, + .powerpc64le => .powerpc64le, + .propeller => .propeller, + .riscv32 => .riscv32, + .riscv32be => .riscv32be, + .riscv64 => .riscv64, + .riscv64be => .riscv64be, + .s390x => .s390x, + .sh => .sh, + .sheb => .sheb, + .sparc => .sparc, + .sparc64 => .sparc64, + .spirv32 => .spirv32, + .spirv64 => .spirv64, + .thumb => .thumb, + .thumbeb => .thumbeb, + .ve => .ve, + .wasm32 => .wasm32, + .wasm64 => .wasm64, + .x86_16 => .x86_16, + .x86 => .x86, + .x86_64 => .x86_64, + .xcore => .xcore, + .xtensa => .xtensa, + .xtensaeb => .xtensaeb, + }; } pub fn unwrap(this: @This()) ?std.Target.Cpu.Arch { - // TODO comptime assert the enums match - if (this == .default) return null; - return @enumFromInt(@intFromEnum(this)); + return switch (this) { + .aarch64 => .aarch64, + .aarch64_be => .aarch64_be, + .alpha => .alpha, + .amdgcn => .amdgcn, + .arc => .arc, + .arceb => .arceb, + .arm => .arm, + .armeb => .armeb, + .avr => .avr, + .bpfeb => .bpfeb, + .bpfel => .bpfel, + .csky => .csky, + .ez80 => .ez80, + .hexagon => .hexagon, + .hppa => .hppa, + .hppa64 => .hppa64, + .kalimba => .kalimba, + .kvx => .kvx, + .lanai => .lanai, + .loongarch32 => .loongarch32, + .loongarch64 => .loongarch64, + .m68k => .m68k, + .microblaze => .microblaze, + .microblazeel => .microblazeel, + .mips => .mips, + .mipsel => .mipsel, + .mips64 => .mips64, + .mips64el => .mips64el, + .msp430 => .msp430, + .nvptx => .nvptx, + .nvptx64 => .nvptx64, + .or1k => .or1k, + .powerpc => .powerpc, + .powerpcle => .powerpcle, + .powerpc64 => .powerpc64, + .powerpc64le => .powerpc64le, + .propeller => .propeller, + .riscv32 => .riscv32, + .riscv32be => .riscv32be, + .riscv64 => .riscv64, + .riscv64be => .riscv64be, + .s390x => .s390x, + .sh => .sh, + .sheb => .sheb, + .sparc => .sparc, + .sparc64 => .sparc64, + .spirv32 => .spirv32, + .spirv64 => .spirv64, + .thumb => .thumb, + .thumbeb => .thumbeb, + .ve => .ve, + .wasm32 => .wasm32, + .wasm64 => .wasm64, + .x86_16 => .x86_16, + .x86 => .x86, + .x86_64 => .x86_64, + .xcore => .xcore, + .xtensa => .xtensa, + .xtensaeb => .xtensaeb, + + .default => null, + }; } }; + pub const OsTag = enum(u6) { freestanding, other, @@ -2390,7 +2567,6 @@ pub const TargetQuery = struct { pub fn unwrap(this: @This()) ?std.Target.Os.Tag { return switch (this) { - .default => null, .freestanding => .freestanding, .other => .other, .contiki => .contiki, @@ -2434,6 +2610,8 @@ pub const TargetQuery = struct { .opengl => .opengl, .vulkan => .vulkan, .tios => .tios, + + .default => null, }; } }; -- 2.54.0 From 88e066bb7b69d065758fa05442de6e317625df68 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 12:50:41 -0700 Subject: [PATCH 123/179] Configuration: fix dynamic linker null unwrap --- lib/std/Build/Configuration.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index c6a8e37aa4c321b3366b00c4e85f8d17b6d22f8d..0a1b56685cf448bdf68fad41c8eaca4318305f83 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -2696,7 +2696,7 @@ pub const TargetQuery = struct { null, .android_api_level = tq.android_api_level.value, .abi = tq.flags.abi.unwrap(), - .dynamic_linker = .init(if (tq.dynamic_linker.value) |s| s.slice(c) else null), + .dynamic_linker = if (tq.dynamic_linker.value) |s| .init(s.slice(c)) else null, .ofmt = tq.flags.object_format.unwrap(), }; } -- 2.54.0 From 91a7ea4ff41fd21e7465ddfe68bace533ab8c77d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 13:51:38 -0700 Subject: [PATCH 124/179] Maker: implement TranslateC --- lib/compiler/Maker/Step.zig | 3 +- lib/compiler/Maker/Step/Compile.zig | 28 ++-- lib/compiler/Maker/Step/TranslateC.zig | 187 ++++++++++++++----------- 3 files changed, 124 insertions(+), 94 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 6b74c1469adf77a577dabfc102d985750349c563..1f1a1a7ec4a6ee994b7b6860f2d200ddf536cf18 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -26,6 +26,7 @@ pub const InstallFile = @import("Step/InstallFile.zig"); pub const ObjCopy = @import("Step/ObjCopy.zig"); pub const Options = @import("Step/Options.zig"); pub const Run = @import("Step/Run.zig"); +pub const TranslateC = @import("Step/TranslateC.zig"); pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); pub const WriteFile = @import("Step/WriteFile.zig"); @@ -86,7 +87,7 @@ pub const Extended = union(enum) { options: Options, run: Run, top_level: TopLevel, - translate_c: Todo, + translate_c: TranslateC, update_source_files: UpdateSourceFiles, write_file: WriteFile, diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 33da0b7df8d3fa714d8f54728f0bea97140296d5..c237fd61e7c5fd80e9b927749930008b099f8194 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -291,18 +291,19 @@ fn lowerZigArgs( system_lib.flags.preferred_link_mode != prev_preferred_link_mode) and conf_comp.flags2.linkage != .static) { + try zig_args.ensureUnusedCapacity(gpa, 1); switch (system_lib.flags.search_strategy) { .no_fallback => switch (system_lib.flags.preferred_link_mode) { - .dynamic => try zig_args.append(gpa, "-search_dylibs_only"), - .static => try zig_args.append(gpa, "-search_static_only"), + .dynamic => zig_args.appendAssumeCapacity("-search_dylibs_only"), + .static => zig_args.appendAssumeCapacity("-search_static_only"), }, .paths_first => switch (system_lib.flags.preferred_link_mode) { - .dynamic => try zig_args.append(gpa, "-search_paths_first"), - .static => try zig_args.append(gpa, "-search_paths_first_static"), + .dynamic => zig_args.appendAssumeCapacity("-search_paths_first"), + .static => zig_args.appendAssumeCapacity("-search_paths_first_static"), }, .mode_first => switch (system_lib.flags.preferred_link_mode) { - .dynamic => try zig_args.append(gpa, "-search_dylibs_first"), - .static => try zig_args.append(gpa, "-search_static_first"), + .dynamic => zig_args.appendAssumeCapacity("-search_dylibs_first"), + .static => zig_args.appendAssumeCapacity("-search_static_first"), }, } prev_search_strategy = system_lib.flags.search_strategy; @@ -1317,6 +1318,7 @@ fn appendModuleFlags( try zig_args.append(gpa, try allocPrint(arena, "--export={s}", .{symbol_name.slice(conf)})); } + try zig_args.ensureUnusedCapacity(gpa, 2 * m.include_dirs.len); for (0..m.include_dirs.len) |i| try appendIncludeDirFlags(m.include_dirs.get(conf.extra, i), zig_args, asking_step, maker); @@ -1343,17 +1345,16 @@ fn appendModuleFlags( }; } -fn appendIncludeDirFlags( +/// Assumes unused capacity for at least 2 items. +pub fn appendIncludeDirFlags( include_dir: Configuration.Module.IncludeDir, zig_args: *std.ArrayList([]const u8), asking_step: Configuration.Step.Index, maker: *const Maker, ) !void { - const gpa = maker.gpa; const graph = maker.graph; const arena = graph.arena; // TODO don't leak into the process arena - try zig_args.ensureUnusedCapacity(gpa, 2); switch (include_dir) { .path => |lp| { zig_args.appendAssumeCapacity("-I"); @@ -1386,12 +1387,9 @@ fn appendIncludeDirFlags( comp.installed_headers_include_tree.?.getDirectory(); }, .embed_path => |lazy_path| { - try zig_args.append( - gpa, - try allocPrint(arena, "--embed-dir={f}", .{ - try maker.resolveLazyPathIndex(arena, lazy_path, asking_step), - }), - ); + zig_args.appendAssumeCapacity(try allocPrint(arena, "--embed-dir={f}", .{ + try maker.resolveLazyPathIndex(arena, lazy_path, asking_step), + })); }, } } diff --git a/lib/compiler/Maker/Step/TranslateC.zig b/lib/compiler/Maker/Step/TranslateC.zig index 37063cb22a912f8cbb6cacf2e224d74c2d81ad8d..68b32b16efab98711bab0c8b497359676f811f14 100644 --- a/lib/compiler/Maker/Step/TranslateC.zig +++ b/lib/compiler/Maker/Step/TranslateC.zig @@ -1,121 +1,152 @@ -fn make(step: *Step, options: Step.MakeOptions) !void { - const prog_node = options.progress_node; - const b = step.owner; - const translate_c: *TranslateC = @fieldParentPtr("step", step); - const arena = b.graph.arena; - - var argv_list = std.array_list.Managed([]const u8).init(b.allocator); - try argv_list.append(b.graph.zig_exe); - try argv_list.append("translate-c"); - if (translate_c.link_libc) { - try argv_list.append("-lc"); - } +const TranslateC = @This(); + +const std = @import("std"); +const Io = std.Io; +const Configuration = std.Build.Configuration; +const allocPrint = std.fmt.allocPrint; +const assert = std.debug.assert; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); +const PkgConfig = @import("../PkgConfig.zig"); - try argv_list.append("--cache-dir"); - try argv_list.append(b.cache_root.path orelse "."); +pub fn make( + translate_c: *TranslateC, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + _ = translate_c; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + const step = maker.stepByIndex(step_index); + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_tc = conf_step.extended.get(conf.extra).translate_c; + const cache_root = graph.local_cache_root; - try argv_list.append("--global-cache-dir"); - try argv_list.append(b.graph.global_cache_root.path orelse "."); + var argv: std.ArrayList([]const u8) = .empty; - if (!translate_c.target.query.isNative()) { - try argv_list.append("-target"); - try argv_list.append(try translate_c.target.query.zigTriple(b.allocator)); + try argv.ensureUnusedCapacity(arena, 10); + argv.appendAssumeCapacity(graph.zig_exe); + argv.appendAssumeCapacity("translate-c"); + if (conf_tc.flags.link_libc) { + argv.appendAssumeCapacity("-lc"); } - switch (translate_c.optimize) { - .Debug => {}, // Skip since it's the default. - else => try argv_list.append(b.fmt("-O{s}", .{@tagName(translate_c.optimize)})), + argv.appendAssumeCapacity("--cache-dir"); + argv.appendAssumeCapacity(cache_root.path orelse "."); + + argv.appendAssumeCapacity("--global-cache-dir"); + argv.appendAssumeCapacity(graph.global_cache_root.path orelse "."); + + if (conf_tc.target.get(conf).?.query.unwrap()) |compact_query| { + const query = compact_query.get(conf).unwrap(conf); + argv.appendAssumeCapacity("-target"); + argv.appendAssumeCapacity(try query.zigTriple(arena)); } - for (translate_c.include_dirs.items) |include_dir| { - try include_dir.appendZigProcessFlags(b, &argv_list, step); + switch (conf_tc.flags.optimize) { + .debug, .default => {}, // Skip since it's the default. + else => argv.appendAssumeCapacity(try allocPrint(arena, "-O{t}", .{conf_tc.flags.optimize})), } - for (translate_c.c_macros.items) |c_macro| { - try argv_list.append("-D"); - try argv_list.append(c_macro); + try argv.ensureUnusedCapacity(arena, conf_tc.include_dirs.len * 2); + for (0..conf_tc.include_dirs.len) |i| + try Step.Compile.appendIncludeDirFlags(conf_tc.include_dirs.get(conf.extra, i), &argv, step_index, maker); + + for (conf_tc.c_macros.slice) |c_macro| { + (try argv.addManyAsArray(arena, 2)).* = .{ "-D", c_macro.slice(conf) }; } var prev_search_strategy: std.Build.Module.SystemLib.SearchStrategy = .paths_first; var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; + var seen_system_libs: std.AutoArrayHashMapUnmanaged(Configuration.String, []const []const u8) = .empty; - for (translate_c.system_libs.items) |*system_lib| { - var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; + for (conf_tc.system_libs.slice) |system_lib_index| { + const system_lib = system_lib_index.get(conf); + const system_lib_name = system_lib.name.slice(conf); const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); if (system_lib_gop.found_existing) { - try argv_list.appendSlice(system_lib_gop.value_ptr.*); + try argv.appendSlice(arena, system_lib_gop.value_ptr.*); continue; } else { system_lib_gop.value_ptr.* = &.{}; } - if (system_lib.search_strategy != prev_search_strategy or - system_lib.preferred_link_mode != prev_preferred_link_mode) + if ((system_lib.flags.search_strategy != prev_search_strategy or + system_lib.flags.preferred_link_mode != prev_preferred_link_mode)) { - switch (system_lib.search_strategy) { - .no_fallback => switch (system_lib.preferred_link_mode) { - .dynamic => try argv_list.append("-search_dylibs_only"), - .static => try argv_list.append("-search_static_only"), + try argv.ensureUnusedCapacity(arena, 1); + switch (system_lib.flags.search_strategy) { + .no_fallback => switch (system_lib.flags.preferred_link_mode) { + .dynamic => argv.appendAssumeCapacity("-search_dylibs_only"), + .static => argv.appendAssumeCapacity("-search_static_only"), }, - .paths_first => switch (system_lib.preferred_link_mode) { - .dynamic => try argv_list.append("-search_paths_first"), - .static => try argv_list.append("-search_paths_first_static"), + .paths_first => switch (system_lib.flags.preferred_link_mode) { + .dynamic => argv.appendAssumeCapacity("-search_paths_first"), + .static => argv.appendAssumeCapacity("-search_paths_first_static"), }, - .mode_first => switch (system_lib.preferred_link_mode) { - .dynamic => try argv_list.append("-search_dylibs_first"), - .static => try argv_list.append("-search_static_first"), + .mode_first => switch (system_lib.flags.preferred_link_mode) { + .dynamic => argv.appendAssumeCapacity("-search_dylibs_first"), + .static => argv.appendAssumeCapacity("-search_static_first"), }, } - prev_search_strategy = system_lib.search_strategy; - prev_preferred_link_mode = system_lib.preferred_link_mode; + prev_search_strategy = system_lib.flags.search_strategy; + prev_preferred_link_mode = system_lib.flags.preferred_link_mode; } const prefix: []const u8 = prefix: { - if (system_lib.needed) break :prefix "-needed-l"; - if (system_lib.weak) break :prefix "-weak-l"; + if (system_lib.flags.needed) break :prefix "-needed-l"; + if (system_lib.flags.weak) break :prefix "-weak-l"; break :prefix "-l"; }; - switch (system_lib.use_pkg_config) { - .no => try argv_list.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), - .yes, .force => { - if (Step.Compile.runPkgConfig(&translate_c.step, system_lib.name)) |result| { - try argv_list.appendSlice(result.cflags); - try argv_list.appendSlice(result.libs); + l: { + pc: { + const force = switch (system_lib.flags.use_pkg_config) { + .no => break :pc, + .yes => false, + .force => true, + }; + + const pkg_conf_node = progress_node.start("pkg-config", 0); + defer pkg_conf_node.end(); + + if (PkgConfig.run(maker, step, pkg_conf_node, system_lib_name, force)) |result| { + try argv.appendSlice(arena, result.cflags); + try argv.appendSlice(arena, result.libs); try seen_system_libs.put(arena, system_lib.name, result.cflags); + break :l; } else |err| switch (err) { - error.PkgConfigInvalidOutput, - error.PkgConfigCrashed, - error.PkgConfigFailed, - error.PkgConfigNotInstalled, + error.PkgConfigUnavailable, error.PackageNotFound, - => switch (system_lib.use_pkg_config) { - .yes => { - // pkg-config failed, so fall back to linking the library - // by name directly. - try argv_list.append(b.fmt("{s}{s}", .{ - prefix, - system_lib.name, - })); - }, - .force => { - std.debug.panic("pkg-config failed for library {s}", .{system_lib.name}); - }, - .no => unreachable, + => { + // pkg-config failed, so fall back to linking the library by name directly. + assert(!force); + break :pc; }, - else => |e| return e, } - }, + } + try argv.append(arena, try allocPrint(arena, "{s}{s}", .{ + prefix, system_lib_name, + })); } } - const c_source_path = translate_c.source.getPath2(b, step); - try argv_list.append(c_source_path); + try argv.ensureUnusedCapacity(arena, 2); - try argv_list.append("--listen=-"); - const output_dir = try step.evalZigProcess(argv_list.items, prog_node, false, options.web_server, options.gpa); + const c_source_path = try maker.resolveLazyPathIndexAbs(arena, conf_tc.src_path, step_index); + argv.appendAssumeCapacity(c_source_path); - const basename = std.fs.path.stem(std.fs.path.basename(c_source_path)); - translate_c.out_basename = b.fmt("{s}.zig", .{basename}); - translate_c.output_file.path = output_dir.?.joinString(b.allocator, translate_c.out_basename) catch @panic("OOM"); + argv.appendAssumeCapacity("--listen=-"); + const output_dir_path = (Step.evalZigProcess(step_index, maker, argv.items, progress_node, false) catch |err| switch (err) { + error.NeedCompileErrorCheck => unreachable, + else => |e| return e, + }).?; + + const stem = Io.Dir.path.stem(Io.Dir.path.basename(c_source_path)); + const out_basename = try allocPrint(arena, "{s}.zig", .{stem}); + + maker.generatedPath(conf_tc.output_file).* = try output_dir_path.join(arena, out_basename); } -- 2.54.0 From 619f23b3c7a5276ab974478223ec41ce81c91b6c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 18:07:51 -0700 Subject: [PATCH 125/179] CLI: support both --fork=[path] and --fork [path] --- lib/compiler/Maker/ScannedConfig.zig | 2 +- lib/compiler/Maker/Step.zig | 6 +++--- src/main.zig | 31 ++++++++++++++++++---------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 5bb3a9871f53a1c50bb994cd73483315673f23c5..0f40833fc33e0672efdb46e1ef88427c91e78492 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -343,7 +343,7 @@ pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit \\ needed (Default) Lazy dependencies are fetched as needed \\ all Lazy dependencies are always fetched - \\ --fork=[path] Override one or more projects from dependency tree + \\ --fork=[path], --fork [path] Override one or more projects from dependency tree \\ \\Advanced Options: \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 1f1a1a7ec4a6ee994b7b6860f2d200ddf536cf18..a4760274cdeb5ab6b64651b2e79be61e1d8ebae6 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -728,19 +728,19 @@ fn failWithCacheError( switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail(maker, "failed to check cache: {t} {t}", .{ + .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail(maker, "failed checking cache: {t} {t}", .{ man.diagnostic, e, }), .file_open, .file_stat, .file_read, .file_hash => |op| { const pp = man.files.keys()[op.file_index].prefixed_path; const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - return s.fail(maker, "failed to check cache: '{s}{c}{s}' {t} {t}", .{ + return s.fail(maker, "failed checking cache: {s}{c}{s} {t} {t}", .{ prefix, Dir.path.sep, pp.sub_path, man.diagnostic, op.err, }); }, }, error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => return s.fail(maker, "failed to check cache: invalid manifest file format", .{}), + error.InvalidFormat => return s.fail(maker, "failed checking cache: invalid manifest file format", .{}), } } diff --git a/src/main.zig b/src/main.zig index cd2ec3c28e53923c20f0297aed6b72cd14284ba0..1567ae0a11f0c5082c7ceebf01960cfc65d65b84 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5082,17 +5082,12 @@ fn cmdBuild( fetch_mode = std.meta.stringToEnum(Package.Fetch.JobQueue.Mode, sub_arg) orelse fatal("expected [needed|all] after '--fetch=', found '{s}'", .{sub_arg}); } else if (mem.cutPrefix(u8, arg, "--fork=")) |sub_arg| { - try forks.append(arena, .{ - .manifest_ast = undefined, - .manifest = undefined, - .error_bundle = undefined, - .arena_allocator = undefined, - .path = .{ - .root_dir = .cwd(), - .sub_path = sub_arg, - }, - .failed = false, - }); + try forks.append(arena, .init(sub_arg)); + continue; + } else if (mem.eql(u8, arg, "--fork")) { + if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); + i += 1; + try forks.append(arena, .init(args[i])); continue; } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { @@ -5874,6 +5869,20 @@ const Fork = struct { failed: bool, arena_allocator: std.heap.ArenaAllocator, + fn init(cwd_relative_path: []const u8) Fork { + return .{ + .manifest_ast = undefined, + .manifest = undefined, + .error_bundle = undefined, + .arena_allocator = undefined, + .path = .{ + .root_dir = .cwd(), + .sub_path = cwd_relative_path, + }, + .failed = false, + }; + } + fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void { loadFallible(io, gpa, fork, color) catch |err| switch (err) { error.Canceled => |e| return e, -- 2.54.0 From df92898ec3f805112150ee4b40e0d02b763524bb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 20:05:55 -0700 Subject: [PATCH 126/179] Maker: fix packagePath function Currently, neither configurer nor Maker is aware of the standard zig package path, and the root path is stored as a bare string rather than relative to a known base directory. Without changing that, we must construct a cwd relative path here rather than using knowledge of the standard package path plus package hash. Also fixes a bug that would have been prevented by implementing the accepted proposal https://github.com/ziglang/zig/issues/25315 --- lib/compiler/Maker.zig | 14 ++++++-------- lib/compiler/Maker/Graph.zig | 1 - lib/compiler/configurer.zig | 22 ++++++++++++---------- lib/std/Build.zig | 1 + lib/std/Build/Cache/Path.zig | 2 +- lib/std/Build/Configuration.zig | 24 +++++++++++++++++------- src/Package/Fetch.zig | 4 ++-- src/main.zig | 32 +++++++++++++++++--------------- 8 files changed, 56 insertions(+), 44 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 52815f794505d7538c628d15fd2503d9540a9716..133c10e5f3daef8e91492018bfedb3c4e59d7824 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -123,10 +123,6 @@ pub fn main(init: process.Init.Minimal) !void { .local_cache_root = local_cache_directory, .zig_lib_directory = zig_lib_directory, .build_root_directory = build_root_directory, - .pkg_root = .{ - .root_dir = build_root_directory, - .sub_path = "zig-pkg", - }, }; graph.cache.addPrefix(.{ .path = null, .handle = cwd }); @@ -1779,11 +1775,13 @@ pub fn packagePath( .root_dir = graph.build_root_directory, .sub_path = sub_path, }; - const hash = package.hash.slice(c); - const pkg_root = graph.pkg_root; + // Currently, neither configurer nor Maker is aware of the standard zig + // package path, and the root path is stored as a bare string rather than + // relative to a known base directory. Without changing that, we must + // construct a cwd relative path here. return .{ - .root_dir = pkg_root.root_dir, - .sub_path = try Dir.path.join(arena, &.{ pkg_root.sub_path, hash, sub_path }), + .root_dir = .cwd(), + .sub_path = try Dir.path.join(arena, &.{ package.root_path.slice(c), sub_path }), }; } diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index 4e1c05b8c56ffdc554f9e52510a14a887c99b80a..667ce6d06d2b23999affbcde5524f48bc7bcf39e 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -18,7 +18,6 @@ global_cache_root: Directory, local_cache_root: Directory, zig_lib_directory: Directory, build_root_directory: Directory, -pkg_root: Path, debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, incremental: ?bool = null, diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index a4c661b1f4431b2cf5bcea46019d3afdae6b4b16..bfbdd14c6e6aabe9e36b2031d3e1e768cd583110 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -169,6 +169,7 @@ const Serialize = struct { gop.value_ptr.* = @enumFromInt(try wc.addExtra(@as(Configuration.Package, .{ .hash = try wc.addString(b.pkg_hash), .dep_prefix = try wc.addString(b.dep_prefix), + .root_path = try wc.addString(try b.root.toString(arena)), }))); } return gop.value_ptr.*; @@ -233,7 +234,7 @@ const Serialize = struct { fn addSystemLib(s: *Serialize, sl: *const std.Build.Module.SystemLib) !Configuration.SystemLib.Index { const wc = s.wc; - return @enumFromInt(try wc.addDeduped(@as(Configuration.SystemLib, .{ + return try wc.addDeduped(Configuration.SystemLib, .{ .flags = .{ .needed = sl.needed, .weak = sl.weak, @@ -242,7 +243,7 @@ const Serialize = struct { .search_strategy = sl.search_strategy, }, .name = try wc.addString(sl.name), - }))); + }); } fn addCSourceFile(s: *Serialize, csf: *const std.Build.Module.CSourceFile) !Configuration.CSourceFile.Index { @@ -291,10 +292,10 @@ const Serialize = struct { fn addEnvironMap(s: *Serialize, opt_map: ?*std.process.Environ.Map) !?Configuration.EnvironMap.Index { const wc = s.wc; const map = opt_map orelse return null; - return @enumFromInt(try wc.addDeduped(@as(Configuration.EnvironMap, .{ + return try wc.addDeduped(Configuration.EnvironMap, .{ .keys = try wc.addStringList(map.array_hash_map.keys()), .values = try wc.addStringList(map.array_hash_map.values()), - }))); + }); } fn initArgsList(s: *Serialize, args: []const Step.Run.Arg) ![]const Configuration.Step.Run.Arg.Index { @@ -643,9 +644,10 @@ const Serialize = struct { comptime assert(std.mem.eql(u8, @typeInfo(Configuration.Module).@"struct".fields[2].name, "import_table")); comptime assert(@typeInfo(Configuration.Module).@"struct".fields[2].type == Configuration.ImportTable.Index); assert(wc.extra.items[@intFromEnum(module_index) + 2] == @intFromEnum(Configuration.ImportTable.Index.invalid)); - wc.extra.items[@intFromEnum(module_index) + 2] = try wc.addDeduped(@as(Configuration.ImportTable, .{ + const import_table_index = try wc.addDeduped(Configuration.ImportTable, .{ .imports = .{ .mal = imports }, - })); + }); + wc.extra.items[@intFromEnum(module_index) + 2] = @intFromEnum(import_table_index); return module_index; } @@ -687,9 +689,9 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { for (dep_steps, step.dependencies.items) |*dest, src| dest.* = @enumFromInt(s.step_map.getIndex(src).?); - const deps: Configuration.Deps.Index = @enumFromInt(try wc.addDeduped(@as(Configuration.Deps, .{ + const deps: Configuration.Deps.Index = try wc.addDeduped(Configuration.Deps, .{ .steps = .{ .slice = dep_steps }, - }))); + }); try wc.steps.ensureTotalCapacity(gpa, s.step_map.entries.capacity); wc.steps.appendAssumeCapacity(.{ @@ -1278,10 +1280,10 @@ fn addOptionalResolvedTarget( optional_resolved_target: ?std.Build.ResolvedTarget, ) !Configuration.ResolvedTarget.OptionalIndex { const resolved_target = optional_resolved_target orelse return .none; - return @enumFromInt(try wc.addDeduped(@as(Configuration.ResolvedTarget, .{ + return .init(try wc.addDeduped(Configuration.ResolvedTarget, .{ .query = try wc.addTargetQuery(&resolved_target.query), .result = try wc.addTarget(resolved_target.result), - }))); + })); } fn addInstallDir(wc: *Configuration.Wip, install_dir: ?std.Build.InstallDir) !Configuration.InstallDestDir { diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 39b4b8993f65fee3ef842cf0ec4139d463e2ae9b..9081e3e29260e763b3915540782d714b16c8b0a2 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1896,6 +1896,7 @@ fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void { /// In other words, if this function returns `null` it means that the only /// purpose of completing the configure phase is to find out all the other lazy /// dependencies that are also required. +/// /// It is allowed to use this function for non-lazy dependencies, in which case /// it will never return `null`. This allows toggling laziness via /// build.zig.zon without changing build.zig logic. diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index 14f200579c3e0ea6b796fa9c663a6f39007bb131..91855a065279fb8ee9d9955878431635c683f4e1 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -24,7 +24,7 @@ pub fn cwd() Path { } pub fn initCwd(sub_path: []const u8) Path { - return .{ .root_dir = Cache.Directory.cwd(), .sub_path = sub_path }; + return .{ .root_dir = .cwd(), .sub_path = sub_path }; } pub fn join(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path { diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 0a1b56685cf448bdf68fad41c8eaca4318305f83..9c25cd4927e19aacc6befb6db00d9a7258296c37 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -374,25 +374,28 @@ pub const Wip = struct { /// Same as `addExtra` but uses a hash map to possibly return an already /// existing index instead of appending to `extra`. - pub fn addDeduped(wip: *Wip, extra: anytype) Allocator.Error!u32 { + pub fn addDeduped(wip: *Wip, comptime T: type, v: T) Allocator.Error!T.Index { const gpa = wip.gpa; const revert_index = wip.extra.items.len; - const extra_len = Storage.extraLen(extra); - try wip.extra.ensureUnusedCapacity(gpa, extra_len); - const new_index = addExtraAssumeCapacity(wip, extra); + const upper_bound_len = Storage.extraLen(v); + try wip.extra.ensureUnusedCapacity(gpa, upper_bound_len); + try wip.dedupe_table.ensureUnusedCapacityContext(gpa, 1, @as(ExtraSlice.Context, .{ + .extra = wip.extra.items, + })); + const new_index = addExtraAssumeCapacity(wip, v); const len: u32 = @intCast(wip.extra.items.len - new_index); assert(len != 0); - const gop = try wip.dedupe_table.getOrPutContext(gpa, .{ + const gop = wip.dedupe_table.getOrPutAssumeCapacityContext(.{ .index = new_index, .len = len, }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items })); if (gop.found_existing) { wip.extra.items.len = revert_index; - return gop.key_ptr.index; + return @enumFromInt(gop.key_ptr.index); } - return new_index; + return @enumFromInt(new_index); } pub fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 { @@ -1518,6 +1521,7 @@ pub const OptionalGeneratedFileIndex = enum(u32) { pub const Package = struct { dep_prefix: String, hash: String, + root_path: String, pub const Index = enum(u32) { root = max_u32, @@ -2075,6 +2079,12 @@ pub const ResolvedTarget = struct { none = max_u32, _, + pub fn init(i: Index) OptionalIndex { + const result: OptionalIndex = @enumFromInt(@intFromEnum(i)); + assert(result != .none); + return result; + } + pub fn unwrap(this: @This()) ?Index { return switch (this) { .none => null, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index ee4670070cd9989401703fe82083f2866ef23a71..c8cb0c8b9d797b06ffda726cf0ed9dca885956ad 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -782,7 +782,7 @@ fn runResource( f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice()); renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| { try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to rename temporary directory {f} into package cache directory {f}: {t}", + "failed renaming temporary directory {f} into package cache directory {f}: {t}", .{ package_sub_path, f.package_root, err }, ) }); return error.FetchFailed; @@ -802,7 +802,7 @@ fn runResource( if (!package_sub_path.eql(tmp_directory_path)) { tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }), + else => |e| log.warn("failed deleting temporary directory {f}: {t}", .{ tmp_directory_path, e }), }; } diff --git a/src/main.zig b/src/main.zig index 1567ae0a11f0c5082c7ceebf01960cfc65d65b84..36340131acb980e1051cea464f1181dadd796d9e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4985,16 +4985,16 @@ fn cmdBuild( configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig-lib-dir", undefined }; - const argv_index_zig_lib_dir = make_argv.items.len - 1; + const make_argv_index_zig_lib_dir = make_argv.items.len - 1; make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; const make_argv_index_build_root = make_argv.items.len - 1; make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--local-cache", undefined }; - const argv_index_cache_dir = make_argv.items.len - 1; + const make_argv_index_cache_dir = make_argv.items.len - 1; make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--global-cache", undefined }; - const argv_index_global_cache_dir = make_argv.items.len - 1; + const make_argv_index_global_cache_dir = make_argv.items.len - 1; make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--configuration", undefined }; const argv_index_configuration_file = make_argv.items.len - 1; @@ -5285,10 +5285,20 @@ fn cmdBuild( } }); defer _ = make_runner_task.cancel(io) catch {}; - make_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; + const pkg_root: Path = if (override_pkg_dir) |p| + .initCwd(p) + else if (system_pkg_dir_path) |p| + .initCwd(p) + else + .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", + }; + + make_argv.items[make_argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path; - make_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; - make_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; + make_argv.items[make_argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; + make_argv.items[make_argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; @@ -5385,15 +5395,7 @@ fn cmdBuild( .global_cache = dirs.global_cache, .local_storage = &.{ .cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" }, - .pkg_root = if (override_pkg_dir) |p| - .initCwd(p) - else if (system_pkg_dir_path) |p| - .initCwd(p) - else - .{ - .root_dir = build_root.directory, - .sub_path = "zig-pkg", - }, + .pkg_root = pkg_root, }, .recursive = true, .debug_hash = false, -- 2.54.0 From 4cbc03dce31b63818da1fb47f959b70708683443 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 20:08:28 -0700 Subject: [PATCH 127/179] Maker: we don't actually need to scan the modules --- lib/compiler/Maker.zig | 6 ------ lib/compiler/Maker/ScannedConfig.zig | 14 -------------- 2 files changed, 20 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 133c10e5f3daef8e91492018bfedb3c4e59d7824..f49090c208e472cf48c2f4da6bd5e93f297ebe71 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -427,7 +427,6 @@ pub fn main(init: process.Init.Minimal) !void { }; const c = &configuration; var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; - var modules: std.AutoArrayHashMapUnmanaged(Configuration.Module.Index, void) = .empty; for (configuration.steps, 0..) |*conf_step, step_index_usize| { if (conf_step.owner != .root) continue; const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); @@ -437,10 +436,6 @@ pub fn main(init: process.Init.Minimal) !void { const name = step_index.ptr(c).name.slice(c); try top_level_steps.put(arena, name, step_index); }, - .compile => { - const root_module = step_index.ptr(c).extended.get(configuration.extra).compile.root_module; - try modules.put(arena, root_module, {}); - }, else => {}, } } @@ -450,7 +445,6 @@ pub fn main(init: process.Init.Minimal) !void { break :sc .{ .configuration = configuration, .top_level_steps = top_level_steps, - .modules = modules, }; }; diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 0f40833fc33e0672efdb46e1ef88427c91e78492..787cd4dc5cc905bf1dc41ca1ebc975de6714e0c4 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -9,7 +9,6 @@ const Graph = @import("Graph.zig"); configuration: Configuration, top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), -modules: std.AutoArrayHashMapUnmanaged(Configuration.Module.Index, void), pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { std.log.err("TODO also print paths", .{}); @@ -45,19 +44,6 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { try tf.end(); } - { - var sf = try s.beginStructField("modules", .{}); - - for (sc.modules.keys()) |module_index| { - var int_buf: [50]u8 = undefined; - const int_str = std.fmt.bufPrint(&int_buf, "{d}", .{module_index}) catch unreachable; - var step_field = try sf.beginStructField(int_str, .{}); - try printStruct(sc, &step_field, Configuration.Module, module_index.get(c)); - try step_field.end(); - } - try sf.end(); - } - try s.end(); } -- 2.54.0 From 996b4118097c747e65ef1127091f7e48a54bfafc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 20:38:05 -0700 Subject: [PATCH 128/179] Configuration: more type safety for adding data erased method still exists for when the result will be converted to an int anyway. --- lib/compiler/configurer.zig | 131 +++++++++++++++----------------- lib/std/Build/Configuration.zig | 30 +++++--- 2 files changed, 83 insertions(+), 78 deletions(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index bfbdd14c6e6aabe9e36b2031d3e1e768cd583110..490d0e9df9bb32a42d23f8ec3a9aeb428e3cf766 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -166,11 +166,11 @@ const Serialize = struct { const wc = s.wc; const gop = try s.package_map.getOrPut(arena, b); if (!gop.found_existing) { - gop.value_ptr.* = @enumFromInt(try wc.addExtra(@as(Configuration.Package, .{ + gop.value_ptr.* = try wc.addExtra(Configuration.Package, .{ .hash = try wc.addString(b.pkg_hash), .dep_prefix = try wc.addString(b.dep_prefix), .root_path = try wc.addString(try b.root.toString(arena)), - }))); + }); } return gop.value_ptr.*; } @@ -180,38 +180,38 @@ const Serialize = struct { return @enumFromInt(switch (lp orelse return .none) { .src_path => |src_path| i: { const sub_path = try wc.addString(src_path.sub_path); - break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ + break :i try wc.addExtraErased(Configuration.LazyPath.SourcePath, .{ .owner = try s.builderToPackage(src_path.owner), .sub_path = sub_path, - })); + }); }, .generated => |generated| i: { const sub_path = try wc.addString(generated.sub_path); - break :i try wc.addExtra(@as(Configuration.LazyPath.Generated, .{ + break :i try wc.addExtraErased(Configuration.LazyPath.Generated, .{ .flags = .{ .up = @intCast(generated.up) }, .index = generated.index, .sub_path = sub_path, - })); + }); }, .cwd_relative => |cwd_relative_sub_path| i: { const sub_path = try wc.addString(cwd_relative_sub_path); - break :i try wc.addExtra(@as(Configuration.LazyPath.Relative, .{ + break :i try wc.addExtraErased(Configuration.LazyPath.Relative, .{ .flags = .{ .base = .cwd }, .sub_path = sub_path, - })); + }); }, .relative => |relative| i: { - break :i try wc.addExtra(@as(Configuration.LazyPath.Relative, .{ + break :i try wc.addExtraErased(Configuration.LazyPath.Relative, .{ .flags = .{ .base = relative.base }, .sub_path = relative.sub_path, - })); + }); }, .dependency => |dependency| i: { const sub_path = try wc.addString(dependency.sub_path); - break :i try wc.addExtra(@as(Configuration.LazyPath.SourcePath, .{ + break :i try wc.addExtraErased(Configuration.LazyPath.SourcePath, .{ .owner = try s.builderToPackage(dependency.dependency.builder), .sub_path = sub_path, - })); + }); }, }); } @@ -249,21 +249,21 @@ const Serialize = struct { fn addCSourceFile(s: *Serialize, csf: *const std.Build.Module.CSourceFile) !Configuration.CSourceFile.Index { const wc = s.wc; const args = try initStringList(s, csf.flags); - return @enumFromInt(try wc.addExtra(@as(Configuration.CSourceFile, .{ + return try wc.addExtra(Configuration.CSourceFile, .{ .flags = .{ .args_len = @intCast(args.len), .lang = .init(csf.language), }, .file = try addLazyPath(s, csf.file), .args = .{ .slice = args }, - }))); + }); } fn addCSourceFiles(s: *Serialize, csf: *const std.Build.Module.CSourceFiles) !Configuration.CSourceFiles.Index { const wc = s.wc; const sub_paths = try initStringList(s, csf.files); const args = try initStringList(s, csf.flags); - return @enumFromInt(try wc.addExtra(@as(Configuration.CSourceFiles, .{ + return try wc.addExtra(Configuration.CSourceFiles, .{ .flags = .{ .args_len = @intCast(args.len), .lang = .init(csf.language), @@ -271,14 +271,14 @@ const Serialize = struct { .root = try addLazyPath(s, csf.root), .sub_paths = .{ .slice = sub_paths }, .args = .{ .slice = args }, - }))); + }); } fn addRcSourceFile(s: *Serialize, rsf: *const std.Build.Module.RcSourceFile) !Configuration.RcSourceFile.Index { const wc = s.wc; const include_paths = try initLazyPathList(s, rsf.include_paths); const args = try initStringList(s, rsf.flags); - return @enumFromInt(try wc.addExtra(@as(Configuration.RcSourceFile, .{ + return try wc.addExtra(Configuration.RcSourceFile, .{ .flags = .{ .args_len = @intCast(args.len), .include_paths = include_paths.len != 0, @@ -286,7 +286,7 @@ const Serialize = struct { .file = try addLazyPath(s, rsf.file), .include_paths = .{ .slice = include_paths }, .args = .{ .slice = args }, - }))); + }); } fn addEnvironMap(s: *Serialize, opt_map: ?*std.process.Environ.Map) !?Configuration.EnvironMap.Index { @@ -302,7 +302,7 @@ const Serialize = struct { const wc = s.wc; const result = try s.arena.alloc(Configuration.Step.Run.Arg.Index, args.len); for (result, args) |*dest, src| { - dest.* = @enumFromInt(try wc.addExtra(@as(Configuration.Step.Run.Arg, switch (src) { + dest.* = try wc.addExtra(Configuration.Step.Run.Arg, switch (src) { .artifact => |a| .{ .flags = .{ .tag = .artifact, @@ -492,7 +492,7 @@ const Serialize = struct { .generated = .{ .value = null }, .target_query = .{ .value = a.target_query.unwrap() }, }, - }))); + }); } return result; } @@ -580,7 +580,7 @@ const Serialize = struct { const c_macros = try initStringList(s, m.c_macros.items); const export_symbol_names = try initStringList(s, m.export_symbol_names); - const module_index: Configuration.Module.Index = @enumFromInt(try wc.addExtra(@as(Configuration.Module, .{ + const module_index: Configuration.Module.Index = try wc.addExtra(Configuration.Module, .{ .flags = .{ .optimize = .init(m.optimize), .strip = .init(m.strip), @@ -622,7 +622,7 @@ const Serialize = struct { .rpaths = .init(rpaths), .link_objects = .init(link_objects), .frameworks = .{ .slice = frameworks }, - }))); + }); // The import table is the only place that modules can form dependency // loops. Therefore, we populate the module indexes only after adding @@ -699,12 +699,12 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .owner = try s.builderToPackage(step.owner), .deps = deps, .max_rss = .fromBytes(step.max_rss), - .extended = switch (step.tag) { + .extended = @enumFromInt(switch (step.tag) { .top_level => e: { const top_level: *Step.TopLevel = @fieldParentPtr("step", step); - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.TopLevel, .{ + break :e try wc.addExtraErased(Configuration.Step.TopLevel, .{ .description = try wc.addString(top_level.description), - }))); + }); }, .compile => e: { const c: *Step.Compile = @fieldParentPtr("step", step); @@ -712,14 +712,14 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { const installed_headers: []u32 = try arena.alloc(u32, c.installed_headers.items.len); for (installed_headers, c.installed_headers.items) |*dst, src| switch (src) { .file => |file| { - dst.* = try wc.addExtra(@as(Configuration.Step.Compile.InstalledHeader.File, .{ + dst.* = try wc.addExtraErased(Configuration.Step.Compile.InstalledHeader.File, .{ .source = try s.addLazyPath(file.source), .dest_sub_path = try wc.addString(file.dest_rel_path), - })); + }); }, .directory => |directory| { const include_extensions = directory.options.include_extensions orelse &.{}; - dst.* = try wc.addExtra(@as(Configuration.Step.Compile.InstalledHeader.Directory, .{ + dst.* = try wc.addExtraErased(Configuration.Step.Compile.InstalledHeader.Directory, .{ .flags = .{ .include_extensions = include_extensions.len != 0, .exclude_extensions = directory.options.exclude_extensions.len != 0, @@ -728,11 +728,11 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .dest_sub_path = try wc.addString(directory.dest_rel_path), .exclude_extensions = .{ .slice = try s.initStringList(directory.options.exclude_extensions) }, .include_extensions = .{ .slice = try s.initStringList(include_extensions) }, - })); + }); }, }; - const extra_index = try wc.addExtra(@as(Configuration.Step.Compile, .{ + break :e try wc.addExtraErased(Configuration.Step.Compile, .{ .flags = .{ .filters_len = c.filters.len != 0, .exec_cmd_args_len = exec_cmd_args.len != 0, @@ -891,13 +891,11 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .generated_llvm_bc = .{ .value = c.generated_llvm_bc.unwrap() }, .generated_llvm_ir = .{ .value = c.generated_llvm_ir.unwrap() }, .generated_h = .{ .value = c.generated_h.unwrap() }, - })); - - break :e @enumFromInt(extra_index); + }); }, .install_artifact => e: { const ia: *Step.InstallArtifact = @fieldParentPtr("step", step); - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.InstallArtifact, .{ + break :e try wc.addExtraErased(Configuration.Step.InstallArtifact, .{ .flags = .{ .dylib_symlinks = ia.dylib_symlinks, .bin_dir = ia.dest_dir != null, @@ -911,15 +909,15 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .pdb_dir = .{ .value = try addInstallDirDefaultNull(wc, ia.pdb_dir) }, .h_dir = .{ .value = try addInstallDirDefaultNull(wc, ia.h_dir) }, .bin_sub_path = .{ .value = try s.addOptionalString(ia.dest_sub_path) }, - }))); + }); }, .install_file => e: { const sif: *Step.InstallFile = @fieldParentPtr("step", step); - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.InstallFile, .{ + break :e try wc.addExtraErased(Configuration.Step.InstallFile, .{ .source = try s.addLazyPath(sif.source), .dest_dir = try addInstallDir(wc, sif.dir), .dest_sub_path = try wc.addString(sif.dest_rel_path), - }))); + }); }, .install_dir => e: { const sid: *Step.InstallDir = @fieldParentPtr("step", step); @@ -928,7 +926,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { else null; const include_extensions = sid.options.include_extensions orelse &.{}; - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.InstallDir, .{ + break :e try wc.addExtraErased(Configuration.Step.InstallDir, .{ .flags = .{ .dest_sub_path = dest_sub_path != null, .exclude_extensions = sid.options.exclude_extensions.len != 0, @@ -942,18 +940,18 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .exclude_extensions = .{ .slice = try s.initStringList(sid.options.exclude_extensions) }, .include_extensions = .{ .slice = try s.initStringList(include_extensions) }, .blank_extensions = .{ .slice = try s.initStringList(sid.options.blank_extensions) }, - }))); + }); }, .fail => e: { const sf: *Step.Fail = @fieldParentPtr("step", step); - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.Fail, .{ + break :e try wc.addExtraErased(Configuration.Step.Fail, .{ .msg = sf.error_msg, - }))); + }); }, .find_program => @panic("TODO"), .fmt => e: { const sf: *Step.Fmt = @fieldParentPtr("step", step); - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.Fmt, .{ + break :e try wc.addExtraErased(Configuration.Step.Fmt, .{ .flags = .{ .paths = sf.paths.len != 0, .exclude_paths = sf.exclude_paths.len != 0, @@ -961,7 +959,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { }, .paths = .{ .slice = try s.initLazyPathList(sf.paths) }, .exclude_paths = .{ .slice = try s.initLazyPathList(sf.exclude_paths) }, - }))); + }); }, .translate_c => e: { const tc: *Step.TranslateC = @fieldParentPtr("step", step); @@ -969,7 +967,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { const system_libs = try arena.alloc(Configuration.SystemLib.Index, tc.system_libs.items.len); for (system_libs, tc.system_libs.items) |*dest, *src| dest.* = try s.addSystemLib(src); - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.TranslateC, .{ + break :e try wc.addExtraErased(Configuration.Step.TranslateC, .{ .flags = .{ .include_dirs = tc.include_dirs.items.len != 0, .system_libs = system_libs.len != 0, @@ -983,7 +981,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .system_libs = .{ .slice = system_libs }, .c_macros = .{ .slice = tc.c_macros.items }, .target = try addOptionalResolvedTarget(wc, tc.target), - }))); + }); }, .write_file => e: { const wf: *Step.WriteFile = @fieldParentPtr("step", step); @@ -999,7 +997,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .include_extensions = src.include_extensions, }; - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.WriteFile, .{ + break :e try wc.addExtraErased(Configuration.Step.WriteFile, .{ .flags = .{ .embeds = wf.embeds.items.len != 0, .copies = wf.copies.items.len != 0, @@ -1018,18 +1016,18 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .mutate => |lp| try s.addLazyPath(lp), .whole_cached, .tmp => null, } }, - }))); + }); }, .update_source_files => e: { const usf: *Step.UpdateSourceFiles = @fieldParentPtr("step", step); - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.UpdateSourceFiles, .{ + break :e try wc.addExtraErased(Configuration.Step.UpdateSourceFiles, .{ .flags = .{ .embeds = usf.embeds.items.len != 0, .copies = usf.copies.items.len != 0, }, .embeds = .{ .slice = usf.embeds.items }, .copies = .{ .slice = try s.initCopyList(usf.copies.items) }, - }))); + }); }, .run => e: { const run: *Step.Run = @fieldParentPtr("step", step); @@ -1061,7 +1059,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { else => {}, } - const extra_index = try wc.addExtra(@as(Configuration.Step.Run, .{ + break :e try wc.addExtraErased(Configuration.Step.Run, .{ .flags = .{ .disable_zig_progress = run.disable_zig_progress, .skip_foreign_checks = run.skip_foreign_checks, @@ -1121,12 +1119,11 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .bytes => |bytes| .{ .bytes = try wc.addBytes(bytes) }, .lazy_path => |lp| .{ .lazy_path = try s.addLazyPath(lp) }, } }, - })); - break :e @enumFromInt(extra_index); + }); }, .check_file => e: { const cf: *Step.CheckFile = @fieldParentPtr("step", step); - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.CheckFile, .{ + break :e try wc.addExtraErased(Configuration.Step.CheckFile, .{ .flags = .{ .expected_exact = cf.expected_exact != null, .expected_matches = cf.expected_matches.len != 0, @@ -1136,7 +1133,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .expected_exact = .{ .value = cf.expected_exact }, .expected_matches = .{ .slice = cf.expected_matches }, .max_bytes = .{ .value = cf.max_bytes }, - }))); + }); }, .config_header => e: { const ch: *Step.ConfigHeader = @fieldParentPtr("step", step); @@ -1154,11 +1151,9 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .int => |x| switch (x) { 0 => .int_0, 1 => .int_1, - else => @enumFromInt(try wc.addExtra( - Configuration.Step.ConfigHeader.Value.initSigned(x), - )), + else => try wc.addExtra(Configuration.Step.ConfigHeader.Value, .initSigned(x)), }, - .ident => |x| @enumFromInt(try wc.addExtra(@as(Configuration.Step.ConfigHeader.Value, .{ + .ident => |x| try wc.addExtra(Configuration.Step.ConfigHeader.Value, .{ .flags = .{ .tag = .ident, .small = 0, @@ -1167,8 +1162,8 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .u64 = .{ .value = null }, .ident = .{ .value = try wc.addString(x) }, .string = .{ .value = null }, - }))), - .string => |x| @enumFromInt(try wc.addExtra(@as(Configuration.Step.ConfigHeader.Value, .{ + }), + .string => |x| try wc.addExtra(Configuration.Step.ConfigHeader.Value, .{ .flags = .{ .tag = .string, .small = 0, @@ -1177,10 +1172,10 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .u64 = .{ .value = null }, .ident = .{ .value = null }, .string = .{ .value = try wc.addString(x) }, - }))), + }), }, }; - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.ConfigHeader, .{ + break :e try wc.addExtraErased(Configuration.Step.ConfigHeader, .{ .flags = .{ .template_file = lazy_path != null, .style = .init(ch.style), @@ -1193,7 +1188,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .include_path = try wc.addString(ch.include_path), .include_guard = .{ .value = ch.include_guard.unwrap() }, .values = .{ .slice = pairs }, - }))); + }); }, .obj_copy => e: { const oc: *Step.ObjCopy = @fieldParentPtr("step", step); @@ -1217,7 +1212,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .file_path = try s.addLazyPath(src.file_path), }; - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.ObjCopy, .{ + break :e try wc.addExtraErased(Configuration.Step.ObjCopy, .{ .flags = .{ .basename = oc.basename != .none, .debug_file = debug_file != null, @@ -1239,7 +1234,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .pad_to = .{ .value = oc.pad_to }, .add_section = .{ .slice = add_sections }, .update_section = .{ .slice = oc.update_sections.items }, - }))); + }); }, .options => e: { const so: *Step.Options = @fieldParentPtr("step", step); @@ -1250,16 +1245,16 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .path = try s.addLazyPath(src.path), }; - break :e @enumFromInt(try wc.addExtra(@as(Configuration.Step.Options, .{ + break :e try wc.addExtraErased(Configuration.Step.Options, .{ .flags = .{ .args = so.args.items.len != 0, }, .generated_file = so.generated_file, .contents = try wc.addBytes(so.contents.items), .args = .{ .slice = args }, - }))); + }); }, - }, + }), }); } } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 9c25cd4927e19aacc6befb6db00d9a7258296c37..cfa156a8229cad2b8bed20b5b27de351fc47d193 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -254,7 +254,7 @@ pub const Wip = struct { null; const cpu_features_add_empty = q.cpu_features_add.isEmpty(); const cpu_features_sub_empty = q.cpu_features_sub.isEmpty(); - const result_index: TargetQuery.Index = @enumFromInt(try wip.addExtra(@as(TargetQuery, .{ + const result_index: TargetQuery.Index = try wip.addExtra(TargetQuery, .{ .flags = .{ .cpu_arch = .init(q.cpu_arch), .cpu_model = .init(q.cpu_model), @@ -277,7 +277,7 @@ pub const Wip = struct { .cpu_name = .{ .value = cpu_name }, .os_version_min = .{ .u = os_version_min }, .os_version_max = .{ .u = os_version_max }, - }))); + }); // Deduplicate. const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ @@ -329,7 +329,7 @@ pub const Wip = struct { }; const dynamic_linker: ?String = if (t.dynamic_linker.get()) |dl| try wip.addString(dl) else null; const cpu_features_add_empty = t.cpu.features.isEmpty(); - const result_index: TargetQuery.Index = @enumFromInt(try wip.addExtra(@as(TargetQuery, .{ + const result_index = try wip.addExtra(TargetQuery, .{ .flags = .{ .cpu_arch = .init(t.cpu.arch), .cpu_model = .explicit, @@ -352,7 +352,7 @@ pub const Wip = struct { .cpu_name = .{ .value = cpu_name }, .os_version_min = .{ .u = os_version_min }, .os_version_max = .{ .u = os_version_max }, - }))); + }); // Deduplicate. const gop = try wip.targets_table.getOrPutContext(gpa, result_index, @as(TargetsTableContext, .{ @@ -366,10 +366,16 @@ pub const Wip = struct { } } - pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { - const extra_len = Storage.extraLen(extra); + pub fn addExtra(wip: *Wip, comptime T: type, v: T) Allocator.Error!T.Index { + const extra_len = Storage.extraLen(v); try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len); - return addExtraAssumeCapacity(wip, extra); + return addExtraReserved(wip, T, v); + } + + pub fn addExtraErased(wip: *Wip, comptime T: type, v: T) Allocator.Error!u32 { + const extra_len = Storage.extraLen(v); + try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len); + return addExtraReservedErased(wip, T, v); } /// Same as `addExtra` but uses a hash map to possibly return an already @@ -382,7 +388,7 @@ pub const Wip = struct { try wip.dedupe_table.ensureUnusedCapacityContext(gpa, 1, @as(ExtraSlice.Context, .{ .extra = wip.extra.items, })); - const new_index = addExtraAssumeCapacity(wip, v); + const new_index = addExtraReservedErased(wip, T, v); const len: u32 = @intCast(wip.extra.items.len - new_index); assert(len != 0); const gop = wip.dedupe_table.getOrPutAssumeCapacityContext(.{ @@ -398,9 +404,13 @@ pub const Wip = struct { return @enumFromInt(new_index); } - pub fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 { + pub fn addExtraReserved(wip: *Wip, comptime T: type, v: T) T.Index { + return @enumFromInt(addExtraReservedErased(wip, T, v)); + } + + pub fn addExtraReservedErased(wip: *Wip, comptime T: type, v: T) u32 { const result: u32 = @intCast(wip.extra.items.len); - wip.extra.items.len = Storage.setExtra(wip.extra.allocatedSlice(), result, extra); + wip.extra.items.len = Storage.setExtra(wip.extra.allocatedSlice(), result, v); return result; } -- 2.54.0 From a874e729df771c23576e8fcab9d58e45a20dc1bd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2026 21:12:01 -0700 Subject: [PATCH 129/179] zig build: remove "cc args" from Run steps Not sure what I was thinking. This is silly, translate-c package simply needs to pass this data (link_libc and target) to the CLI application, which can then do the appropriate behavior. --- lib/compiler/Maker/Step/Run.zig | 3 --- lib/compiler/configurer.zig | 45 --------------------------------- lib/std/Build/Configuration.zig | 6 +---- lib/std/Build/Step/Run.zig | 23 ----------------- 4 files changed, 1 insertion(+), 76 deletions(-) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index c927d6d4af90ed114483a0b53504cf3c33a5fd7b..c0bb82cdef3799653b45090045f2c3d6105a5da6 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -177,9 +177,6 @@ pub fn make( man.hash.addListOfBytes(run_args); } }, - .cc_args => { - @panic("TODO Run make cc_args"); - }, } } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 490d0e9df9bb32a42d23f8ec3a9aeb428e3cf766..e16e9caac769b7c9eb53d315bbd0f9d47adb52eb 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -313,8 +313,6 @@ const Serialize = struct { .producer = true, .generated = false, .dep_file = false, - .target_query = false, - .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -322,7 +320,6 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = stepIndex(s, &a.artifact.step) }, .generated = .{ .value = null }, - .target_query = .{ .value = null }, }, .lazy_path => |a| .{ .flags = .{ @@ -334,8 +331,6 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, - .target_query = false, - .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -343,7 +338,6 @@ const Serialize = struct { .path = .{ .value = try addLazyPath(s, a.lazy_path) }, .producer = .{ .value = null }, .generated = .{ .value = null }, - .target_query = .{ .value = null }, }, .decorated_directory => |a| .{ .flags = .{ @@ -355,8 +349,6 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, - .target_query = false, - .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = if (a.suffix.len != 0) try wc.addString(a.suffix) else null }, @@ -364,7 +356,6 @@ const Serialize = struct { .path = .{ .value = try addLazyPath(s, a.lazy_path) }, .producer = .{ .value = null }, .generated = .{ .value = null }, - .target_query = .{ .value = null }, }, .file_content => |a| .{ .flags = .{ @@ -376,8 +367,6 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, - .target_query = false, - .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -385,7 +374,6 @@ const Serialize = struct { .path = .{ .value = try addLazyPath(s, a.lazy_path) }, .producer = .{ .value = null }, .generated = .{ .value = null }, - .target_query = .{ .value = null }, }, .bytes => |a| .{ .flags = .{ @@ -397,8 +385,6 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, - .target_query = false, - .link_libc = false, }, .prefix = .{ .value = try wc.addString(a) }, .suffix = .{ .value = null }, @@ -406,7 +392,6 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = null }, - .target_query = .{ .value = null }, }, .output_file, .output_file_dep => |a, tag| .{ .flags = .{ @@ -418,8 +403,6 @@ const Serialize = struct { .producer = false, .generated = true, .dep_file = tag == .output_file_dep, - .target_query = false, - .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -427,7 +410,6 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, - .target_query = .{ .value = null }, }, .output_directory => |a| .{ .flags = .{ @@ -439,8 +421,6 @@ const Serialize = struct { .producer = false, .generated = true, .dep_file = false, - .target_query = false, - .link_libc = false, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -448,7 +428,6 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = a.generated_file }, - .target_query = .{ .value = null }, }, .passthru => .{ .flags = .{ @@ -460,8 +439,6 @@ const Serialize = struct { .producer = false, .generated = false, .dep_file = false, - .target_query = false, - .link_libc = false, }, .prefix = .{ .value = null }, .suffix = .{ .value = null }, @@ -469,28 +446,6 @@ const Serialize = struct { .path = .{ .value = null }, .producer = .{ .value = null }, .generated = .{ .value = null }, - .target_query = .{ .value = null }, - }, - .cc_args => |a| .{ - .flags = .{ - .tag = .cc_args, - .prefix = false, - .suffix = false, - .basename = false, - .path = false, - .producer = false, - .generated = false, - .dep_file = false, - .target_query = a.target_query != .none, - .link_libc = a.link_libc, - }, - .prefix = .{ .value = null }, - .suffix = .{ .value = null }, - .basename = .{ .value = null }, - .path = .{ .value = null }, - .producer = .{ .value = null }, - .generated = .{ .value = null }, - .target_query = .{ .value = a.target_query.unwrap() }, }, }); } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index cfa156a8229cad2b8bed20b5b27de351fc47d193..b96e7922c5ac98db8aa1da14ab4a137ec5410293 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -586,7 +586,6 @@ pub const Step = extern struct { /// Always a compile step. producer: Storage.FlagOptional(.flags, .producer, Step.Index), generated: Storage.FlagOptional(.flags, .generated, GeneratedFileIndex), - target_query: Storage.FlagOptional(.flags, .target_query, TargetQuery.Index), pub const Flags = packed struct(u32) { tag: Arg.Tag, @@ -597,9 +596,7 @@ pub const Step = extern struct { producer: bool, generated: bool, dep_file: bool, - target_query: bool, - link_libc: bool, - _: u19 = 0, + _: u21 = 0, }; pub const Tag = enum(u4) { @@ -613,7 +610,6 @@ pub const Step = extern struct { output_file, output_directory, passthru, - cc_args, }; pub const Index = IndexType(@This()); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index c2c5a4f7a38e8217d7e79e4414abbab5d3451b4f..35699051fbfd511c399cb7033e613abebd9cf9d5 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -143,13 +143,6 @@ pub const Arg = union(enum) { output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. passthru, - /// Adds standard "-isystem" and "-iframework" arguments corresponding to the libc of the target. - cc_args: CcArgs, -}; - -pub const CcArgs = struct { - link_libc: bool, - target_query: Configuration.TargetQuery.OptionalIndex, }; pub const PrefixedArtifact = struct { @@ -536,22 +529,6 @@ pub fn addPassthruArgs(run: *Run) void { run.argv.append(arena, .passthru) catch @panic("OOM"); } -pub const AddCcArgs = struct { - link_libc: bool = false, - target_query: ?*const std.Target.Query = null, -}; - -/// Appends C compiler flags for the target and for including libc. -pub fn addCcArgs(run: *Run, options: AddCcArgs) void { - const graph = run.step.owner.graph; - const arena = graph.arena; - const wc = &graph.wip_configuration; - run.argv.append(arena, .{ .cc_args = .{ - .link_libc = options.link_libc, - .target_query = if (options.target_query) |q| wc.addTargetQuery(q) catch @panic("OOM") else .none, - } }) catch @panic("OOM"); -} - pub fn setStdIn(run: *Run, stdin: StdIn) void { switch (stdin) { .lazy_path => |lazy_path| lazy_path.addStepDependencies(&run.step), -- 2.54.0 From c678f94daa31f7ba8ea9f459eb4c442b02a8610c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2026 17:29:02 -0700 Subject: [PATCH 130/179] std.array_list: add last method, deprecated getLast --- lib/std/array_list.zig | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index 889915d8ba60cec429bd0dc10a931d8c78912782..1c665f3bdb892db15a3de341dc55d4194fb3db2b 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -1391,12 +1391,19 @@ pub fn Aligned(comptime T: type, comptime alignment: ?mem.Alignment) type { return self.allocatedSlice()[self.items.len..]; } - /// Returns the last element from the list, or `null` if the list is empty. + /// Deprecated in favor of `last`. pub fn getLast(self: Self) ?T { if (self.items.len == 0) return null; return self.items[self.items.len - 1]; } + /// Returns a pointer to the last element from the list, or `null` if + /// the list is empty. + pub fn last(self: Self) ?*T { + if (self.items.len == 0) return null; + return &self.items[self.items.len - 1]; + } + /// Called when memory growth is necessary. Returns a capacity larger than /// minimum that grows super-linearly. pub fn growCapacity(minimum: usize) usize { @@ -2378,17 +2385,16 @@ test "Managed(?u32).pop()" { try testing.expect(list.pop() == null); } -test "Managed(u32).getLast()" { +test "last" { const a = testing.allocator; - var list = Managed(u32).init(a); - defer list.deinit(); + var list: ArrayList(u32) = .empty; + defer list.deinit(a); - try testing.expectEqual(list.getLast(), null); + try testing.expectEqual(list.last(), null); - try list.append(2); - const const_list = list; - try testing.expectEqual(const_list.getLast().?, 2); + try list.append(a, 2); + try testing.expectEqual(list.last().?.*, 2); } test "return OutOfMemory when capacity would exceed maximum usize integer value" { -- 2.54.0 From df8aaad05852b18e5a47aa09d5acdfd312583d1c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2026 17:29:31 -0700 Subject: [PATCH 131/179] maker: extract some pkg-config logic into reusable API --- lib/compiler/Maker.zig | 7 +- lib/compiler/Maker/PkgConfig.zig | 168 +++++++------------------------ lib/std/zig.zig | 1 + lib/std/zig/PkgConfig.zig | 146 +++++++++++++++++++++++++++ 4 files changed, 184 insertions(+), 138 deletions(-) create mode 100644 lib/std/zig/PkgConfig.zig diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index f49090c208e472cf48c2f4da6bd5e93f297ebe71..1c26b6052aadf3a5d100a5730e90a755c27a5ba0 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -532,11 +532,7 @@ pub fn main(init: process.Init.Minimal) !void { .web_server = undefined, // set after `prepare` .memory_blocked_steps = .empty, .step_stack = .empty, - .pkg_config = .{ - .mutex = .init, - .list = null, - .debug = debug_pkg_config, - }, + .pkg_config = .{ .debug = debug_pkg_config }, .error_style = error_style, .multiline_errors = multiline_errors, @@ -1882,7 +1878,6 @@ pub fn truncatePath( if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ "truncate", try dest_path.toString(arena), }); - // https://codeberg.org/ziglang/zig/issues/35353 const err = e: { var file = f: { break :f dest_path.root_dir.handle.createFile(io, dest_path.sub_path, .{}) catch |err| switch (err) { diff --git a/lib/compiler/Maker/PkgConfig.zig b/lib/compiler/Maker/PkgConfig.zig index cafcf7c492e3be5986beb62e4b00759a6b01aa83..09fcbacfbdab1c07973a6fabd942f406a3ebb941 100644 --- a/lib/compiler/Maker/PkgConfig.zig +++ b/lib/compiler/Maker/PkgConfig.zig @@ -7,13 +7,8 @@ const Maker = @import("../Maker.zig"); const Step = @import("Step.zig"); const Graph = @import("Graph.zig"); -pub const Pkg = struct { - name: []const u8, - desc: []const u8, -}; - mutex: Io.Mutex = .init, -list: ?[]const Pkg = null, +pkgs: ?std.zig.PkgConfig = null, debug: bool = false, pub const RunError = error{ @@ -21,10 +16,7 @@ pub const RunError = error{ PkgConfigUnavailable, } || Step.ExtendedMakeError; -pub const Result = struct { - cflags: []const []const u8, - libs: []const []const u8, -}; +pub const Result = std.zig.PkgConfig.Parsed; /// Run pkg-config for the given library name and parse the output, returning the arguments /// that should be passed to zig to link the given library. @@ -34,131 +26,50 @@ pub fn run( progress_node: std.Progress.Node, lib_name: []const u8, /// If true, reports failure error messages on step rather than returning - /// error.PackageNotFound or error.PkgConfigInvalidOutput, + /// error.PackageNotFound or error.PkgConfigUnavailable, force: bool, ) RunError!Result { const pc = &maker.pkg_config; const graph = maker.graph; const arena = graph.arena; // TODO don't leak into process arena - const pkg_name = match: { - // First we have to map the library name to pkg config name. Unfortunately, - // there are several examples where this is not straightforward: - // -lSDL2 -> pkg-config sdl2 - // -lgdk-3 -> pkg-config gdk-3.0 - // -latk-1.0 -> pkg-config atk - // -lpulse -> pkg-config libpulse - const pkgs = try getList(maker, step, progress_node, force); - - // Exact match means instant winner. - for (pkgs) |pkg| { - if (mem.eql(u8, pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Next we'll try ignoring case. - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Prefixed "lib" or suffixed ".0". - for (pkgs) |pkg| { - if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { - const prefix = pkg.name[0..pos]; - const suffix = pkg.name[pos + lib_name.len ..]; - if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; - if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; - break :match pkg.name; - } - } - - // Trimming "-1.0". - if (mem.endsWith(u8, lib_name, "-1.0")) { - const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { - break :match pkg.name; - } - } - } - - if (force) return step.fail(maker, "{s}: package not found: {s}", .{ - getExe(graph), lib_name, - }); - + const pkg_config_exe = getExe(graph); + const pkgs = try getPkgs(maker, step, progress_node, force); + const found_index = pkgs.find(lib_name) orelse { + if (force) return step.fail(maker, "{s}: package not found: {s}", .{ pkg_config_exe, lib_name }); return error.PackageNotFound; }; + const pkg = pkgs.all[found_index]; - const pkg_config_exe = getExe(graph); const stdout = try captureChildProcess(maker, step, .{ - .argv = &.{ pkg_config_exe, pkg_name, "--cflags", "--libs" }, + .argv = &.{ pkg_config_exe, pkg.name, "--cflags", "--libs" }, .progress_node = progress_node, .allow_failure = !force, }); - var zig_cflags: std.ArrayList([]const u8) = .empty; - var zig_libs: std.ArrayList([]const u8) = .empty; - var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); - - while (arg_it.next()) |arg| { - if (mem.eql(u8, arg, "-I")) { - const dir = arg_it.next() orelse return missingArg(maker, step, pkg_config_exe, lib_name, arg, force); - try zig_cflags.appendSlice(arena, &.{ "-I", dir }); - } else if (mem.startsWith(u8, arg, "-I")) { - try zig_cflags.append(arena, arg); - } else if (mem.eql(u8, arg, "-L")) { - const dir = arg_it.next() orelse return missingArg(maker, step, pkg_config_exe, lib_name, arg, force); - try zig_libs.appendSlice(arena, &.{ "-L", dir }); - } else if (mem.startsWith(u8, arg, "-L")) { - try zig_libs.append(arena, arg); - } else if (mem.eql(u8, arg, "-l")) { - const lib = arg_it.next() orelse return missingArg(maker, step, pkg_config_exe, lib_name, arg, force); - try zig_libs.appendSlice(arena, &.{ "-l", lib }); - } else if (mem.startsWith(u8, arg, "-l")) { - try zig_libs.append(arena, arg); - } else if (mem.eql(u8, arg, "-D")) { - const macro = arg_it.next() orelse return missingArg(maker, step, pkg_config_exe, lib_name, arg, force); - try zig_cflags.appendSlice(arena, &.{ "-D", macro }); - } else if (mem.startsWith(u8, arg, "-D")) { - try zig_cflags.append(arena, arg); - } else if (mem.cutPrefix(u8, arg, "-Wl,-rpath,")) |rest| { - try zig_cflags.appendSlice(arena, &.{ "-rpath", rest }); - } else if (force or pc.debug) { - return step.fail(maker, "{s} package {s} unknown flag: {s}", .{ pkg_config_exe, lib_name, arg }); + const parsed = std.zig.PkgConfig.parse(arena, stdout) catch |err| switch (err) { + error.InvalidPkgConfigOutput => { + if (force) return step.fail(maker, "{s} package {s} invalid output: {s}", .{ + pkg_config_exe, lib_name, stdout, + }); + return error.PkgConfigUnavailable; + }, + else => |e| return e, + }; + if (force or pc.debug) { + for (parsed.unknown_flags) |unknown_flag| { + return step.fail(maker, "{s} package {s} unknown flag: {s}", .{ pkg_config_exe, lib_name, unknown_flag }); } } - try zig_cflags.shrinkToLen(arena); - try zig_libs.shrinkToLen(arena); - - return .{ - .cflags = zig_cflags.toOwnedSliceAssert(), - .libs = zig_libs.toOwnedSliceAssert(), - }; -} - -fn missingArg( - maker: *Maker, - step: *Step, - pkg_config_exe: []const u8, - lib_name: []const u8, - arg: []const u8, - force: bool, -) RunError { - if (force) return step.fail(maker, "{s} package {s} missing arg after flag: {s}", .{ - pkg_config_exe, lib_name, arg, - }); - return error.PkgConfigUnavailable; + return parsed; } fn getExe(graph: *const Graph) []const u8 { - return std.zig.EnvVar.PKG_CONFIG.get(&graph.environ_map) orelse "pkg-config"; + return std.zig.PkgConfig.exe(&graph.environ_map); } -fn getList(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force: bool) RunError![]const Pkg { +fn getPkgs(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force: bool) RunError!std.zig.PkgConfig { const graph = maker.graph; const arena = graph.arena; // TODO don't leak into process arena const io = graph.io; @@ -167,7 +78,7 @@ fn getList(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force: try pc.mutex.lock(io); defer pc.mutex.unlock(io); - if (pc.list) |list| return list; + if (pc.pkgs) |pkgs| return pkgs; const pkg_config_exe = getExe(graph); const stdout = try captureChildProcess(maker, step, .{ @@ -176,25 +87,18 @@ fn getList(maker: *Maker, step: *Step, progress_node: std.Progress.Node, force: .allow_failure = !force, }); - var list: std.ArrayList(Pkg) = .empty; - var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); - while (line_it.next()) |line| { - if (mem.trim(u8, line, " \t").len == 0) continue; - var tok_it = mem.tokenizeAny(u8, line, " \t"); - try list.append(arena, .{ - .name = tok_it.next() orelse { - if (force) return step.fail(maker, "{s}: invalid line: {s}", .{ - pkg_config_exe, line, - }); - return error.PkgConfigUnavailable; - }, - .desc = tok_it.rest(), - }); - } - try list.shrinkToLen(arena); + var diagnostic: std.zig.PkgConfig.Diagnostic = undefined; + const result = std.zig.PkgConfig.init(arena, stdout, &diagnostic) catch |err| switch (err) { + error.InvalidPkgConfigOutput => { + if (force) return step.fail(maker, "{s}: invalid line({d}): {s}", .{ + pkg_config_exe, diagnostic.invalid_line_index + 1, diagnostic.invalid_line, + }); + return error.PkgConfigUnavailable; + }, + else => |e| return e, + }; - const result = list.toOwnedSliceAssert(); - pc.list = result; + pc.pkgs = result; return result; } diff --git a/lib/std/zig.zig b/lib/std/zig.zig index faa816c575337ae1a08d8967244502621cf40e88..2a3b4792a826e5fc28ffe9373c73bb7b6f63f6b1 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -33,6 +33,7 @@ pub const AstRlAnnotate = @import("zig/AstRlAnnotate.zig"); pub const LibCInstallation = @import("zig/LibCInstallation.zig"); pub const WindowsSdk = @import("zig/WindowsSdk.zig"); pub const LibCDirs = @import("zig/LibCDirs.zig"); +pub const PkgConfig = @import("zig/PkgConfig.zig"); pub const target = @import("zig/target.zig"); pub const llvm = @import("zig/llvm.zig"); diff --git a/lib/std/zig/PkgConfig.zig b/lib/std/zig/PkgConfig.zig new file mode 100644 index 0000000000000000000000000000000000000000..c37b1476436f87363132b2f2f567d844759abddc --- /dev/null +++ b/lib/std/zig/PkgConfig.zig @@ -0,0 +1,146 @@ +//! The more reusable pieces of the build system's pkg-config integration logic. +const PkgConfig = @This(); + +const std = @import("../std.zig"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; + +all: []const Pkg, + +pub const Pkg = struct { + name: []const u8, + desc: []const u8, +}; + +pub const InitError = Allocator.Error || error{InvalidPkgConfigOutput}; + +pub const Diagnostic = struct { + invalid_line_index: usize, + invalid_line: []const u8, +}; + +/// Parses the output of `pkg-config --list-all`. +pub fn init(arena: Allocator, stdout: []const u8, diagnostic: ?*Diagnostic) InitError!PkgConfig { + var list: std.ArrayList(Pkg) = .empty; + var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); + var line_index: usize = 0; + while (line_it.next()) |line| : (line_index += 1) { + if (mem.trim(u8, line, " \t").len == 0) continue; + var tok_it = mem.tokenizeAny(u8, line, " \t"); + try list.append(arena, .{ + .name = tok_it.next() orelse { + if (diagnostic) |d| d.* = .{ + .invalid_line_index = line_index, + .invalid_line = line, + }; + return error.InvalidPkgConfigOutput; + }, + .desc = tok_it.rest(), + }); + } + try list.shrinkToLen(arena); + return .{ .all = list.toOwnedSliceAssert() }; +} + +// Maps the library name to pkg config name. Unfortunately, there are several +// examples where this is not straightforward: +// * -lSDL2 -> pkg-config sdl2 +// * -lgdk-3 -> pkg-config gdk-3.0 +// * -latk-1.0 -> pkg-config atk +// * -lpulse -> pkg-config libpulse +pub fn find(pc: *const PkgConfig, lib_name: []const u8) ?usize { + const all = pc.all; + + // Exact match means instant winner. + for (all, 0..) |pkg, i| { + if (mem.eql(u8, pkg.name, lib_name)) + return i; + } + + // Next we'll try ignoring case. + for (all, 0..) |pkg, i| { + if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) + return i; + } + + // Prefixed "lib" or suffixed ".0". + for (all, 0..) |pkg, i| { + if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { + const prefix = pkg.name[0..pos]; + const suffix = pkg.name[pos + lib_name.len ..]; + if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; + if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; + return i; + } + } + + // Trimming "-1.0". + if (mem.cutSuffix(u8, lib_name, "-1.0")) |trimmed| { + for (all, 0..) |pkg, i| { + if (std.ascii.eqlIgnoreCase(pkg.name, trimmed)) { + return i; + } + } + } + + return null; +} + +pub fn exe(environ_map: *const std.process.Environ.Map) []const u8 { + return std.zig.EnvVar.PKG_CONFIG.get(environ_map) orelse "pkg-config"; +} + +pub const Parsed = struct { + cflags: []const []const u8, + libs: []const []const u8, + unknown_flags: []const []const u8, +}; + +pub const ParseError = Allocator.Error || error{InvalidPkgConfigOutput}; + +/// Parses the output of `pkg-config [name] --cflags --libs`. +pub fn parse(arena: Allocator, stdout: []const u8) ParseError!Parsed { + var zig_cflags: std.ArrayList([]const u8) = .empty; + var zig_libs: std.ArrayList([]const u8) = .empty; + var unknown_flags: std.ArrayList([]const u8) = .empty; + var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); + + while (arg_it.next()) |arg| { + if (mem.eql(u8, arg, "-I")) { + const dir = arg_it.next() orelse return error.InvalidPkgConfigOutput; + try zig_cflags.appendSlice(arena, &.{ "-I", dir }); + } else if (mem.startsWith(u8, arg, "-I")) { + try zig_cflags.append(arena, arg); + } else if (mem.eql(u8, arg, "-L")) { + const dir = arg_it.next() orelse return error.InvalidPkgConfigOutput; + try zig_libs.appendSlice(arena, &.{ "-L", dir }); + } else if (mem.startsWith(u8, arg, "-L")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-l")) { + const lib = arg_it.next() orelse return error.InvalidPkgConfigOutput; + try zig_libs.appendSlice(arena, &.{ "-l", lib }); + } else if (mem.startsWith(u8, arg, "-l")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-D")) { + const macro = arg_it.next() orelse return error.InvalidPkgConfigOutput; + try zig_cflags.appendSlice(arena, &.{ "-D", macro }); + } else if (mem.startsWith(u8, arg, "-D")) { + try zig_cflags.append(arena, arg); + } else if (mem.cutPrefix(u8, arg, "-Wl,-rpath,")) |rest| { + try zig_cflags.appendSlice(arena, &.{ "-rpath", rest }); + } else { + try unknown_flags.append(arena, arg); + } + } + + try zig_cflags.shrinkToLen(arena); + try zig_libs.shrinkToLen(arena); + try unknown_flags.shrinkToLen(arena); + + return .{ + .cflags = zig_cflags.toOwnedSliceAssert(), + .libs = zig_libs.toOwnedSliceAssert(), + .unknown_flags = unknown_flags.toOwnedSliceAssert(), + }; +} -- 2.54.0 From 54bb8d2dd9369f5e5b43b4773878edc32fd3851e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2026 22:00:29 -0700 Subject: [PATCH 132/179] implement the concept of configure cache poisoning --- lib/compiler/Maker.zig | 27 +++- lib/compiler/Maker/ScannedConfig.zig | 6 + lib/compiler/configurer.zig | 7 + lib/std/Build.zig | 114 +++++++++++++- lib/std/Build/Configuration.zig | 13 ++ src/main.zig | 228 +++++++++++++-------------- 6 files changed, 270 insertions(+), 125 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 1c26b6052aadf3a5d100a5730e90a755c27a5ba0..b1b106969be24639dd23bc03b0fff317725dc094 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -425,6 +425,10 @@ pub fn main(init: process.Init.Minimal) !void { break :c Configuration.loadFile(arena, io, file) catch |err| fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); }; + // Technically if the configuration is marked as poisoned, we could + // already delete the file now, but we leave it around in case the + // maker process fails or crashes and it's helpful to be able to repeat + // execution of the command line or otherwise inspect the configuration file. const c = &configuration; var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; for (configuration.steps, 0..) |*conf_step, step_index_usize| { @@ -445,6 +449,7 @@ pub fn main(init: process.Init.Minimal) !void { break :sc .{ .configuration = configuration, .top_level_steps = top_level_steps, + .path = configure_path, }; }; @@ -455,7 +460,7 @@ pub fn main(init: process.Init.Minimal) !void { else => |e| return e, }; w.flush() catch return stdout_writer_allocation.err.?; - return; + return cleanExit(io, &scanned_config); } else if (steps_menu) { var w = initStdoutWriter(io); scanned_config.printSteps(&graph, w) catch |err| switch (err) { @@ -463,12 +468,12 @@ pub fn main(init: process.Init.Minimal) !void { else => |e| return e, }; w.flush() catch return stdout_writer_allocation.err.?; - return; + return cleanExit(io, &scanned_config); } else if (print_configuration) { var w = initStdoutWriter(io); scanned_config.print(w) catch return stdout_writer_allocation.err.?; w.flush() catch return stdout_writer_allocation.err.?; - return; + return cleanExit(io, &scanned_config); } if (webui_listen != null) { @@ -1000,6 +1005,8 @@ fn makeStepNames( if (maker.error_style.verboseContext()) break :code 1; // failure; print build command break :code 2; // failure; do not print build command }; + if (code == 0) removePoisonedConfiguration(io, maker.scanned_config); + cleanup_task.await(io); // There is a defer above but an exit below. _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(code); } @@ -2000,3 +2007,17 @@ fn installSymLinksInner( return step.fail(maker, "unable to symlink {f} -> {s}: {t}", .{ name_only_path, filename_major_only, err }); }; } + +fn cleanExit(io: Io, scanned_config: *const ScannedConfig) void { + removePoisonedConfiguration(io, scanned_config); + return process.cleanExit(io); +} + +fn removePoisonedConfiguration(io: Io, scanned_config: *const ScannedConfig) void { + if (scanned_config.configuration.poisoned) { + // This configuration file was good for only 1 invocation of the maker + // process. Delete it to save space on disk. + Io.Dir.cwd().deleteFile(io, scanned_config.path) catch |err| + log.warn("failed deleting poisoned configuration file {s}: {t}", .{ scanned_config.path, err }); + } +} diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 787cd4dc5cc905bf1dc41ca1ebc975de6714e0c4..748cc84e97fc52872056cc32102ecf382ca97038 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -9,6 +9,7 @@ const Graph = @import("Graph.zig"); configuration: Configuration, top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), +path: []const u8, pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { std.log.err("TODO also print paths", .{}); @@ -343,6 +344,11 @@ pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { \\ --zig-lib-dir [arg] Override path to Zig lib directory \\ --build-runner [file] Override path to build runner \\ --seed [integer] For shuffling dependency traversal order (default: random) + \\ --cache-poison[=mode] Override configuration caching behavior + \\ pure (default) Avoid false positive cache hits + \\ poisoned Don't cache the configuration + \\ disallowed Panics when cache would be poisoned + \\ ignored A little poison never hurt anybody \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index e16e9caac769b7c9eb53d315bbd0f9d47adb52eb..c8aa77edd8d79c61533ec00e6ac0cd5248cbc86f 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -114,6 +114,9 @@ pub fn main(init: process.Init.Minimal) !void { graph.system_package_mode = true; } else if (mem.eql(u8, arg, "--verbose")) { graph.verbose = true; + } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { + graph.cache_poison = std.meta.stringToEnum(std.Build.Graph.CachePoison, rest) orelse + fatalWithHint("expected --cache-poison=[pure|poisoned|disallowed|ignored]; found: {s}", .{arg}); } else { fatalWithHint("unrecognized argument: {s}", .{arg}); } @@ -1222,6 +1225,10 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { try wc.write(writer, .{ .default_step = s.stepIndex(b.default_step), .generated_files_len = @intCast(graph.generated_files.items.len), + .poisoned = switch (graph.cache_poison) { + .pure, .disallowed, .ignored => false, + .poisoned => true, + }, }); } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 9081e3e29260e763b3915540782d714b16c8b0a2..fbf2fb0e5047b7c598b86635fdf9e139ce96fc35 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -98,6 +98,35 @@ pub const Graph = struct { generated_files: std.ArrayList(*Step), wip_configuration: Configuration.Wip, + cache_poison: CachePoison = .pure, + + /// If the cache is poisoned means that the **configure logic** had side + /// effects, or otherwise did something that could not be tracked by the + /// cache system. + /// + /// This is not to be confused with whether individual steps may have side + /// effects when being evaluated; it has to do with the logic inside build.zig + /// itself. For example, a `Run` step that prints "hello world" has side + /// effects *at make time* and therefore does not warrant setting this flag, + /// while checking for the existence of `scdoc` *at configure time* in order to + /// choose the default value for a configuration option does. + /// + /// Keeping the cache pure will make `zig build` faster, bypassing the + /// configurer process when identical configuration would be generated. + /// + /// When the cache is poisoned, the maker process will delete the build + /// configuration file upon ingesting it since it cannot be reused. + pub const CachePoison = enum { + pure, + poisoned, + /// Indicates the user would like to see a stack trace if the cache + /// would become poisoned. + disallowed, + /// Indicates the user would like to ignore the cache being poisoned + /// and cache anyway, opting into cache hits on stale configuration. + ignored, + }; + pub fn addGeneratedFile(graph: *Graph, owner: *Step) Configuration.GeneratedFileIndex { graph.generated_files.append(graph.arena, owner) catch @panic("OOM"); return @enumFromInt(graph.generated_files.items.len - 1); @@ -169,6 +198,19 @@ pub const Graph = struct { const wc = &graph.wip_configuration; return wc.addString(bytes) catch @panic("OOM"); } + + /// Indicates that the **configure logic** had side effects, or otherwise + /// did something that could not be tracked by the cache system. + /// + /// See `CachePoison` documentation for more details. + pub fn poisonCache(graph: *Graph) void { + switch (graph.cache_poison) { + .pure => graph.cache_poison = .poisoned, + .poisoned => return, + .disallowed => @panic("cache poisoned"), + .ignored => log.warn("ignoring cache poisoning", .{}), + } + } }; const AvailableDeps = []const struct { []const u8, []const u8 }; @@ -798,11 +840,23 @@ pub fn createModule(b: *Build, options: Module.CreateOptions) *Module { return Module.create(b, options); } -/// Initializes a `Step.Run` with argv, which must at least have the path to the -/// executable. More command line arguments can be added with `addArg`, -/// `addArgs`, and `addArtifactArg`. -/// Be careful using this function, as it introduces a system dependency. -/// To run an executable built with zig build, see `Step.Compile.run`. +/// Creates a step that executes a process on the host system. +/// +/// `argv` is one or more command line arguments passed to the executed +/// process. The first element is the name of the executable to run. More +/// command line arguments can be added with methods of `Step.Run`, such as: +/// * `Step.Run.addArgs` +/// * `Step.Run.addArtifactArg` +/// * `Step.Run.addFileArg` +/// * `Step.Run.addOutputFileArg` +/// +/// This function introduces a system dependency, compromising reproducibility +/// and making it more difficult to set up one's computer in order to build the +/// project from source. +/// +/// See also: +/// * `addRunArtifact` +/// * `addRunFile` pub fn addSystemCommand(b: *Build, argv: []const []const u8) *Step.Run { assert(argv.len >= 1); const run_step = Step.Run.create(b, b.fmt("run {s}", .{argv[0]})); @@ -818,8 +872,11 @@ pub fn addSystemCommand(b: *Build, argv: []const []const u8) *Step.Run { /// /// This is declarative; it constructs a build step that may or may not be run /// depending on the options provided by the user to the build command. +/// +/// See also: +/// * `addSystemCommand` +/// * `addRunFile` pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run { - // Avoid the common case of the step name looking like "run test test". const step_name = if (exe.kind.isTest() and mem.eql(u8, exe.name, "test")) b.fmt("run {t}", .{exe.kind}) @@ -879,6 +936,19 @@ pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run { return run_step; } +/// Creates a step that executes the provided file. +/// +/// Add more command line arguments via methods of `Step.Run`. +/// +/// See also: +/// * `addSystemCommand` +/// * `addRunArtifact` +pub fn addRunFile(b: *Build, executable: LazyPath) *Step.Run { + const run_step = Step.Run.create(b, b.fmt("run {f}", .{executable.fmt(b.graph)})); + run_step.addFileArg(executable); + return run_step; +} + /// Using the `values` provided, produces a C header file, possibly based on a /// template input file (e.g. config.h.in). /// When an input template file is provided, this function will fail the build @@ -1641,7 +1711,17 @@ pub fn fmt(b: *Build, comptime format: []const u8, args: anytype) []u8 { /// /// Returns the `LazyPath` of the found executable. The search only takes place /// if the `LazyPath` will be used by a depending `Step`. -pub fn findProgram(b: *Build, names: []const []const u8) LazyPath { +/// +/// This API is useful in the following cases: +/// * The binary is not named the same across all systems (for example "python" +/// vs "python3"). +/// * The binary may be produced by building from source rather than being +/// globally installed and will therefore be possibly found in one of the +/// search prefix paths. +/// +/// See also: +/// * `findProgram` +pub fn findProgramLazy(b: *Build, names: []const []const u8) LazyPath { const graph = b.graph; const wc = &graph.wip_configuration; const string_list = wc.addStringList(names) catch @panic("OOM"); @@ -1649,6 +1729,26 @@ pub fn findProgram(b: *Build, names: []const []const u8) LazyPath { @panic("TODO"); } +/// Immediately (in the configure phase), searches for an executable on the host +/// that has more than one possible name. +/// +/// Names are searched in order, observing search prefixes first and then PATH +/// environment variable. +/// +/// Calling this function poisons the configuration cache. For more +/// information, see `Graph.CachePoison` documentation. +/// +/// See also: +/// * `findProgramLazy` +pub fn findProgram(b: *Build, names: []const []const u8) ?[]const u8 { + const graph = b.graph; + const wc = &graph.wip_configuration; + const string_list = wc.addStringList(names) catch @panic("OOM"); + _ = string_list; + graph.poisonCache(); + @panic("TODO"); +} + /// Deprecated; use `runFallible`. pub fn runAllowFail( b: *Build, diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index b96e7922c5ac98db8aa1da14ab4a137ec5410293..02a47a543575deb5fdef442c06b2a52a2eaccd16 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -17,6 +17,7 @@ search_prefixes: []String, extra: []u32, default_step: Step.Index, generated_files_len: u32, +poisoned: bool, /// The field order here matches `Configuration` which documents the order in /// the serialized format. @@ -34,6 +35,12 @@ pub const Header = extern struct { /// There is not actually any data stored for this - it just provides a way /// for maker process to preallocate an array for these. generated_files_len: u32, + flags: Flags, + + pub const Flags = packed struct(u32) { + poisoned: bool, + _: u31 = 0, + }; }; pub const Wip = struct { @@ -52,6 +59,7 @@ pub const Wip = struct { search_prefixes: std.ArrayList(String) = .empty, extra: std.ArrayList(u32) = .empty, next_generated_file_index: u32 = 0, + cache_poison: bool = false, const DedupeTable = std.HashMapUnmanaged(ExtraSlice, void, ExtraSlice.Context, std.hash_map.default_max_load_percentage); const TargetsTable = std.HashMapUnmanaged(TargetQuery.Index, void, TargetsTableContext, std.hash_map.default_max_load_percentage); @@ -137,6 +145,7 @@ pub const Wip = struct { pub const Static = struct { default_step: Step.Index, generated_files_len: u32, + poisoned: bool, }; pub fn write(wip: *Wip, w: *Io.Writer, static: Static) Io.Writer.Error!void { @@ -152,6 +161,9 @@ pub const Wip = struct { .default_step = static.default_step, .generated_files_len = static.generated_files_len, + .flags = .{ + .poisoned = static.poisoned, + }, }; var buffers = [_][]const u8{ @ptrCast(&header), @@ -3325,6 +3337,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { .extra = try arena.alloc(u32, header.extra_len), .default_step = header.default_step, .generated_files_len = header.generated_files_len, + .poisoned = header.flags.poisoned, }; var vecs = [_][]u8{ result.string_bytes, diff --git a/src/main.zig b/src/main.zig index 36340131acb980e1051cea464f1181dadd796d9e..9073b863f9f6185bdcbf95c1b5624a3b19254ce4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4970,6 +4970,7 @@ fn cmdBuild( var system_pkg_dir_path: ?[]const u8 = null; var debug_target: ?[]const u8 = null; var debug_libc_paths_file: ?[]const u8 = null; + var cache_poison: std.Build.Graph.CachePoison = .pure; const self_exe_path = try process.executablePathAlloc(io, arena); const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}); @@ -5039,6 +5040,21 @@ fn cmdBuild( try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); configure_argv.appendAssumeCapacity(arg); continue; + } else if (mem.eql(u8, arg, "--cache-poison")) { + cache_poison = .poisoned; + configure_argv.appendAssumeCapacity("--cache-poison=poisoned"); + continue; + } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { + // Allow the configurer process to report parse failure. + if (std.meta.stringToEnum(std.Build.Graph.CachePoison, rest)) |poison| { + cache_poison = poison; + } + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--verbose")) { + // Intentionally is added both to make and configure but + // does not go into the cache hash. + configure_argv.appendAssumeCapacity(arg); } else if (mem.eql(u8, arg, "--build-file")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; @@ -5069,10 +5085,6 @@ fn cmdBuild( i += 1; override_global_cache_dir = args[i]; continue; - } else if (mem.eql(u8, arg, "--verbose")) { - // Intentionally is added both to make and configure but - // does not go into the cache hash. - configure_argv.appendAssumeCapacity(arg); } else if (mem.eql(u8, arg, "-freference-trace")) { reference_trace = 256; } else if (mem.eql(u8, arg, "--fetch")) { @@ -5229,6 +5241,10 @@ fn cmdBuild( for (cached_passthru_configure.items) |i| config_man.hash.addBytes(configure_argv.items[i]); + // Prevents a `zig build` from getting a false positive cache hit following + // a `zig build --cache-poison=ignored`. + config_man.hash.add(cache_poison == .ignored); + // Normally the build runner is compiled for the host target but here is // some code to help when debugging edits to the build runner so that you // can make sure it compiles successfully on other targets. @@ -5338,7 +5354,7 @@ fn cmdBuild( // This loop is re-evaluated when the build script exits with an indication that it // could not continue due to missing lazy dependencies. - const configuration_path: Path = cp: while (true) { + const configuration_path: Path, const poisoned: bool = cp: while (true) { // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. { @@ -5609,12 +5625,18 @@ fn cmdBuild( _ = try config_man.addFilePath(exe_path, null); configure_argv.items[0] = try exe_path.toString(arena); - if (try config_man.hit()) { - const digest = config_man.final(); - break :cp .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), - }; + switch (cache_poison) { + .pure, .disallowed, .ignored => if (try config_man.hit()) { + const digest = config_man.final(); + break :cp .{ + .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + }, + false, + }; + }, + .poisoned => {}, // Don't bother checking for cache hit. } } @@ -5636,7 +5658,7 @@ fn cmdBuild( ); defer config_tmp_file.close(io); - switch (term: { + const term = term: { const child_node = root_prog_node.start("Run Configure Script", 0); defer child_node.end(); var child = std.process.spawn(io, .{ @@ -5647,101 +5669,86 @@ fn cmdBuild( defer child.kill(io); break :term child.wait(io) catch |err| fatal("failed to wait configure script {s}: {t}", .{ configure_argv.items[0], err }); - }) { - .exited => |code| { - if (code != 0) { - // Failure to produce the configuration file. - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following configure command failed with exit code {d}:\n{s}", .{ code, cmd }); - } - // Even though the file is designed to be sent directly to make - // runner, we must load it now because: - // * If it contains additional file dependencies, we need to - // add them to `config_man` before obtaining the final digest. - // * If it contains a set of lazy packages that need to be - // fetched, we need to fetch those now and re-run configure. - var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| - fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); + }; + if (!term.success()) { + // Failure to produce the configuration file. + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following configure command {f}:\n{s}", .{ term, cmd }); + } + // Even though the file is designed to be sent directly to make + // runner, we must load it now because: + // * If it contains additional file dependencies, we need to + // add them to `config_man` before obtaining the final digest. + // * If it contains a set of lazy packages that need to be + // fetched, we need to fetch those now and re-run configure. + var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); - if (configuration.unlazy_deps.len != 0) { - if (!dev.env.supports(.fetch_command)) process.exit(1); - var any_errors = false; - for (configuration.unlazy_deps) |hash_string| { - const hash = hash_string.slice(&configuration); - assert(hash.len != 0); - if (hash.len > Package.Hash.max_len) { - std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ - hash.len, hash, - }); - any_errors = true; - continue; - } - try unlazy_set.put(arena, .fromSlice(hash), {}); - } - if (any_errors) process.exit(1); - if (system_pkg_dir_path) |p| { - // In this mode, the system needs to provide these packages; they - // cannot be fetched by Zig. - const s = fs.path.sep_str; - for (unlazy_set.keys()) |*hash| { - std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ - p, hash.toSlice(), - }); - } - std.log.info("remote package fetching disabled due to --system mode", .{}); - std.log.info("dependencies might be avoidable depending on build configuration", .{}); - process.exit(1); - } - continue :cp; + if (configuration.unlazy_deps.len != 0) { + if (!dev.env.supports(.fetch_command)) process.exit(1); + var any_errors = false; + for (configuration.unlazy_deps) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); + if (hash.len > Package.Hash.max_len) { + std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ hash.len, hash }); + any_errors = true; + continue; } - - for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { - const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; - try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); + try unlazy_set.put(arena, .fromSlice(hash), {}); + } + if (any_errors) process.exit(1); + if (system_pkg_dir_path) |p| { + // In this mode, the system needs to provide these packages; they + // cannot be fetched by Zig. + const s = fs.path.sep_str; + for (unlazy_set.keys()) |*hash| { + std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); } + std.log.info("remote package fetching disabled due to --system mode", .{}); + std.log.info("dependencies might be avoidable depending on build configuration", .{}); + process.exit(1); + } + continue :cp; + } - const digest = config_man.final(); - const final_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), - }; - Io.Dir.rename( - config_tmp_path.root_dir.handle, - config_tmp_path.sub_path, - final_path.root_dir.handle, - final_path.sub_path, - io, - ) catch |err| { - fatal("failed to rename configuration file from {f} into {f}: {t}", .{ - config_tmp_path, final_path, err, - }); - }; - config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { + const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; + try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); + } - break :cp final_path; - }, - .signal => |sig| { - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following configure command terminated with signal {t}:\n{s}", .{ sig, cmd }); - }, - .stopped => |sig| { - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following build command stopped with signal {t}:\n{s}", .{ sig, cmd }); - }, - .unknown => { - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following build command crashed:\n{s}", .{cmd}); - }, + // If it is poisoned, there is no point in moving it to cached + // location. Just leave it in the tmp directory. + if (configuration.poisoned) { + break :cp .{ config_tmp_path, true }; + } else { + const digest = config_man.final(); + const final_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + }; + Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + ) catch |err| { + fatal("failed to rename configuration file from {f} into {f}: {t}", .{ + config_tmp_path, final_path, err, + }); + }; + config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + break :cp .{ final_path, false }; } }; { // Release all file system locks just before running the maker process. - var configuration_lock = config_man.toOwnedLock(); - defer configuration_lock.release(io); + var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; + defer if (configuration_lock) |*l| l.release(io); - const make_runner = make_runner_task.await(io) catch |err| - fatal("failed to compile maker: {t}", .{err}); + const make_runner = make_runner_task.await(io) catch |err| fatal("failed compiling maker: {t}", .{err}); make_argv.items[0] = try make_runner.exe_path.toString(arena); make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); @@ -5749,33 +5756,24 @@ fn cmdBuild( if (!process.can_spawn) { const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); + fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ + native_os, cmd, + }); } - switch (term: { + const term = term: { _ = try io.lockStderr(&.{}, .no_color); defer io.unlockStderr(); var child = std.process.spawn(io, .{ .argv = make_argv.items, - }) catch |err| fatal("failed to spawn maker {s}: {t}", .{ make_argv.items[0], err }); + }) catch |err| fatal("failed spawning maker {s}: {t}", .{ make_argv.items[0], err }); defer child.kill(io); break :term child.wait(io) catch |err| - fatal("failed to wait maker {s}: {t}", .{ make_argv.items[0], err }); - }) { - .exited => |code| { - if (code == 0) return cleanExit(io); - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following maker command failed with exit code {d}:\n{s}", .{ code, cmd }); - }, - .signal => |sig| { - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following maker command terminated with signal {t}:\n{s}", .{ sig, cmd }); - }, - else => { - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following maker command crashed:\n{s}", .{cmd}); - }, - } + fatal("failed waiting on maker {s}: {t}", .{ make_argv.items[0], err }); + }; + if (term.success()) return cleanExit(io); + const cmd = try std.mem.join(arena, " ", make_argv.items); + fatal("the following maker command {f}:\n{s}", .{ term, cmd }); } const MakeRunner = struct { -- 2.54.0 From 1edc5d7d67f084941c2162dc71bd8a417189f265 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2026 17:51:19 -0700 Subject: [PATCH 133/179] Maker: implement FindProgram (lazy) --- lib/compiler/Maker/Step.zig | 3 +- lib/compiler/Maker/Step/FindProgram.zig | 120 ++++++++++++++++++++++++ lib/compiler/configurer.zig | 8 +- lib/std/Build.zig | 13 +-- lib/std/Build/Configuration.zig | 4 +- lib/std/Build/Step.zig | 5 +- lib/std/Build/Step/FindProgram.zig | 31 ++++++ lib/std/Io/Dir.zig | 3 + 8 files changed, 175 insertions(+), 12 deletions(-) create mode 100644 lib/compiler/Maker/Step/FindProgram.zig create mode 100644 lib/std/Build/Step/FindProgram.zig diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index a4760274cdeb5ab6b64651b2e79be61e1d8ebae6..5c618660ce787ce8913b9dfb597f0bf882856b7e 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -19,6 +19,7 @@ const WebServer = @import("WebServer.zig"); const Maker = @import("../Maker.zig"); pub const Compile = @import("Step/Compile.zig"); +pub const FindProgram = @import("Step/FindProgram.zig"); pub const Fmt = @import("Step/Fmt.zig"); pub const InstallArtifact = @import("Step/InstallArtifact.zig"); pub const InstallDir = @import("Step/InstallDir.zig"); @@ -78,7 +79,7 @@ pub const Extended = union(enum) { compile: Compile, config_header: Todo, fail: Fail, - find_program: Todo, + find_program: FindProgram, fmt: Fmt, install_artifact: InstallArtifact, install_dir: InstallDir, diff --git a/lib/compiler/Maker/Step/FindProgram.zig b/lib/compiler/Maker/Step/FindProgram.zig new file mode 100644 index 0000000000000000000000000000000000000000..c21374c90522d07257c1b4ce3439f1cbf87c7b61 --- /dev/null +++ b/lib/compiler/Maker/Step/FindProgram.zig @@ -0,0 +1,120 @@ +const FindProgram = @This(); +const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; +const Configuration = std.Build.Configuration; +const assert = std.debug.assert; + +const Step = @import("../Step.zig"); +const Maker = @import("../../Maker.zig"); + +pub fn make( + find_program: *FindProgram, + step_index: Configuration.Step.Index, + maker: *Maker, + progress_node: std.Progress.Node, +) Step.ExtendedMakeError!void { + _ = find_program; + _ = progress_node; + const graph = maker.graph; + const step = maker.stepByIndex(step_index); + const arena = graph.arena; // TODO don't leak into the process arena + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_fp = conf_step.extended.get(conf.extra).find_program; + const found_path = conf_fp.found_path; + const names = conf_fp.names.slice(conf); + + // In case we fail at the end. + var err_msg: std.ArrayList(u8) = .empty; + try err_msg.appendSlice(arena, "program not found. searched paths:\n"); + + for (names) |name_index| { + const name = name_index.slice(conf); + + if (Io.Dir.path.isAbsolute(name)) { + if (try checkCandidate(maker, step, found_path, &err_msg, name)) return; + + continue; + } + + for (graph.search_prefixes.items) |search_prefix| { + const full_path = try Io.Dir.path.join(arena, &.{ search_prefix, "bin", name }); + + if (try checkCandidate(maker, step, found_path, &err_msg, full_path)) return; + } + } + + if (graph.environ_map.get("PATH")) |PATH| { + for (names) |name_index| { + const name = name_index.slice(conf); + + var it = std.mem.tokenizeScalar(u8, PATH, Io.Dir.path.delimiter); + while (it.next()) |p| { + const full_path = try Io.Dir.path.join(arena, &.{ p, name }); + + if (try checkCandidate(maker, step, found_path, &err_msg, full_path)) return; + } + } + } + + assert(err_msg.items[err_msg.items.len - 1] == '\n'); + const chopped = err_msg.items[0 .. err_msg.items.len - 1]; + try step.result_error_msgs.append(arena, chopped); + return error.MakeFailed; +} + +fn checkCandidate( + maker: *Maker, + step: *Step, + found_path: Configuration.GeneratedFileIndex, + err_msg: *std.ArrayList(u8), + full_path: []const u8, +) !bool { + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into process arena + const io = graph.io; + + if (Io.Dir.cwd().access(io, full_path, .{ .execute = true })) |_| { + maker.generatedPath(found_path).* = .initCwd(full_path); + return true; + } else |err| switch (err) { + error.Canceled => |e| return e, + error.FileNotFound, error.AccessDenied, error.PermissionDenied => |e| { + try err_msg.print(arena, "{t} {s}\n", .{ e, full_path }); + }, + else => |e| return step.fail(maker, "failed accessing {s}: {t}", .{ full_path, e }), + } + + if (builtin.os.tag == .windows) { + if (graph.environ_map.get("PATHEXT")) |PATHEXT| { + var it = std.mem.tokenizeScalar(u8, PATHEXT, Io.Dir.path.delimiter); + while (it.next()) |ext| { + if (!supportedWindowsProgramExtension(ext)) continue; + + const extended_path = try std.mem.concat(arena, &.{ full_path, ext }); + + if (Io.Dir.cwd().access(io, extended_path, .{ .execute = true })) |_| { + maker.generatedPath(found_path).* = .initCwd(extended_path); + return true; + } else |err| switch (err) { + error.Canceled => |e| return e, + error.FileNotFound, error.AccessDenied, error.PermissionDenied => |e| { + try err_msg.print(arena, "{s} {t}\n", .{ extended_path, e }); + }, + else => |e| return step.fail(maker, "failed accessing {s}: {t}", .{ extended_path, e }), + } + } + } + } + + return false; +} + +fn supportedWindowsProgramExtension(ext: []const u8) bool { + inline for (@typeInfo(std.process.WindowsExtension).@"enum".fields) |field| { + if (std.ascii.eqlIgnoreCase(ext, "." ++ field.name)) return true; + } + return false; +} diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index c8aa77edd8d79c61533ec00e6ac0cd5248cbc86f..2e5b26c4b2fc87ec43473238d61fe893f71765e1 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -906,7 +906,13 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .msg = sf.error_msg, }); }, - .find_program => @panic("TODO"), + .find_program => e: { + const fp: *Step.FindProgram = @fieldParentPtr("step", step); + break :e try wc.addExtraErased(Configuration.Step.FindProgram, .{ + .names = fp.names, + .found_path = fp.found_path, + }); + }, .fmt => e: { const sf: *Step.Fmt = @fieldParentPtr("step", step); break :e try wc.addExtraErased(Configuration.Step.Fmt, .{ diff --git a/lib/std/Build.zig b/lib/std/Build.zig index fbf2fb0e5047b7c598b86635fdf9e139ce96fc35..abaedb24daecee2b49a9c39db2f17d06860ee3b1 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1719,14 +1719,15 @@ pub fn fmt(b: *Build, comptime format: []const u8, args: anytype) []u8 { /// globally installed and will therefore be possibly found in one of the /// search prefix paths. /// +/// Windows file name extensions are searched automatically, respecting the +/// PATHEXT environment variable, so they need not be included in this list. +/// However, even on Windows, the names will be checked without appending +/// extensions first, so that can be used as a priority system. +/// /// See also: /// * `findProgram` -pub fn findProgramLazy(b: *Build, names: []const []const u8) LazyPath { - const graph = b.graph; - const wc = &graph.wip_configuration; - const string_list = wc.addStringList(names) catch @panic("OOM"); - _ = string_list; - @panic("TODO"); +pub fn findProgramLazy(b: *Build, options: Step.FindProgram.Options) LazyPath { + return .{ .generated = .{ .index = Step.FindProgram.create(b, options).found_path } }; } /// Immediately (in the configure phase), searches for an executable on the host diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 02a47a543575deb5fdef442c06b2a52a2eaccd16..63bc3027386868ecf5def1d62f32448f19402367 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1192,9 +1192,9 @@ pub const Step = extern struct { }; pub const FindProgram = struct { - flags: @This().Flags, + flags: @This().Flags = .{}, names: StringList, - generated_file: GeneratedFileIndex, + found_path: GeneratedFileIndex, pub const Flags = packed struct(u32) { tag: Tag = .find_program, diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index e115732d0346774d3ea059129a404d29d9375172..4332737c16ed6e7250cde394b15b285c938ac542 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -78,19 +78,20 @@ pub fn Type(comptime tag: Tag) type { } pub const CheckFile = @import("Step/CheckFile.zig"); +pub const Compile = @import("Step/Compile.zig"); pub const ConfigHeader = @import("Step/ConfigHeader.zig"); pub const Fail = @import("Step/Fail.zig"); +pub const FindProgram = @import("Step/FindProgram.zig"); pub const Fmt = @import("Step/Fmt.zig"); pub const InstallArtifact = @import("Step/InstallArtifact.zig"); pub const InstallDir = @import("Step/InstallDir.zig"); pub const InstallFile = @import("Step/InstallFile.zig"); pub const ObjCopy = @import("Step/ObjCopy.zig"); -pub const Compile = @import("Step/Compile.zig"); pub const Options = @import("Step/Options.zig"); pub const Run = @import("Step/Run.zig"); pub const TranslateC = @import("Step/TranslateC.zig"); -pub const WriteFile = @import("Step/WriteFile.zig"); pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); +pub const WriteFile = @import("Step/WriteFile.zig"); pub const TopLevel = struct { pub const base_tag: Step.Tag = .top_level; diff --git a/lib/std/Build/Step/FindProgram.zig b/lib/std/Build/Step/FindProgram.zig new file mode 100644 index 0000000000000000000000000000000000000000..079f1037dccb68efdb38897166c831954a97834a --- /dev/null +++ b/lib/std/Build/Step/FindProgram.zig @@ -0,0 +1,31 @@ +const FindProgram = @This(); + +const std = @import("std"); +const Step = std.Build.Step; +const Configuration = std.Build.Configuration; + +step: Step, +found_path: Configuration.GeneratedFileIndex, +names: Configuration.StringList, + +pub const base_tag: Step.Tag = .find_program; + +pub const Options = struct { + names: []const []const u8, +}; + +pub fn create(owner: *std.Build, options: Options) *FindProgram { + const graph = owner.graph; + const wc = &graph.wip_configuration; + const fp = graph.create(FindProgram); + fp.* = .{ + .step = .init(.{ + .tag = base_tag, + .name = owner.fmt("find program {s} ({d} candidates)", .{ options.names[0], options.names.len }), + .owner = owner, + }), + .found_path = graph.addGeneratedFile(&fp.step), + .names = wc.addStringList(options.names) catch @panic("OOM"), + }; + return fp; +} diff --git a/lib/std/Io/Dir.zig b/lib/std/Io/Dir.zig index de003734cf58d31e333ed76fb46b782e9835928a..eda18e2579525f5299fda9a357bb89edb67adbd5 100644 --- a/lib/std/Io/Dir.zig +++ b/lib/std/Io/Dir.zig @@ -409,7 +409,10 @@ pub const PathNameError = error{ }; pub const AccessError = error{ + /// The requested `AccessOptions` would be denied to the file, or search + /// permission is denied for one of the directories in the path prefix. AccessDenied, + /// Write permission was requested but the file is immutable. PermissionDenied, FileNotFound, InputOutput, -- 2.54.0 From 92038675af545ecdbe1f1b4cbd1095a8b3264e07 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2026 18:36:51 -0700 Subject: [PATCH 134/179] zig build: implement findProgram (not lazy) --- lib/compiler/Maker/Step/FindProgram.zig | 2 +- lib/compiler/configurer.zig | 8 +++ lib/std/Build.zig | 96 ++++++++++++++++++++++--- src/main.zig | 11 ++- 4 files changed, 105 insertions(+), 12 deletions(-) diff --git a/lib/compiler/Maker/Step/FindProgram.zig b/lib/compiler/Maker/Step/FindProgram.zig index c21374c90522d07257c1b4ce3439f1cbf87c7b61..d3b8d067388bd1a6ff875b27aa314aa0d764980a 100644 --- a/lib/compiler/Maker/Step/FindProgram.zig +++ b/lib/compiler/Maker/Step/FindProgram.zig @@ -101,7 +101,7 @@ fn checkCandidate( } else |err| switch (err) { error.Canceled => |e| return e, error.FileNotFound, error.AccessDenied, error.PermissionDenied => |e| { - try err_msg.print(arena, "{s} {t}\n", .{ extended_path, e }); + try err_msg.print(arena, "{t} {s}\n", .{ e, extended_path }); }, else => |e| return step.fail(maker, "failed accessing {s}: {t}", .{ extended_path, e }), } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 2e5b26c4b2fc87ec43473238d61fe893f71765e1..b47fd2da92c1a5ae9850ee2649b1c0668e5700bf 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -117,6 +117,8 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { graph.cache_poison = std.meta.stringToEnum(std.Build.Graph.CachePoison, rest) orelse fatalWithHint("expected --cache-poison=[pure|poisoned|disallowed|ignored]; found: {s}", .{arg}); + } else if (mem.eql(u8, arg, "--search-prefix")) { + try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_i)); } else { fatalWithHint("unrecognized argument: {s}", .{arg}); } @@ -1319,6 +1321,12 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { return args[idx.*]; } +fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { + return nextArg(args, idx) orelse { + fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); + }; +} + fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index abaedb24daecee2b49a9c39db2f17d06860ee3b1..27c151fcacde135da78be692575219082df5ad17 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -99,6 +99,8 @@ pub const Graph = struct { wip_configuration: Configuration.Wip, cache_poison: CachePoison = .pure, + /// Observing this data causes cache poisoning. See `CachePoison`. + search_prefixes: std.ArrayList([]const u8) = .empty, /// If the cache is poisoned means that the **configure logic** had side /// effects, or otherwise did something that could not be tracked by the @@ -1706,9 +1708,6 @@ pub fn fmt(b: *Build, comptime format: []const u8, args: anytype) []u8 { /// Creates an anonymous `Step` that searches for an executable on the host that /// has more than one possible name. /// -/// Names are searched in order, observing search prefixes first and then PATH -/// environment variable. -/// /// Returns the `LazyPath` of the found executable. The search only takes place /// if the `LazyPath` will be used by a depending `Step`. /// @@ -1719,6 +1718,9 @@ pub fn fmt(b: *Build, comptime format: []const u8, args: anytype) []u8 { /// globally installed and will therefore be possibly found in one of the /// search prefix paths. /// +/// Names are searched in order, observing search prefixes first and then PATH +/// environment variable. +/// /// Windows file name extensions are searched automatically, respecting the /// PATHEXT environment variable, so they need not be included in this list. /// However, even on Windows, the names will be checked without appending @@ -1730,24 +1732,98 @@ pub fn findProgramLazy(b: *Build, options: Step.FindProgram.Options) LazyPath { return .{ .generated = .{ .index = Step.FindProgram.create(b, options).found_path } }; } +pub const FindProgramOptions = Step.FindProgram.Options; + /// Immediately (in the configure phase), searches for an executable on the host /// that has more than one possible name. /// +/// Calling this function poisons the configuration cache, so it is only +/// appropriate when the existence of the program or its output needs to be +/// observed by configuration logic. For more information, see +/// `Graph.CachePoison` documentation. +/// /// Names are searched in order, observing search prefixes first and then PATH /// environment variable. /// -/// Calling this function poisons the configuration cache. For more -/// information, see `Graph.CachePoison` documentation. +/// Windows file name extensions are searched automatically, respecting the +/// PATHEXT environment variable, so they need not be included in this list. +/// However, even on Windows, the names will be checked without appending +/// extensions first, so that can be used as a priority system. /// /// See also: /// * `findProgramLazy` -pub fn findProgram(b: *Build, names: []const []const u8) ?[]const u8 { +pub fn findProgram(b: *Build, options: FindProgramOptions) ?[]const u8 { const graph = b.graph; - const wc = &graph.wip_configuration; - const string_list = wc.addStringList(names) catch @panic("OOM"); - _ = string_list; + + // Because it observes search prefixes and contents of directories in PATH. graph.poisonCache(); - @panic("TODO"); + + for (options.names) |name| { + if (Io.Dir.path.isAbsolute(name)) { + if (tryFindProgram(b, name)) |found| return found; + } + for (graph.search_prefixes.items) |search_prefix| { + const full_path = b.pathJoin(&.{ search_prefix, "bin", name }); + if (tryFindProgram(b, full_path)) |found| return found; + } + } + + if (b.graph.environ_map.get("PATH")) |PATH| { + for (options.names) |name| { + var it = mem.tokenizeScalar(u8, PATH, Io.Dir.path.delimiter); + while (it.next()) |p| { + const full_path = b.pathJoin(&.{ p, name }); + if (tryFindProgram(b, full_path)) |found| return found; + } + } + } + + return null; +} + +fn supportedWindowsProgramExtension(ext: []const u8) bool { + inline for (@typeInfo(std.process.WindowsExtension).@"enum".fields) |field| { + if (std.ascii.eqlIgnoreCase(ext, "." ++ field.name)) return true; + } + return false; +} + +fn tryFindProgram(b: *Build, full_path: []const u8) ?[]const u8 { + const graph = b.graph; + const io = graph.io; + const arena = graph.arena; + + if (Io.Dir.cwd().access(io, full_path, .{ .execute = true })) |_| { + return full_path; + } else |err| switch (err) { + error.FileNotFound, error.AccessDenied, error.PermissionDenied => |e| { + if (graph.verbose) log.info("searched: {t} {s}", .{ e, full_path }); + }, + else => |e| return panic("failed accessing {s}: {t}", .{ full_path, e }), + } + + if (builtin.os.tag == .windows) { + if (b.graph.environ_map.get("PATHEXT")) |PATHEXT| { + var it = mem.tokenizeScalar(u8, PATHEXT, fs.path.delimiter); + + while (it.next()) |ext| { + if (!supportedWindowsProgramExtension(ext)) continue; + + const extended_path = try mem.concat(arena, &.{ full_path, ext }); + + if (Io.Dir.cwd().access(io, extended_path, .{ .execute = true })) |_| { + return extended_path; + } else |err| switch (err) { + error.FileNotFound, error.AccessDenied, error.PermissionDenied => |e| { + if (graph.verbose) log.info("searched: {t} {s}", .{ e, extended_path }); + }, + else => |e| return panic("failed accessing {s}: {t}", .{ extended_path, e }), + } + } + } + } + + return null; } /// Deprecated; use `runFallible`. diff --git a/src/main.zig b/src/main.zig index 9073b863f9f6185bdcbf95c1b5624a3b19254ce4..5db4c1c95ff50d2f3faf1244801975c41f3dd3b4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5014,7 +5014,7 @@ fn cmdBuild( while (i < args.len) : (i += 1) { const arg = args[i]; if (mem.startsWith(u8, arg, "-")) { - try configure_argv.ensureUnusedCapacity(arena, 1); + try configure_argv.ensureUnusedCapacity(arena, 2); if (mem.startsWith(u8, arg, "-D") or mem.startsWith(u8, arg, "-fsys=") or @@ -5055,6 +5055,15 @@ fn cmdBuild( // Intentionally is added both to make and configure but // does not go into the cache hash. configure_argv.appendAssumeCapacity(arg); + } else if (mem.eql(u8, arg, "--search-prefix")) { + if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); + i += 1; + // This argument is cache poisonous: it does not go into + // the cache and configurer must set the poison bit when + // choosing to observe it. + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, args[i] }; + (try make_argv.addManyAsArray(arena, 2)).* = .{ arg, args[i] }; + continue; } else if (mem.eql(u8, arg, "--build-file")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; -- 2.54.0 From 31c159c228ecf6c2ffcf44ee44621c3d6b1ceb61 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2026 18:56:09 -0700 Subject: [PATCH 135/179] Maker: implement CheckFile --- lib/compiler/Maker/Step.zig | 3 ++- lib/compiler/Maker/Step/CheckFile.zig | 17 ++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 5c618660ce787ce8913b9dfb597f0bf882856b7e..5854bbbf25305e46eb3483e89948caa843f9f408 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -18,6 +18,7 @@ const assert = std.debug.assert; const WebServer = @import("WebServer.zig"); const Maker = @import("../Maker.zig"); +pub const CheckFile = @import("Step/CheckFile.zig"); pub const Compile = @import("Step/Compile.zig"); pub const FindProgram = @import("Step/FindProgram.zig"); pub const Fmt = @import("Step/Fmt.zig"); @@ -75,7 +76,7 @@ comptime { } pub const Extended = union(enum) { - check_file: Todo, + check_file: CheckFile, compile: Compile, config_header: Todo, fail: Fail, diff --git a/lib/compiler/Maker/Step/CheckFile.zig b/lib/compiler/Maker/Step/CheckFile.zig index 0ed23c05e01952091d0c02ff26a642446e9ae1a0..ab563a6092ee759f1e636eab9cee1b3b6759d203 100644 --- a/lib/compiler/Maker/Step/CheckFile.zig +++ b/lib/compiler/Maker/Step/CheckFile.zig @@ -13,6 +13,7 @@ pub fn make( maker: *Maker, progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { + _ = check_file; _ = progress_node; const graph = maker.graph; const arena = maker.graph.arena; // TODO don't leak into process arena @@ -20,7 +21,7 @@ pub fn make( const step = maker.stepByIndex(step_index); const conf = &maker.scanned_config.configuration; const conf_step = step_index.ptr(conf); - const conf_cf = conf_step.extended.get(conf.extra).install_file; + const conf_cf = conf_step.extended.get(conf.extra).check_file; const lazy_path = conf_cf.file.get(conf); try step.singleUnchangingWatchInput(maker, arena, lazy_path); @@ -29,11 +30,12 @@ pub fn make( const limit: Io.Limit = if (conf_cf.max_bytes.value) |x| .limited(x) else .unlimited; const contents = src_path.root_dir.handle.readFileAlloc(io, src_path.sub_path, arena, limit) catch |err| - return step.fail("failed to read {f}: {t}", .{ src_path, err }); + return step.fail(maker, "failed to read {f}: {t}", .{ src_path, err }); - for (check_file.expected_matches) |expected_match| { + for (conf_cf.expected_matches.slice) |expected_match_index| { + const expected_match = expected_match_index.slice(conf); if (std.mem.find(u8, contents, expected_match) == null) { - return step.fail( + return step.fail(maker, \\ \\========= expected to find: =================== \\{s} @@ -44,16 +46,17 @@ pub fn make( } } - if (check_file.expected_exact) |expected_exact| { + if (conf_cf.expected_exact.value) |expected_exact_index| { + const expected_exact = expected_exact_index.slice(conf); if (!std.mem.eql(u8, expected_exact, contents)) { - return step.fail( + return step.fail(maker, \\ \\========= expected: ===================== \\{s} \\========= but found: ==================== \\{s} \\========= from the following file: ====== - \\{s} + \\{f} , .{ expected_exact, contents, src_path }); } } -- 2.54.0 From ed1f00582670df44c15875914a1cc8fc0ff6329f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2026 19:13:04 -0700 Subject: [PATCH 136/179] Maker: finish implementing Step.Compile.appendIncludeDirFlags --- lib/compiler/Maker/Step/Compile.zig | 13 +++++-------- lib/compiler/configurer.zig | 2 +- lib/std/Build/Configuration.zig | 2 -- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index c237fd61e7c5fd80e9b927749930008b099f8194..859a24d4c4d7e44e9a7f212047e58af6a524db84 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -1354,6 +1354,7 @@ pub fn appendIncludeDirFlags( ) !void { const graph = maker.graph; const arena = graph.arena; // TODO don't leak into the process arena + const conf = &maker.scanned_config.configuration; switch (include_dir) { .path => |lp| { @@ -1376,15 +1377,11 @@ pub fn appendIncludeDirFlags( zig_args.appendAssumeCapacity("-iframework"); zig_args.appendAssumeCapacity(try maker.resolveLazyPathIndexAbs(arena, lp, asking_step)); }, - .config_header_step => |ch| { + .config_header_step => |ch_index| { + const conf_ch = ch_index.ptr(conf).extended.get(conf.extra).config_header; + const path = maker.generatedPath(conf_ch.generated_dir).*; zig_args.appendAssumeCapacity("-I"); - if (true) @panic("TODO appendIncludeDirFlags"); - ch.getOutputDir(); - }, - .other_step => |comp| { - zig_args.appendAssumeCapacity("-I"); - if (true) @panic("TODO appendIncludeDirFlags"); - comp.installed_headers_include_tree.?.getDirectory(); + zig_args.appendAssumeCapacity(try path.toString(arena)); }, .embed_path => |lazy_path| { zig_args.appendAssumeCapacity(try allocPrint(arena, "--embed-dir={f}", .{ diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index b47fd2da92c1a5ae9850ee2649b1c0668e5700bf..b81ee9de0d351115befc777ec102403a41cb49b7 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -469,7 +469,7 @@ const Serialize = struct { .framework_path => |lp| .{ .framework_path = try addLazyPath(s, lp) }, .framework_path_system => |lp| .{ .framework_path_system = try addLazyPath(s, lp) }, .embed_path => |lp| .{ .embed_path = try addLazyPath(s, lp) }, - .other_step => |cs| .{ .other_step = stepIndex(s, &cs.step) }, + .other_step => |cs| .{ .path = try addLazyPath(s, cs.installed_headers_include_tree.?.getDirectory()) }, .config_header_step => |chs| .{ .config_header_step = stepIndex(s, &chs.step) }, }; return result; diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 63bc3027386868ecf5def1d62f32448f19402367..d19a357b50571ceb93b455e38a590241957414a7 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1680,8 +1680,6 @@ pub const Module = struct { path_after: LazyPath.Index, framework_path: LazyPath.Index, framework_path_system: LazyPath.Index, - /// Always `Step.Tag.compile`. - other_step: Step.Index, /// Always `Step.Tag.config_header`. config_header_step: Step.Index, embed_path: LazyPath.Index, -- 2.54.0 From e435299cfa6a6bd446483e6c95a56bf1cdca8cfd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2026 21:51:11 -0700 Subject: [PATCH 137/179] Maker: progress towards ConfigHeader however... why is this done in the make phase anyway? making a header like this is typically done by the configure phase... --- lib/compiler/Maker/Step.zig | 23 +-- lib/compiler/Maker/Step/ConfigHeader.zig | 234 +++++++++++++---------- lib/compiler/Maker/Step/ObjCopy.zig | 2 +- 3 files changed, 139 insertions(+), 120 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 5854bbbf25305e46eb3483e89948caa843f9f408..66e6079ab3950edd5aa9d4f0d358aeed6a2d18e2 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -20,6 +20,7 @@ const Maker = @import("../Maker.zig"); pub const CheckFile = @import("Step/CheckFile.zig"); pub const Compile = @import("Step/Compile.zig"); +pub const ConfigHeader = @import("Step/ConfigHeader.zig"); pub const FindProgram = @import("Step/FindProgram.zig"); pub const Fmt = @import("Step/Fmt.zig"); pub const InstallArtifact = @import("Step/InstallArtifact.zig"); @@ -78,7 +79,7 @@ comptime { pub const Extended = union(enum) { check_file: CheckFile, compile: Compile, - config_header: Todo, + config_header: ConfigHeader, fail: Fail, find_program: FindProgram, fmt: Fmt, @@ -114,21 +115,6 @@ pub const Extended = union(enum) { }; } - pub const Todo = struct { - pub fn make( - todo: *Todo, - step_index: Configuration.Step.Index, - maker: *Maker, - progress_node: std.Progress.Node, - ) Step.ExtendedMakeError!void { - _ = todo; - _ = progress_node; - const conf = &maker.scanned_config.configuration; - const conf_step = step_index.ptr(conf); - std.debug.panic("TODO implement another step type: {s}", .{conf_step.name.slice(conf)}); - } - }; - pub const TopLevel = struct { pub fn make( top_level: *TopLevel, @@ -750,8 +736,9 @@ fn failWithCacheError( /// separately from using the cache system. pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { if (s.test_results.isSuccess()) { - man.writeManifest() catch |err| { - try s.addError(maker, "failed writing cache manifest: {t}", .{err}); + man.writeManifest() catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| try s.addError(maker, "failed writing cache manifest: {t}", .{e}), }; } } diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index f823d5346807113e7ae69bc3f537240f4958a60a..b86548b561a65686adf73a50402dcd86d9eede42 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -4,22 +4,35 @@ const std = @import("std"); const Io = std.Io; const Configuration = std.Build.Configuration; const Writer = std.Io.Writer; +const Path = std.Build.Cache.Path; +const Allocator = std.mem.Allocator; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); + const header_text = "This file was generated by ConfigHeader using the Zig Build System."; + const c_generated_line = "/* " ++ header_text ++ " */\n"; + const asm_generated_line = "; " ++ header_text ++ "\n"; + pub fn make( config_header: *ConfigHeader, step_index: Configuration.Step.Index, maker: *Maker, progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { + _ = config_header; + _ = progress_node; const graph = maker.graph; - const arena = maker.graph.arena; // TODO don't leak into process arena + const gpa = maker.gpa; const step = maker.stepByIndex(step_index); const io = graph.io; + const arena = graph.arena; // TODO don't leak into the process arena + const conf = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(conf); + const conf_ch = conf_step.extended.get(conf.extra).config_header; + const cache_root = graph.local_cache_root; - if (config_header.style.getPath()) |lp| + if (conf_ch.style.getPath()) |lp| try step.singleUnchangingWatchInput(maker, arena, lp); var man = graph.cache.obtain(); @@ -29,45 +42,51 @@ pub fn make( // random bytes when ConfigHeader implementation is modified in a // non-backwards-compatible way. man.hash.add(@as(u32, 0xdef08d23)); - man.hash.addBytes(config_header.include_path); - man.hash.addOptionalBytes(config_header.include_guard_override); + man.hash.addBytes(conf_ch.include_path); + man.hash.addOptionalBytes(conf_ch.include_guard_override); var aw: Writer.Allocating = .init(arena); defer aw.deinit(); - const bw = &aw.writer; - const header_text = "This file was generated by ConfigHeader using the Zig Build System."; - const c_generated_line = "/* " ++ header_text ++ " */\n"; - const asm_generated_line = "; " ++ header_text ++ "\n"; - - switch (config_header.style) { - .autoconf_undef, .autoconf_at => |file_source| { - try bw.writeAll(c_generated_line); - const src_path = file_source.getPath2(b, step); - const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(config_header.max_bytes)) catch |err| { + switch (conf_ch.flags.style) { + .autoconf_undef => { + const src_path = try maker.resolveLazyPathIndex(arena, conf_ch.template_file.value.?, step_index); + const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(conf_ch.max_bytes)) catch |err| + return step.fail("unable to read autoconf input file {s}: {t}", .{ src_path, err }); + renderAutoConfUndef(step, contents, &aw.writer, &conf_ch.values, src_path) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + else => |e| return e, + }; + }, + .autoconf_at => { + const src_path = try maker.resolveLazyPathIndex(arena, conf_ch.template_file.value.?, step_index); + const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(conf_ch.max_bytes)) catch |err| return step.fail("unable to read autoconf input file {s}: {t}", .{ src_path, err }); + renderAutoconfAt(step, contents, &aw, &conf_ch.values, src_path) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + else => |e| return e, }; - switch (config_header.style) { - .autoconf_undef => try render_autoconf_undef(step, contents, bw, &config_header.values, src_path), - .autoconf_at => try render_autoconf_at(step, contents, &aw, &config_header.values, src_path), - else => unreachable, - } }, - .cmake => |file_source| { - try bw.writeAll(c_generated_line); - const src_path = file_source.getPath2(b, step); - const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(config_header.max_bytes)) catch |err| { + .cmake => { + const src_path = try maker.resolveLazyPathIndex(arena, conf_ch.template_file.value.?, step_index); + const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(conf_ch.max_bytes)) catch |err| return step.fail("unable to read cmake input file {s}: {t}", .{ src_path, err }); + renderCmake(step, contents, &aw.writer, conf_ch.values, src_path) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + else => |e| return e, }; - try render_cmake(step, contents, bw, config_header.values, src_path); }, .blank => { - try bw.writeAll(c_generated_line); - try render_blank(gpa, bw, config_header.values, config_header.include_path, config_header.include_guard_override); + renderBlank(gpa, &aw.writer, conf_ch.values, conf_ch.include_path, conf_ch.include_guard_override) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + else => |e| return e, + }; }, .nasm => { - try bw.writeAll(asm_generated_line); - try render_nasm(bw, config_header.values); + renderNasm(&aw.writer, conf_ch.values) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + else => |e| return e, + }; }, } @@ -76,7 +95,10 @@ pub fn make( if (try step.cacheHit(&man)) { const digest = man.final(); - config_header.generated_dir.path = try b.cache_root.join(arena, &.{ "o", &digest }); + maker.generatedPath().* = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), + }; return; } @@ -87,35 +109,38 @@ pub fn make( // output_path is libavutil/avconfig.h // We want to open directory zig-cache/o/HASH/libavutil/ // but keep output_dir as zig-cache/o/HASH for -I include - const sub_path = b.pathJoin(&.{ "o", &digest, config_header.include_path }); - const sub_path_dirname = std.fs.path.dirname(sub_path).?; - - b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, sub_path_dirname, @errorName(err), - }); + const out_path: Path = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, conf_ch.include_path.slice(conf) }), }; + const out_path_dirname = out_path.dirname().?; + + out_path_dirname.root_dir.handle.createDirPath(io, out_path_dirname.sub_path) catch |err| + return step.fail("unable to make path {f}: {t}", .{ out_path_dirname, err }); + + out_path.root_dir.handle.writeFile(io, .{ .sub_path = out_path.sub_path, .data = output }) catch |err| + return step.fail("unable to write file {f}: {t}", .{ out_path, err }); - b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = output }) catch |err| { - return step.fail("unable to write file '{f}{s}': {s}", .{ - b.cache_root, sub_path, @errorName(err), - }); + maker.generatedPath().* = .{ + .root_dir = cache_root, + .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), }; - config_header.generated_dir.path = try b.cache_root.join(arena, &.{ "o", &digest }); - try man.writeManifest(); + try step.writeManifest(maker, &man); } -fn render_autoconf_undef( +fn renderAutoConfUndef( step: *Step, contents: []const u8, - bw: *Writer, + w: *Writer, values: *const std.array_hash_map.String(Value), src_path: []const u8, ) !void { const build = step.owner; const allocator = build.allocator; + try w.writeAll(c_generated_line); + var is_used: std.bit_set.Dynamic = try .initEmpty(allocator, values.count()); defer is_used.deinit(allocator); @@ -124,15 +149,15 @@ fn render_autoconf_undef( var line_it = std.mem.splitScalar(u8, contents, '\n'); while (line_it.next()) |line| : (line_index += 1) { if (!std.mem.startsWith(u8, line, "#")) { - try bw.writeAll(line); - try bw.writeByte('\n'); + try w.writeAll(line); + try w.writeByte('\n'); continue; } var it = std.mem.tokenizeAny(u8, line[1..], " \t\r"); const undef = it.next().?; if (!std.mem.eql(u8, undef, "undef")) { - try bw.writeAll(line); - try bw.writeByte('\n'); + try w.writeAll(line); + try w.writeByte('\n'); continue; } const name = it.next().?; @@ -144,7 +169,7 @@ fn render_autoconf_undef( continue; }; is_used.set(index); - try renderValueC(bw, name, values.values()[index]); + try renderValueC(w, name, values.values()[index]); } var unused_value_it = is_used.iterator(.{ .kind = .unset }); @@ -158,7 +183,7 @@ fn render_autoconf_undef( } } -fn render_autoconf_at( +fn renderAutoconfAt( step: *Step, contents: []const u8, aw: *Writer.Allocating, @@ -167,7 +192,9 @@ fn render_autoconf_at( ) !void { const build = step.owner; const allocator = build.allocator; - const bw = &aw.writer; + const w = &aw.writer; + + try w.writeAll(c_generated_line); const used = allocator.alloc(bool, values.count()) catch @panic("OOM"); for (used) |*u| u.* = false; @@ -180,7 +207,7 @@ fn render_autoconf_at( const last_line = line_it.index == line_it.buffer.len; const old_len = aw.written().len; - expand_variables_autoconf_at(bw, line, values, used) catch |err| switch (err) { + expandVariablesAutoconfAt(w, line, values, used) catch |err| switch (err) { error.MissingValue => { const name = aw.written()[old_len..]; defer aw.shrinkRetainingCapacity(old_len); @@ -198,7 +225,7 @@ fn render_autoconf_at( continue; }, }; - if (!last_line) try bw.writeByte('\n'); + if (!last_line) try w.writeByte('\n'); } for (values.entries.slice().items(.key), used) |name, u| { @@ -211,16 +238,18 @@ fn render_autoconf_at( if (any_errors) return error.MakeFailed; } -fn render_cmake( +fn renderCmake( step: *Step, contents: []const u8, - bw: *Writer, + w: *Writer, values: std.array_hash_map.String(Value), src_path: []const u8, ) !void { const build = step.owner; const allocator = build.allocator; + try w.writeAll(c_generated_line); + var values_copy = try values.clone(allocator); defer values_copy.deinit(allocator); @@ -230,7 +259,7 @@ fn render_cmake( while (line_it.next()) |raw_line| : (line_index += 1) { const last_line = line_it.index == line_it.buffer.len; - const line = expand_variables_cmake(allocator, raw_line, values) catch |err| switch (err) { + const line = expandVariablesCmake(allocator, raw_line, values) catch |err| switch (err) { error.InvalidCharacter => { try step.addError("{s}:{d}: error: invalid character in a variable name", .{ src_path, line_index + 1, @@ -249,16 +278,16 @@ fn render_cmake( defer allocator.free(line); const line_start = std.mem.findNone(u8, line, " \t\r") orelse { - try bw.writeAll(line); - if (!last_line) try bw.writeByte('\n'); + try w.writeAll(line); + if (!last_line) try w.writeByte('\n'); continue; }; const whitespace_prefix = line[0..line_start]; const trimmed_line = line[line_start..]; if (!std.mem.startsWith(u8, trimmed_line, "#")) { - try bw.writeAll(line); - if (!last_line) try bw.writeByte('\n'); + try w.writeAll(line); + if (!last_line) try w.writeByte('\n'); continue; } @@ -267,8 +296,8 @@ fn render_cmake( if (!std.mem.eql(u8, cmakedefine, "cmakedefine") and !std.mem.eql(u8, cmakedefine, "cmakedefine01")) { - try bw.writeAll(line); - if (!last_line) try bw.writeByte('\n'); + try w.writeAll(line); + if (!last_line) try w.writeByte('\n'); continue; } @@ -339,8 +368,8 @@ fn render_cmake( value = Value{ .ident = it.rest() }; } - try bw.writeAll(whitespace_prefix); - try renderValueC(bw, name, value); + try w.writeAll(whitespace_prefix); + try renderValueC(w, name, value); } if (any_errors) { @@ -348,13 +377,15 @@ fn render_cmake( } } -fn render_blank( +fn renderBlank( gpa: std.mem.Allocator, - bw: *Writer, + w: *Writer, defines: std.array_hash_map.String(Value), include_path: []const u8, include_guard_override: ?[]const u8, ) !void { + try w.writeAll(c_generated_line); + const include_guard_name = include_guard_override orelse blk: { const name = try gpa.dupe(u8, include_path); for (name) |*byte| { @@ -368,51 +399,52 @@ fn render_blank( }; defer if (include_guard_override == null) gpa.free(include_guard_name); - try bw.print( + try w.print( \\#ifndef {[0]s} \\#define {[0]s} \\ , .{include_guard_name}); const values = defines.values(); - for (defines.keys(), 0..) |name, i| try renderValueC(bw, name, values[i]); + for (defines.keys(), 0..) |name, i| try renderValueC(w, name, values[i]); - try bw.print( + try w.print( \\#endif /* {s} */ \\ , .{include_guard_name}); } -fn render_nasm(bw: *Writer, defines: std.array_hash_map.String(Value)) !void { - for (defines.keys(), defines.values()) |name, value| try renderValueNasm(bw, name, value); +fn renderNasm(w: *Writer, defines: std.array_hash_map.String(Value)) !void { + try w.writeAll(asm_generated_line); + for (defines.keys(), defines.values()) |name, value| try renderValueNasm(w, name, value); } -fn renderValueC(bw: *Writer, name: []const u8, value: Value) !void { +fn renderValueC(w: *Writer, name: []const u8, value: Value) !void { switch (value) { - .undef => try bw.print("/* #undef {s} */\n", .{name}), - .defined => try bw.print("#define {s}\n", .{name}), - .boolean => |b| try bw.print("#define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), - .int => |i| try bw.print("#define {s} {d}\n", .{ name, i }), - .ident => |ident| try bw.print("#define {s} {s}\n", .{ name, ident }), + .undef => try w.print("/* #undef {s} */\n", .{name}), + .defined => try w.print("#define {s}\n", .{name}), + .boolean => |b| try w.print("#define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), + .int => |i| try w.print("#define {s} {d}\n", .{ name, i }), + .ident => |ident| try w.print("#define {s} {s}\n", .{ name, ident }), // TODO: use C-specific escaping instead of zig string literals - .string => |string| try bw.print("#define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), + .string => |string| try w.print("#define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), } } -fn renderValueNasm(bw: *Writer, name: []const u8, value: Value) !void { +fn renderValueNasm(w: *Writer, name: []const u8, value: Value) !void { switch (value) { - .undef => try bw.print("; %undef {s}\n", .{name}), - .defined => try bw.print("%define {s}\n", .{name}), - .boolean => |b| try bw.print("%define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), - .int => |i| try bw.print("%define {s} {d}\n", .{ name, i }), - .ident => |ident| try bw.print("%define {s} {s}\n", .{ name, ident }), + .undef => try w.print("; %undef {s}\n", .{name}), + .defined => try w.print("%define {s}\n", .{name}), + .boolean => |b| try w.print("%define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), + .int => |i| try w.print("%define {s} {d}\n", .{ name, i }), + .ident => |ident| try w.print("%define {s} {s}\n", .{ name, ident }), // TODO: use nasm-specific escaping instead of zig string literals - .string => |string| try bw.print("%define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), + .string => |string| try w.print("%define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), } } -fn expand_variables_autoconf_at( - bw: *Writer, +fn expandVariablesAutoconfAt( + w: *Writer, contents: []const u8, values: *const std.array_hash_map.String(Value), used: []bool, @@ -437,17 +469,17 @@ fn expand_variables_autoconf_at( const key = contents[curr + 1 .. close_pos]; const index = values.getIndex(key) orelse { // Report the missing key to the caller. - try bw.writeAll(key); + try w.writeAll(key); return error.MissingValue; }; const value = values.entries.slice().items(.value)[index]; used[index] = true; - try bw.writeAll(contents[source_offset..curr]); + try w.writeAll(contents[source_offset..curr]); switch (value) { .undef, .defined => {}, - .boolean => |b| try bw.writeByte(@as(u8, '0') + @intFromBool(b)), - .int => |i| try bw.print("{d}", .{i}), - .ident, .string => |s| try bw.writeAll(s), + .boolean => |b| try w.writeByte(@as(u8, '0') + @intFromBool(b)), + .int => |i| try w.print("{d}", .{i}), + .ident, .string => |s| try w.writeAll(s), } curr = close_pos; @@ -455,10 +487,10 @@ fn expand_variables_autoconf_at( } } - try bw.writeAll(contents[source_offset..]); + try w.writeAll(contents[source_offset..]); } -fn expand_variables_cmake( +fn expandVariablesCmake( allocator: Allocator, contents: []const u8, values: std.array_hash_map.String(Value), @@ -602,7 +634,7 @@ fn testReplaceVariablesAutoconfAt( for (used) |*u| u.* = false; defer allocator.free(used); - try expand_variables_autoconf_at(&aw.writer, contents, values, used); + try expandVariablesAutoconfAt(&aw.writer, contents, values, used); for (used) |u| if (!u) return error.UnusedValue; try std.testing.expectEqualStrings(expected, aw.written()); @@ -614,13 +646,13 @@ fn testReplaceVariablesCMake( expected: []const u8, values: std.array_hash_map.String(Value), ) !void { - const actual = try expand_variables_cmake(allocator, contents, values); + const actual = try expandVariablesCmake(allocator, contents, values); defer allocator.free(actual); try std.testing.expectEqualStrings(expected, actual); } -test "expand_variables_autoconf_at simple cases" { +test "expandVariablesAutoconfAt simple cases" { const allocator = std.testing.allocator; var values: std.array_hash_map.String(Value) = .init(allocator); defer values.deinit(); @@ -716,7 +748,7 @@ test "expand_variables_autoconf_at simple cases" { values.clearRetainingCapacity(); } -test "expand_variables_autoconf_at edge cases" { +test "expandVariablesAutoconfAt edge cases" { const allocator = std.testing.allocator; var values: std.array_hash_map.String(Value) = .init(allocator); defer values.deinit(); @@ -732,7 +764,7 @@ test "expand_variables_autoconf_at edge cases" { values.clearRetainingCapacity(); } -test "expand_variables_cmake simple cases" { +test "expandVariablesCmake simple cases" { const allocator = std.testing.allocator; var values: std.array_hash_map.String(Value) = .init(allocator); defer values.deinit(); @@ -820,7 +852,7 @@ test "expand_variables_cmake simple cases" { try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${bad}", "", values)); } -test "expand_variables_cmake edge cases" { +test "expandVariablesCmake edge cases" { const allocator = std.testing.allocator; var values: std.array_hash_map.String(Value) = .init(allocator); defer values.deinit(); @@ -881,7 +913,7 @@ test "expand_variables_cmake edge cases" { try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str@ing}", "", values)); } -test "expand_variables_cmake escaped characters" { +test "expandVariablesCmake escaped characters" { const allocator = std.testing.allocator; var values: std.array_hash_map.String(Value) = .init(allocator); defer values.deinit(); diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index 96fd1dfb041b22e572c35a2e5920770f7aa0b1bd..937270a00601575050753421199db0d1964603ff 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -166,7 +166,7 @@ pub fn make( maker.generatedPath(conf_oc.output_file).* = dest_path; - man.writeManifest() catch |err| switch (err) { + step.writeManifest(maker, &man) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| try step.addError(maker, "failed writing cache manifest: {t}", .{e}), }; -- 2.54.0 From 9989f72c61e4c4e98c69f6191a63c201336a6a19 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 00:40:44 -0700 Subject: [PATCH 138/179] Maker: implement ConfigHeader --- lib/compiler/Maker/Step/ConfigHeader.zig | 782 +++++++++++------------ lib/std/Build/Configuration.zig | 32 + test/standalone/cmakedefine/build.zig | 1 - 3 files changed, 416 insertions(+), 399 deletions(-) diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index b86548b561a65686adf73a50402dcd86d9eede42..762b0ff483856df1aaa2a5ddad3d19b5c47199e7 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -10,9 +10,13 @@ const Allocator = std.mem.Allocator; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); - const header_text = "This file was generated by ConfigHeader using the Zig Build System."; - const c_generated_line = "/* " ++ header_text ++ " */\n"; - const asm_generated_line = "; " ++ header_text ++ "\n"; +const header_text = "This file was generated by ConfigHeader using the Zig Build System."; +const c_generated_line = "/* " ++ header_text ++ " */\n"; +const asm_generated_line = "; " ++ header_text ++ "\n"; + +/// Table value is whether the value is used. +const ValueMap = std.array_hash_map.String(bool); +const Value = Configuration.Step.ConfigHeader.Value; pub fn make( config_header: *ConfigHeader, @@ -23,7 +27,6 @@ pub fn make( _ = config_header; _ = progress_node; const graph = maker.graph; - const gpa = maker.gpa; const step = maker.stepByIndex(step_index); const io = graph.io; const arena = graph.arena; // TODO don't leak into the process arena @@ -32,8 +35,20 @@ pub fn make( const conf_ch = conf_step.extended.get(conf.extra).config_header; const cache_root = graph.local_cache_root; - if (conf_ch.style.getPath()) |lp| - try step.singleUnchangingWatchInput(maker, arena, lp); + const input_size_limit: Io.Limit = if (conf_ch.input_size_limit.value) |x| .limited64(x) else .unlimited; + const include_guard_override: ?[]const u8 = if (conf_ch.include_guard.value) |s| s.slice(conf) else null; + const include_path: []const u8 = conf_ch.include_path.slice(conf); + const template_file = if (conf_ch.template_file.value) |lp| + try maker.resolveLazyPathIndex(arena, lp, step_index) + else + null; + const value_pairs = conf_ch.values.slice; + + if (conf_ch.template_file.value) |lp| try step.singleUnchangingWatchInput(maker, arena, lp.get(conf)); + + var value_map: ValueMap = .empty; + try value_map.ensureTotalCapacity(arena, value_pairs.len); + for (value_pairs) |pair| value_map.putAssumeCapacityNoClobber(pair.key.slice(conf), false); var man = graph.cache.obtain(); defer man.deinit(); @@ -42,48 +57,49 @@ pub fn make( // random bytes when ConfigHeader implementation is modified in a // non-backwards-compatible way. man.hash.add(@as(u32, 0xdef08d23)); - man.hash.addBytes(conf_ch.include_path); - man.hash.addOptionalBytes(conf_ch.include_guard_override); + man.hash.add(@as(u32, @bitCast(conf_ch.flags))); + man.hash.addBytes(include_path); + man.hash.addOptionalBytes(include_guard_override); var aw: Writer.Allocating = .init(arena); defer aw.deinit(); switch (conf_ch.flags.style) { .autoconf_undef => { - const src_path = try maker.resolveLazyPathIndex(arena, conf_ch.template_file.value.?, step_index); - const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(conf_ch.max_bytes)) catch |err| - return step.fail("unable to read autoconf input file {s}: {t}", .{ src_path, err }); - renderAutoConfUndef(step, contents, &aw.writer, &conf_ch.values, src_path) catch |err| switch (err) { + const tf = template_file.?; + const contents = tf.root_dir.handle.readFileAlloc(io, tf.sub_path, arena, input_size_limit) catch |err| + return step.fail(maker, "unable to read autoconf input file {f}: {t}", .{ tf, err }); + renderAutoConfUndef(maker, step, contents, &aw.writer, value_pairs, &value_map, tf) catch |err| switch (err) { error.WriteFailed => return error.OutOfMemory, else => |e| return e, }; }, .autoconf_at => { - const src_path = try maker.resolveLazyPathIndex(arena, conf_ch.template_file.value.?, step_index); - const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(conf_ch.max_bytes)) catch |err| - return step.fail("unable to read autoconf input file {s}: {t}", .{ src_path, err }); - renderAutoconfAt(step, contents, &aw, &conf_ch.values, src_path) catch |err| switch (err) { + const tf = template_file.?; + const contents = tf.root_dir.handle.readFileAlloc(io, tf.sub_path, arena, input_size_limit) catch |err| + return step.fail(maker, "unable to read autoconf input file {f}: {t}", .{ tf, err }); + renderAutoconfAt(maker, step, contents, &aw, value_pairs, &value_map, tf) catch |err| switch (err) { error.WriteFailed => return error.OutOfMemory, else => |e| return e, }; }, .cmake => { - const src_path = try maker.resolveLazyPathIndex(arena, conf_ch.template_file.value.?, step_index); - const contents = Io.Dir.cwd().readFileAlloc(io, src_path, arena, .limited(conf_ch.max_bytes)) catch |err| - return step.fail("unable to read cmake input file {s}: {t}", .{ src_path, err }); - renderCmake(step, contents, &aw.writer, conf_ch.values, src_path) catch |err| switch (err) { + const tf = template_file.?; + const contents = tf.root_dir.handle.readFileAlloc(io, tf.sub_path, arena, input_size_limit) catch |err| + return step.fail(maker, "unable to read cmake input file {f}: {t}", .{ tf, err }); + renderCmake(arena, maker, step, contents, &aw.writer, value_pairs, &value_map, tf) catch |err| switch (err) { error.WriteFailed => return error.OutOfMemory, else => |e| return e, }; }, .blank => { - renderBlank(gpa, &aw.writer, conf_ch.values, conf_ch.include_path, conf_ch.include_guard_override) catch |err| switch (err) { + renderBlank(conf, &aw.writer, value_pairs, &value_map, include_path, include_guard_override) catch |err| switch (err) { error.WriteFailed => return error.OutOfMemory, else => |e| return e, }; }, .nasm => { - renderNasm(&aw.writer, conf_ch.values) catch |err| switch (err) { + renderNasm(conf, &aw.writer, value_pairs, &value_map) catch |err| switch (err) { error.WriteFailed => return error.OutOfMemory, else => |e| return e, }; @@ -93,9 +109,9 @@ pub fn make( const output = aw.written(); man.hash.addBytes(output); - if (try step.cacheHit(&man)) { + if (try step.cacheHit(maker, &man)) { const digest = man.final(); - maker.generatedPath().* = .{ + maker.generatedPath(conf_ch.generated_dir).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), }; @@ -116,12 +132,12 @@ pub fn make( const out_path_dirname = out_path.dirname().?; out_path_dirname.root_dir.handle.createDirPath(io, out_path_dirname.sub_path) catch |err| - return step.fail("unable to make path {f}: {t}", .{ out_path_dirname, err }); + return step.fail(maker, "unable to make path {f}: {t}", .{ out_path_dirname, err }); out_path.root_dir.handle.writeFile(io, .{ .sub_path = out_path.sub_path, .data = output }) catch |err| - return step.fail("unable to write file {f}: {t}", .{ out_path, err }); + return step.fail(maker, "unable to write file {f}: {t}", .{ out_path, err }); - maker.generatedPath().* = .{ + maker.generatedPath(conf_ch.generated_dir).* = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), }; @@ -129,21 +145,34 @@ pub fn make( try step.writeManifest(maker, &man); } +fn ensureAllValuesUsed( + maker: *Maker, + step: *Step, + value_map: *const ValueMap, + src_path: Path, +) Step.ExtendedMakeError!void { + var any_errors = false; + for (value_map.keys(), value_map.values()) |name, used| { + if (used) continue; + try step.addError(maker, "{f}: config header value unused: {s}", .{ src_path, name }); + any_errors = true; + } + if (any_errors) return error.MakeFailed; +} + fn renderAutoConfUndef( + maker: *Maker, step: *Step, contents: []const u8, w: *Writer, - values: *const std.array_hash_map.String(Value), - src_path: []const u8, + value_pairs: []const Value.Pair, + value_map: *ValueMap, + src_path: Path, ) !void { - const build = step.owner; - const allocator = build.allocator; + const conf = &maker.scanned_config.configuration; try w.writeAll(c_generated_line); - var is_used: std.bit_set.Dynamic = try .initEmpty(allocator, values.count()); - defer is_used.deinit(allocator); - var any_errors = false; var line_index: u32 = 0; var line_it = std.mem.splitScalar(u8, contents, '\n'); @@ -161,45 +190,35 @@ fn renderAutoConfUndef( continue; } const name = it.next().?; - const index = values.getIndex(name) orelse { - try step.addError("{s}:{d}: error: unspecified config header value: '{s}'", .{ + const index = value_map.getIndex(name) orelse { + try step.addError(maker, "{f}:{d}: unspecified config header value: {s}", .{ src_path, line_index + 1, name, }); any_errors = true; continue; }; - is_used.set(index); - try renderValueC(w, name, values.values()[index]); + value_map.values()[index] = true; // Set to used. + try renderValueC(conf, w, name, value_pairs[index].index); } - var unused_value_it = is_used.iterator(.{ .kind = .unset }); - while (unused_value_it.next()) |index| { - try step.addError("{s}: error: config header value unused: '{s}'", .{ src_path, values.keys()[index] }); - any_errors = true; - } - - if (any_errors) { - return error.MakeFailed; - } + try ensureAllValuesUsed(maker, step, value_map, src_path); + if (any_errors) return error.MakeFailed; } fn renderAutoconfAt( + maker: *Maker, step: *Step, contents: []const u8, aw: *Writer.Allocating, - values: *const std.array_hash_map.String(Value), - src_path: []const u8, + value_pairs: []const Value.Pair, + value_map: *const ValueMap, + src_path: Path, ) !void { - const build = step.owner; - const allocator = build.allocator; const w = &aw.writer; + const conf = &maker.scanned_config.configuration; try w.writeAll(c_generated_line); - const used = allocator.alloc(bool, values.count()) catch @panic("OOM"); - for (used) |*u| u.* = false; - defer allocator.free(used); - var any_errors = false; var line_index: u32 = 0; var line_it = std.mem.splitScalar(u8, contents, '\n'); @@ -207,19 +226,19 @@ fn renderAutoconfAt( const last_line = line_it.index == line_it.buffer.len; const old_len = aw.written().len; - expandVariablesAutoconfAt(w, line, values, used) catch |err| switch (err) { + expandVariablesAutoconfAt(w, line, conf, value_pairs, value_map) catch |err| switch (err) { error.MissingValue => { const name = aw.written()[old_len..]; defer aw.shrinkRetainingCapacity(old_len); - try step.addError("{s}:{d}: error: unspecified config header value: '{s}'", .{ + try step.addError(maker, "{f}:{d}: error: unspecified config header value: {s}", .{ src_path, line_index + 1, name, }); any_errors = true; continue; }, else => { - try step.addError("{s}:{d}: unable to substitute variable: error: {s}", .{ - src_path, line_index + 1, @errorName(err), + try step.addError(maker, "{f}:{d}: unable to substitute variable: error: {t}", .{ + src_path, line_index + 1, err, }); any_errors = true; continue; @@ -228,30 +247,23 @@ fn renderAutoconfAt( if (!last_line) try w.writeByte('\n'); } - for (values.entries.slice().items(.key), used) |name, u| { - if (!u) { - try step.addError("{s}: error: config header value unused: '{s}'", .{ src_path, name }); - any_errors = true; - } - } - + try ensureAllValuesUsed(maker, step, value_map, src_path); if (any_errors) return error.MakeFailed; } fn renderCmake( + arena: Allocator, + maker: *Maker, step: *Step, contents: []const u8, w: *Writer, - values: std.array_hash_map.String(Value), - src_path: []const u8, + value_pairs: []const Value.Pair, + value_map: *ValueMap, + src_path: Path, ) !void { - const build = step.owner; - const allocator = build.allocator; + const conf = &maker.scanned_config.configuration; - try w.writeAll(c_generated_line); - - var values_copy = try values.clone(allocator); - defer values_copy.deinit(allocator); + try w.writeAll(c_generated_line); var any_errors = false; var line_index: u32 = 0; @@ -259,23 +271,22 @@ fn renderCmake( while (line_it.next()) |raw_line| : (line_index += 1) { const last_line = line_it.index == line_it.buffer.len; - const line = expandVariablesCmake(allocator, raw_line, values) catch |err| switch (err) { + const line = expandVariablesCmake(arena, raw_line, conf, value_pairs, value_map) catch |err| switch (err) { error.InvalidCharacter => { - try step.addError("{s}:{d}: error: invalid character in a variable name", .{ + try step.addError(maker, "{f}:{d}: invalid character in a variable name", .{ src_path, line_index + 1, }); any_errors = true; continue; }, else => { - try step.addError("{s}:{d}: unable to substitute variable: error: {s}", .{ - src_path, line_index + 1, @errorName(err), + try step.addError(maker, "{f}:{d}: failed substituting variable: {t}", .{ + src_path, line_index + 1, err, }); any_errors = true; continue; }, }; - defer allocator.free(line); const line_start = std.mem.findNone(u8, line, " \t\r") orelse { try w.writeAll(line); @@ -293,152 +304,134 @@ fn renderCmake( var it = std.mem.tokenizeAny(u8, trimmed_line[1..], " \t\r"); const cmakedefine = it.next().?; - if (!std.mem.eql(u8, cmakedefine, "cmakedefine") and - !std.mem.eql(u8, cmakedefine, "cmakedefine01")) - { + + const booldefine = if (std.mem.eql(u8, cmakedefine, "cmakedefine01")) + true + else if (std.mem.eql(u8, cmakedefine, "cmakedefine")) + false + else { try w.writeAll(line); if (!last_line) try w.writeByte('\n'); continue; - } - - const booldefine = std.mem.eql(u8, cmakedefine, "cmakedefine01"); + }; const name = it.next() orelse { - try step.addError("{s}:{d}: error: missing define name", .{ - src_path, line_index + 1, - }); + try step.addError(maker, "{f}:{d}: error: missing define name", .{ src_path, line_index + 1 }); any_errors = true; continue; }; - var value = values_copy.get(name) orelse blk: { - if (booldefine) { - break :blk Value{ .int = 0 }; - } - break :blk Value.undef; + const orig_value: Value.Index = v: { + const index = value_map.getIndex(name) orelse break :v if (booldefine) .int_0 else .undef; + value_map.values()[index] = true; // Mark as used. + break :v value_pairs[index].index; }; - - value = blk: { - switch (value) { - .boolean => |b| { - if (!b) { - break :blk Value.undef; - } - }, - .int => |i| { - if (i == 0) { - break :blk Value.undef; - } - }, - .string => |string| { - if (string.len == 0) { - break :blk Value.undef; - } - }, - - else => {}, - } - break :blk value; + const value = switch (orig_value.unpack(conf)) { + .bool => |b| if (!b) .undef else orig_value, + inline .i64, .u64 => |i| if (i == 0) .undef else orig_value, + .string => |s| if (s.len == 0) .undef else orig_value, + else => orig_value, }; - if (booldefine) { - value = blk: { - switch (value) { - .undef => { - break :blk Value{ .boolean = false }; - }, - .defined => { - break :blk Value{ .boolean = false }; - }, - .boolean => |b| { - break :blk Value{ .boolean = b }; - }, - .int => |i| { - break :blk Value{ .boolean = i != 0 }; - }, - .string => |string| { - break :blk Value{ .boolean = string.len != 0 }; - }, - - else => { - break :blk Value{ .boolean = false }; - }, - } - }; - } else if (value != Value.undef) { - value = Value{ .ident = it.rest() }; - } - try w.writeAll(whitespace_prefix); - try renderValueC(w, name, value); - } - if (any_errors) { - return error.HeaderConfigFailed; + if (booldefine) { + try renderValueCBool(w, name, switch (value.unpack(conf)) { + .undef, .defined => false, + .bool => |b| b, + inline .u64, .i64 => |i| i != 0, + .string => |s| s.len != 0, + .ident => false, + }); + } else if (value != .undef) { + try renderValueCIdent(w, name, it.rest()); + } else { + try renderValueC(conf, w, name, value); + } } + + try ensureAllValuesUsed(maker, step, value_map, src_path); + if (any_errors) return error.MakeFailed; } fn renderBlank( - gpa: std.mem.Allocator, + conf: *const Configuration, w: *Writer, - defines: std.array_hash_map.String(Value), + value_pairs: []const Value.Pair, + value_map: *const ValueMap, include_path: []const u8, include_guard_override: ?[]const u8, ) !void { try w.writeAll(c_generated_line); - const include_guard_name = include_guard_override orelse blk: { - const name = try gpa.dupe(u8, include_path); - for (name) |*byte| { - switch (byte.*) { - 'a'...'z' => byte.* = byte.* - 'a' + 'A', - 'A'...'Z', '0'...'9' => continue, - else => byte.* = '_', - } - } - break :blk name; + const include_guard_fmt: IncludeGuardFmt = .{ + .include_path = include_path, + .override = include_guard_override, }; - defer if (include_guard_override == null) gpa.free(include_guard_name); try w.print( - \\#ifndef {[0]s} - \\#define {[0]s} + \\#ifndef {[0]f} + \\#define {[0]f} \\ - , .{include_guard_name}); + , .{include_guard_fmt}); - const values = defines.values(); - for (defines.keys(), 0..) |name, i| try renderValueC(w, name, values[i]); + for (value_map.keys(), value_pairs) |name, pair| try renderValueC(conf, w, name, pair.index); try w.print( - \\#endif /* {s} */ + \\#endif /* {f} */ \\ - , .{include_guard_name}); + , .{include_guard_fmt}); } -fn renderNasm(w: *Writer, defines: std.array_hash_map.String(Value)) !void { +const IncludeGuardFmt = struct { + include_path: []const u8, + override: ?[]const u8, + + pub fn format(this: @This(), w: *Writer) Writer.Error!void { + if (this.override) |s| return w.writeAll(s); + for (this.include_path) |byte| switch (byte) { + 'a'...'z' => try w.writeByte(byte - 'a' + 'A'), + 'A'...'Z', '0'...'9' => continue, + else => try w.writeByte('_'), + }; + } +}; + +fn renderNasm( + conf: *const Configuration, + w: *Writer, + value_pairs: []const Value.Pair, + value_map: *const ValueMap, +) !void { try w.writeAll(asm_generated_line); - for (defines.keys(), defines.values()) |name, value| try renderValueNasm(w, name, value); + for (value_map.keys(), value_pairs) |name, pair| try renderValueNasm(conf, w, name, pair.index); } -fn renderValueC(w: *Writer, name: []const u8, value: Value) !void { - switch (value) { +fn renderValueC(conf: *const Configuration, w: *Writer, name: []const u8, value: Value.Index) !void { + switch (value.unpack(conf)) { .undef => try w.print("/* #undef {s} */\n", .{name}), .defined => try w.print("#define {s}\n", .{name}), - .boolean => |b| try w.print("#define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), - .int => |i| try w.print("#define {s} {d}\n", .{ name, i }), - .ident => |ident| try w.print("#define {s} {s}\n", .{ name, ident }), - // TODO: use C-specific escaping instead of zig string literals + .bool => |b| return renderValueCBool(w, name, b), + inline .u64, .i64 => |i| try w.print("#define {s} {d}\n", .{ name, i }), + .ident => |ident| return renderValueCIdent(w, name, ident), .string => |string| try w.print("#define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), } } -fn renderValueNasm(w: *Writer, name: []const u8, value: Value) !void { - switch (value) { +fn renderValueCIdent(w: *Writer, name: []const u8, ident: []const u8) Writer.Error!void { + return w.print("#define {s} {s}\n", .{ name, ident }); +} + +fn renderValueCBool(w: *Writer, name: []const u8, b: bool) Writer.Error!void { + return w.print("#define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }); +} + +fn renderValueNasm(conf: *const Configuration, w: *Writer, name: []const u8, value: Value.Index) !void { + switch (value.unpack(conf)) { .undef => try w.print("; %undef {s}\n", .{name}), .defined => try w.print("%define {s}\n", .{name}), - .boolean => |b| try w.print("%define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), - .int => |i| try w.print("%define {s} {d}\n", .{ name, i }), + .bool => |b| try w.print("%define {s} {c}\n", .{ name, @as(u8, '0') + @intFromBool(b) }), + inline .u64, .i64 => |i| try w.print("%define {s} {d}\n", .{ name, i }), .ident => |ident| try w.print("%define {s} {s}\n", .{ name, ident }), - // TODO: use nasm-specific escaping instead of zig string literals .string => |string| try w.print("%define {s} \"{f}\"\n", .{ name, std.zig.fmtString(string) }), } } @@ -446,8 +439,9 @@ fn renderValueNasm(w: *Writer, name: []const u8, value: Value) !void { fn expandVariablesAutoconfAt( w: *Writer, contents: []const u8, - values: *const std.array_hash_map.String(Value), - used: []bool, + conf: *const Configuration, + value_pairs: []const Value.Pair, + value_map: *const ValueMap, ) !void { const valid_varname_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"; @@ -467,18 +461,18 @@ fn expandVariablesAutoconfAt( } const key = contents[curr + 1 .. close_pos]; - const index = values.getIndex(key) orelse { + const index = value_map.getIndex(key) orelse { // Report the missing key to the caller. try w.writeAll(key); return error.MissingValue; }; - const value = values.entries.slice().items(.value)[index]; - used[index] = true; + const value = value_pairs[index].index; + value_map.values()[index] = true; // Mark as used. try w.writeAll(contents[source_offset..curr]); - switch (value) { + switch (value.unpack(conf)) { .undef, .defined => {}, - .boolean => |b| try w.writeByte(@as(u8, '0') + @intFromBool(b)), - .int => |i| try w.print("{d}", .{i}), + .bool => |b| try w.writeByte(@as(u8, '0') + @intFromBool(b)), + inline .u64, .i64 => |i| try w.print("{d}", .{i}), .ident, .string => |s| try w.writeAll(s), } @@ -491,12 +485,13 @@ fn expandVariablesAutoconfAt( } fn expandVariablesCmake( - allocator: Allocator, + arena: Allocator, contents: []const u8, - values: std.array_hash_map.String(Value), + conf: *const Configuration, + value_pairs: []const Value.Pair, + value_map: *const ValueMap, ) ![]const u8 { - var result: std.array_list.Managed(u8) = .init(allocator); - errdefer result.deinit(); + var result: std.ArrayList(u8) = .empty; const valid_varname_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789/_.+-"; const open_var = "${"; @@ -507,8 +502,7 @@ fn expandVariablesCmake( source: usize, target: usize, }; - var var_stack: std.array_list.Managed(Position) = .init(allocator); - defer var_stack.deinit(); + var var_stack: std.ArrayList(Position) = .empty; loop: while (curr < contents.len) : (curr += 1) { switch (contents[curr]) { '@' => blk: { @@ -524,20 +518,16 @@ fn expandVariablesCmake( } const key = contents[curr + 1 .. close_pos]; - const value = values.get(key) orelse return error.MissingValue; + const index = value_map.getIndex(key) orelse return error.MissingValue; + value_map.values()[index] = true; // Mark as used. + const value = value_pairs[index].index; const missing = contents[source_offset..curr]; - try result.appendSlice(missing); - switch (value) { + try result.appendSlice(arena, missing); + switch (value.unpack(conf)) { .undef, .defined => {}, - .boolean => |b| { - try result.append(if (b) '1' else '0'); - }, - .int => |i| { - try result.print("{d}", .{i}); - }, - .ident, .string => |s| { - try result.appendSlice(s); - }, + .bool => |b| try result.append(arena, if (b) '1' else '0'), + inline .i64, .u64 => |i| try result.print(arena, "{d}", .{i}), + .ident, .string => |s| try result.appendSlice(arena, s), } curr = close_pos; @@ -553,12 +543,12 @@ fn expandVariablesCmake( break :blk; } const missing = contents[source_offset..curr]; - try result.appendSlice(missing); - try result.appendSlice(open_var); + try result.appendSlice(arena, missing); + try result.appendSlice(arena, open_var); source_offset = curr + open_var.len; curr = next; - try var_stack.append(Position{ + try var_stack.append(arena, .{ .source = curr, .target = result.items.len - open_var.len, }); @@ -575,26 +565,22 @@ fn expandVariablesCmake( source_offset += open_var.len; } const missing = contents[source_offset..curr]; - try result.appendSlice(missing); + try result.appendSlice(arena, missing); const key_start = open_pos.target + open_var.len; const key = result.items[key_start..]; if (key.len == 0) { return error.MissingKey; } - const value = values.get(key) orelse return error.MissingValue; + const index = value_map.getIndex(key) orelse return error.MissingValue; + value_map.values()[index] = true; // Mark as used. + const value = value_pairs[index].index; result.shrinkRetainingCapacity(result.items.len - key.len - open_var.len); - switch (value) { + switch (value.unpack(conf)) { .undef, .defined => {}, - .boolean => |b| { - try result.append(if (b) '1' else '0'); - }, - .int => |i| { - try result.print("{d}", .{i}); - }, - .ident, .string => |s| { - try result.appendSlice(s); - }, + .bool => |b| try result.append(arena, if (b) '1' else '0'), + inline .i64, .u64 => |i| try result.print(arena, "{d}", .{i}), + .ident, .string => |s| try result.appendSlice(arena, s), } source_offset = curr + 1; @@ -615,320 +601,320 @@ fn expandVariablesCmake( if (source_offset != contents.len) { const missing = contents[source_offset..]; - try result.appendSlice(missing); + try result.appendSlice(arena, missing); } - return result.toOwnedSlice(); + try result.shrinkToLen(arena); + + return result.toOwnedSliceAssert(); } fn testReplaceVariablesAutoconfAt( - allocator: Allocator, + arena: Allocator, contents: []const u8, expected: []const u8, - values: std.array_hash_map.String(Value), + value_map: *const ValueMap, ) !void { - var aw: Writer.Allocating = .init(allocator); + var aw: Writer.Allocating = .init(arena); defer aw.deinit(); - const used = try allocator.alloc(bool, values.count()); + const used = try arena.alloc(bool, value_map.count()); for (used) |*u| u.* = false; - defer allocator.free(used); - try expandVariablesAutoconfAt(&aw.writer, contents, values, used); + try expandVariablesAutoconfAt(&aw.writer, contents, value_map, used); for (used) |u| if (!u) return error.UnusedValue; try std.testing.expectEqualStrings(expected, aw.written()); } fn testReplaceVariablesCMake( - allocator: Allocator, + arena: Allocator, contents: []const u8, expected: []const u8, - values: std.array_hash_map.String(Value), + value_map: *const ValueMap, ) !void { - const actual = try expandVariablesCmake(allocator, contents, values); - defer allocator.free(actual); + const actual = try expandVariablesCmake(arena, contents, value_map); try std.testing.expectEqualStrings(expected, actual); } test "expandVariablesAutoconfAt simple cases" { const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); + var value_map: ValueMap = .empty; + defer value_map.deinit(); // empty strings are preserved - try testReplaceVariablesAutoconfAt(allocator, "", "", values); + try testReplaceVariablesAutoconfAt(allocator, "", "", value_map); // line with misc content is preserved - try testReplaceVariablesAutoconfAt(allocator, "no substitution", "no substitution", values); + try testReplaceVariablesAutoconfAt(allocator, "no substitution", "no substitution", value_map); // empty @ sigils are preserved - try testReplaceVariablesAutoconfAt(allocator, "@", "@", values); - try testReplaceVariablesAutoconfAt(allocator, "@@", "@@", values); - try testReplaceVariablesAutoconfAt(allocator, "@@@", "@@@", values); - try testReplaceVariablesAutoconfAt(allocator, "@@@@", "@@@@", values); + try testReplaceVariablesAutoconfAt(allocator, "@", "@", value_map); + try testReplaceVariablesAutoconfAt(allocator, "@@", "@@", value_map); + try testReplaceVariablesAutoconfAt(allocator, "@@@", "@@@", value_map); + try testReplaceVariablesAutoconfAt(allocator, "@@@@", "@@@@", value_map); // simple substitution - try values.putNoClobber("undef", .undef); - try testReplaceVariablesAutoconfAt(allocator, "@undef@", "", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("undef", .undef); + try testReplaceVariablesAutoconfAt(allocator, "@undef@", "", value_map); + value_map.clearRetainingCapacity(); - try values.putNoClobber("defined", .defined); - try testReplaceVariablesAutoconfAt(allocator, "@defined@", "", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("defined", .defined); + try testReplaceVariablesAutoconfAt(allocator, "@defined@", "", value_map); + value_map.clearRetainingCapacity(); - try values.putNoClobber("true", Value{ .boolean = true }); - try testReplaceVariablesAutoconfAt(allocator, "@true@", "1", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("true", Value{ .boolean = true }); + try testReplaceVariablesAutoconfAt(allocator, "@true@", "1", value_map); + value_map.clearRetainingCapacity(); - try values.putNoClobber("false", Value{ .boolean = false }); - try testReplaceVariablesAutoconfAt(allocator, "@false@", "0", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("false", Value{ .boolean = false }); + try testReplaceVariablesAutoconfAt(allocator, "@false@", "0", value_map); + value_map.clearRetainingCapacity(); - try values.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@", "42", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("int", Value{ .int = 42 }); + try testReplaceVariablesAutoconfAt(allocator, "@int@", "42", value_map); + value_map.clearRetainingCapacity(); - try values.putNoClobber("ident", Value{ .string = "value" }); - try testReplaceVariablesAutoconfAt(allocator, "@ident@", "value", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("ident", Value{ .string = "value" }); + try testReplaceVariablesAutoconfAt(allocator, "@ident@", "value", value_map); + value_map.clearRetainingCapacity(); - try values.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@", "text", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("string", Value{ .string = "text" }); + try testReplaceVariablesAutoconfAt(allocator, "@string@", "text", value_map); + value_map.clearRetainingCapacity(); // double packed substitution - try values.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@@string@", "texttext", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("string", Value{ .string = "text" }); + try testReplaceVariablesAutoconfAt(allocator, "@string@@string@", "texttext", value_map); + value_map.clearRetainingCapacity(); // triple packed substitution - try values.putNoClobber("int", Value{ .int = 42 }); - try values.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@@int@@string@", "text42text", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("int", Value{ .int = 42 }); + try value_map.putNoClobber("string", Value{ .string = "text" }); + try testReplaceVariablesAutoconfAt(allocator, "@string@@int@@string@", "text42text", value_map); + value_map.clearRetainingCapacity(); // double separated substitution - try values.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@.@int@", "42.42", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("int", Value{ .int = 42 }); + try testReplaceVariablesAutoconfAt(allocator, "@int@.@int@", "42.42", value_map); + value_map.clearRetainingCapacity(); // triple separated substitution - try values.putNoClobber("true", Value{ .boolean = true }); - try values.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@.@true@.@int@", "42.1.42", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("true", Value{ .boolean = true }); + try value_map.putNoClobber("int", Value{ .int = 42 }); + try testReplaceVariablesAutoconfAt(allocator, "@int@.@true@.@int@", "42.1.42", value_map); + value_map.clearRetainingCapacity(); // misc prefix is preserved - try values.putNoClobber("false", Value{ .boolean = false }); - try testReplaceVariablesAutoconfAt(allocator, "false is @false@", "false is 0", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("false", Value{ .boolean = false }); + try testReplaceVariablesAutoconfAt(allocator, "false is @false@", "false is 0", value_map); + value_map.clearRetainingCapacity(); // misc suffix is preserved - try values.putNoClobber("true", Value{ .boolean = true }); - try testReplaceVariablesAutoconfAt(allocator, "@true@ is true", "1 is true", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("true", Value{ .boolean = true }); + try testReplaceVariablesAutoconfAt(allocator, "@true@ is true", "1 is true", value_map); + value_map.clearRetainingCapacity(); // surrounding content is preserved - try values.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("int", Value{ .int = 42 }); + try testReplaceVariablesAutoconfAt(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", value_map); + value_map.clearRetainingCapacity(); // incomplete key is preserved - try testReplaceVariablesAutoconfAt(allocator, "@undef", "@undef", values); + try testReplaceVariablesAutoconfAt(allocator, "@undef", "@undef", value_map); // unknown key leads to an error - try std.testing.expectError(error.MissingValue, testReplaceVariablesAutoconfAt(allocator, "@bad@", "", values)); + try std.testing.expectError(error.MissingValue, testReplaceVariablesAutoconfAt(allocator, "@bad@", "", value_map)); // unused key leads to an error - try values.putNoClobber("int", Value{ .int = 42 }); - try values.putNoClobber("false", Value{ .boolean = false }); - try std.testing.expectError(error.UnusedValue, testReplaceVariablesAutoconfAt(allocator, "@int", "", values)); - values.clearRetainingCapacity(); + try value_map.putNoClobber("int", Value{ .int = 42 }); + try value_map.putNoClobber("false", Value{ .boolean = false }); + try std.testing.expectError(error.UnusedValue, testReplaceVariablesAutoconfAt(allocator, "@int", "", value_map)); + value_map.clearRetainingCapacity(); } test "expandVariablesAutoconfAt edge cases" { const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); + var value_map: std.array_hash_map.String(Value) = .init(allocator); + defer value_map.deinit(); // @-vars resolved only when they wrap valid characters, otherwise considered literals - try values.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@@string@@", "@text@", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("string", Value{ .string = "text" }); + try testReplaceVariablesAutoconfAt(allocator, "@@string@@", "@text@", value_map); + value_map.clearRetainingCapacity(); // expanded variables are considered strings after expansion - try values.putNoClobber("string_at", Value{ .string = "@string@" }); - try testReplaceVariablesAutoconfAt(allocator, "@string_at@", "@string@", values); - values.clearRetainingCapacity(); + try value_map.putNoClobber("string_at", Value{ .string = "@string@" }); + try testReplaceVariablesAutoconfAt(allocator, "@string_at@", "@string@", value_map); + value_map.clearRetainingCapacity(); } test "expandVariablesCmake simple cases" { const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); + var value_map: std.array_hash_map.String(Value) = .init(allocator); + defer value_map.deinit(); - try values.putNoClobber("undef", .undef); - try values.putNoClobber("defined", .defined); - try values.putNoClobber("true", Value{ .boolean = true }); - try values.putNoClobber("false", Value{ .boolean = false }); - try values.putNoClobber("int", Value{ .int = 42 }); - try values.putNoClobber("ident", Value{ .string = "value" }); - try values.putNoClobber("string", Value{ .string = "text" }); + try value_map.putNoClobber("undef", .undef); + try value_map.putNoClobber("defined", .defined); + try value_map.putNoClobber("true", Value{ .boolean = true }); + try value_map.putNoClobber("false", Value{ .boolean = false }); + try value_map.putNoClobber("int", Value{ .int = 42 }); + try value_map.putNoClobber("ident", Value{ .string = "value" }); + try value_map.putNoClobber("string", Value{ .string = "text" }); // empty strings are preserved - try testReplaceVariablesCMake(allocator, "", "", values); + try testReplaceVariablesCMake(allocator, "", "", value_map); // line with misc content is preserved - try testReplaceVariablesCMake(allocator, "no substitution", "no substitution", values); + try testReplaceVariablesCMake(allocator, "no substitution", "no substitution", value_map); // empty ${} wrapper leads to an error - try std.testing.expectError(error.MissingKey, testReplaceVariablesCMake(allocator, "${}", "", values)); + try std.testing.expectError(error.MissingKey, testReplaceVariablesCMake(allocator, "${}", "", value_map)); // empty @ sigils are preserved - try testReplaceVariablesCMake(allocator, "@", "@", values); - try testReplaceVariablesCMake(allocator, "@@", "@@", values); - try testReplaceVariablesCMake(allocator, "@@@", "@@@", values); - try testReplaceVariablesCMake(allocator, "@@@@", "@@@@", values); + try testReplaceVariablesCMake(allocator, "@", "@", value_map); + try testReplaceVariablesCMake(allocator, "@@", "@@", value_map); + try testReplaceVariablesCMake(allocator, "@@@", "@@@", value_map); + try testReplaceVariablesCMake(allocator, "@@@@", "@@@@", value_map); // simple substitution - try testReplaceVariablesCMake(allocator, "@undef@", "", values); - try testReplaceVariablesCMake(allocator, "${undef}", "", values); - try testReplaceVariablesCMake(allocator, "@defined@", "", values); - try testReplaceVariablesCMake(allocator, "${defined}", "", values); - try testReplaceVariablesCMake(allocator, "@true@", "1", values); - try testReplaceVariablesCMake(allocator, "${true}", "1", values); - try testReplaceVariablesCMake(allocator, "@false@", "0", values); - try testReplaceVariablesCMake(allocator, "${false}", "0", values); - try testReplaceVariablesCMake(allocator, "@int@", "42", values); - try testReplaceVariablesCMake(allocator, "${int}", "42", values); - try testReplaceVariablesCMake(allocator, "@ident@", "value", values); - try testReplaceVariablesCMake(allocator, "${ident}", "value", values); - try testReplaceVariablesCMake(allocator, "@string@", "text", values); - try testReplaceVariablesCMake(allocator, "${string}", "text", values); + try testReplaceVariablesCMake(allocator, "@undef@", "", value_map); + try testReplaceVariablesCMake(allocator, "${undef}", "", value_map); + try testReplaceVariablesCMake(allocator, "@defined@", "", value_map); + try testReplaceVariablesCMake(allocator, "${defined}", "", value_map); + try testReplaceVariablesCMake(allocator, "@true@", "1", value_map); + try testReplaceVariablesCMake(allocator, "${true}", "1", value_map); + try testReplaceVariablesCMake(allocator, "@false@", "0", value_map); + try testReplaceVariablesCMake(allocator, "${false}", "0", value_map); + try testReplaceVariablesCMake(allocator, "@int@", "42", value_map); + try testReplaceVariablesCMake(allocator, "${int}", "42", value_map); + try testReplaceVariablesCMake(allocator, "@ident@", "value", value_map); + try testReplaceVariablesCMake(allocator, "${ident}", "value", value_map); + try testReplaceVariablesCMake(allocator, "@string@", "text", value_map); + try testReplaceVariablesCMake(allocator, "${string}", "text", value_map); // double packed substitution - try testReplaceVariablesCMake(allocator, "@string@@string@", "texttext", values); - try testReplaceVariablesCMake(allocator, "${string}${string}", "texttext", values); + try testReplaceVariablesCMake(allocator, "@string@@string@", "texttext", value_map); + try testReplaceVariablesCMake(allocator, "${string}${string}", "texttext", value_map); // triple packed substitution - try testReplaceVariablesCMake(allocator, "@string@@int@@string@", "text42text", values); - try testReplaceVariablesCMake(allocator, "@string@${int}@string@", "text42text", values); - try testReplaceVariablesCMake(allocator, "${string}@int@${string}", "text42text", values); - try testReplaceVariablesCMake(allocator, "${string}${int}${string}", "text42text", values); + try testReplaceVariablesCMake(allocator, "@string@@int@@string@", "text42text", value_map); + try testReplaceVariablesCMake(allocator, "@string@${int}@string@", "text42text", value_map); + try testReplaceVariablesCMake(allocator, "${string}@int@${string}", "text42text", value_map); + try testReplaceVariablesCMake(allocator, "${string}${int}${string}", "text42text", value_map); // double separated substitution - try testReplaceVariablesCMake(allocator, "@int@.@int@", "42.42", values); - try testReplaceVariablesCMake(allocator, "${int}.${int}", "42.42", values); + try testReplaceVariablesCMake(allocator, "@int@.@int@", "42.42", value_map); + try testReplaceVariablesCMake(allocator, "${int}.${int}", "42.42", value_map); // triple separated substitution - try testReplaceVariablesCMake(allocator, "@int@.@true@.@int@", "42.1.42", values); - try testReplaceVariablesCMake(allocator, "@int@.${true}.@int@", "42.1.42", values); - try testReplaceVariablesCMake(allocator, "${int}.@true@.${int}", "42.1.42", values); - try testReplaceVariablesCMake(allocator, "${int}.${true}.${int}", "42.1.42", values); + try testReplaceVariablesCMake(allocator, "@int@.@true@.@int@", "42.1.42", value_map); + try testReplaceVariablesCMake(allocator, "@int@.${true}.@int@", "42.1.42", value_map); + try testReplaceVariablesCMake(allocator, "${int}.@true@.${int}", "42.1.42", value_map); + try testReplaceVariablesCMake(allocator, "${int}.${true}.${int}", "42.1.42", value_map); // misc prefix is preserved - try testReplaceVariablesCMake(allocator, "false is @false@", "false is 0", values); - try testReplaceVariablesCMake(allocator, "false is ${false}", "false is 0", values); + try testReplaceVariablesCMake(allocator, "false is @false@", "false is 0", value_map); + try testReplaceVariablesCMake(allocator, "false is ${false}", "false is 0", value_map); // misc suffix is preserved - try testReplaceVariablesCMake(allocator, "@true@ is true", "1 is true", values); - try testReplaceVariablesCMake(allocator, "${true} is true", "1 is true", values); + try testReplaceVariablesCMake(allocator, "@true@ is true", "1 is true", value_map); + try testReplaceVariablesCMake(allocator, "${true} is true", "1 is true", value_map); // surrounding content is preserved - try testReplaceVariablesCMake(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", values); - try testReplaceVariablesCMake(allocator, "what is 6*7? ${int}!", "what is 6*7? 42!", values); + try testReplaceVariablesCMake(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", value_map); + try testReplaceVariablesCMake(allocator, "what is 6*7? ${int}!", "what is 6*7? 42!", value_map); // incomplete key is preserved - try testReplaceVariablesCMake(allocator, "@undef", "@undef", values); - try testReplaceVariablesCMake(allocator, "${undef", "${undef", values); - try testReplaceVariablesCMake(allocator, "{undef}", "{undef}", values); - try testReplaceVariablesCMake(allocator, "undef@", "undef@", values); - try testReplaceVariablesCMake(allocator, "undef}", "undef}", values); + try testReplaceVariablesCMake(allocator, "@undef", "@undef", value_map); + try testReplaceVariablesCMake(allocator, "${undef", "${undef", value_map); + try testReplaceVariablesCMake(allocator, "{undef}", "{undef}", value_map); + try testReplaceVariablesCMake(allocator, "undef@", "undef@", value_map); + try testReplaceVariablesCMake(allocator, "undef}", "undef}", value_map); // unknown key leads to an error - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@bad@", "", values)); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${bad}", "", values)); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@bad@", "", value_map)); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${bad}", "", value_map)); } test "expandVariablesCmake edge cases" { const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); + var value_map: std.array_hash_map.String(Value) = .init(allocator); + defer value_map.deinit(); // special symbols - try values.putNoClobber("at", Value{ .string = "@" }); - try values.putNoClobber("dollar", Value{ .string = "$" }); - try values.putNoClobber("underscore", Value{ .string = "_" }); + try value_map.putNoClobber("at", Value{ .string = "@" }); + try value_map.putNoClobber("dollar", Value{ .string = "$" }); + try value_map.putNoClobber("underscore", Value{ .string = "_" }); // basic value - try values.putNoClobber("string", Value{ .string = "text" }); + try value_map.putNoClobber("string", Value{ .string = "text" }); - // proxy case values - try values.putNoClobber("string_proxy", Value{ .string = "string" }); - try values.putNoClobber("string_at", Value{ .string = "@string@" }); - try values.putNoClobber("string_curly", Value{ .string = "{string}" }); - try values.putNoClobber("string_var", Value{ .string = "${string}" }); + // proxy case value_map + try value_map.putNoClobber("string_proxy", Value{ .string = "string" }); + try value_map.putNoClobber("string_at", Value{ .string = "@string@" }); + try value_map.putNoClobber("string_curly", Value{ .string = "{string}" }); + try value_map.putNoClobber("string_var", Value{ .string = "${string}" }); - // stack case values - try values.putNoClobber("nest_underscore_proxy", Value{ .string = "underscore" }); - try values.putNoClobber("nest_proxy", Value{ .string = "nest_underscore_proxy" }); + // stack case value_map + try value_map.putNoClobber("nest_underscore_proxy", Value{ .string = "underscore" }); + try value_map.putNoClobber("nest_proxy", Value{ .string = "nest_underscore_proxy" }); // @-vars resolved only when they wrap valid characters, otherwise considered literals - try testReplaceVariablesCMake(allocator, "@@string@@", "@text@", values); - try testReplaceVariablesCMake(allocator, "@${string}@", "@text@", values); + try testReplaceVariablesCMake(allocator, "@@string@@", "@text@", value_map); + try testReplaceVariablesCMake(allocator, "@${string}@", "@text@", value_map); // @-vars are resolved inside ${}-vars - try testReplaceVariablesCMake(allocator, "${@string_proxy@}", "text", values); + try testReplaceVariablesCMake(allocator, "${@string_proxy@}", "text", value_map); // expanded variables are considered strings after expansion - try testReplaceVariablesCMake(allocator, "@string_at@", "@string@", values); - try testReplaceVariablesCMake(allocator, "${string_at}", "@string@", values); - try testReplaceVariablesCMake(allocator, "$@string_curly@", "${string}", values); - try testReplaceVariablesCMake(allocator, "$${string_curly}", "${string}", values); - try testReplaceVariablesCMake(allocator, "${string_var}", "${string}", values); - try testReplaceVariablesCMake(allocator, "@string_var@", "${string}", values); - try testReplaceVariablesCMake(allocator, "${dollar}{${string}}", "${text}", values); - try testReplaceVariablesCMake(allocator, "@dollar@{${string}}", "${text}", values); - try testReplaceVariablesCMake(allocator, "@dollar@{@string@}", "${text}", values); + try testReplaceVariablesCMake(allocator, "@string_at@", "@string@", value_map); + try testReplaceVariablesCMake(allocator, "${string_at}", "@string@", value_map); + try testReplaceVariablesCMake(allocator, "$@string_curly@", "${string}", value_map); + try testReplaceVariablesCMake(allocator, "$${string_curly}", "${string}", value_map); + try testReplaceVariablesCMake(allocator, "${string_var}", "${string}", value_map); + try testReplaceVariablesCMake(allocator, "@string_var@", "${string}", value_map); + try testReplaceVariablesCMake(allocator, "${dollar}{${string}}", "${text}", value_map); + try testReplaceVariablesCMake(allocator, "@dollar@{${string}}", "${text}", value_map); + try testReplaceVariablesCMake(allocator, "@dollar@{@string@}", "${text}", value_map); // when expanded variables contain invalid characters, they prevent further expansion - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${${string_var}}", "", values)); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${@string_var@}", "", values)); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${${string_var}}", "", value_map)); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${@string_var@}", "", value_map)); // nested expanded variables are expanded from the inside out - try testReplaceVariablesCMake(allocator, "${string${underscore}proxy}", "string", values); - try testReplaceVariablesCMake(allocator, "${string@underscore@proxy}", "string", values); + try testReplaceVariablesCMake(allocator, "${string${underscore}proxy}", "string", value_map); + try testReplaceVariablesCMake(allocator, "${string@underscore@proxy}", "string", value_map); // nested vars are only expanded when ${} is closed - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@underscore@proxy@", "", values)); - try testReplaceVariablesCMake(allocator, "${nest${underscore}proxy}", "nest_underscore_proxy", values); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@@nest_underscore@underscore@proxy@@proxy@", "", values)); - try testReplaceVariablesCMake(allocator, "${nest${${nest_underscore${underscore}proxy}}proxy}", "nest_underscore_proxy", values); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@underscore@proxy@", "", value_map)); + try testReplaceVariablesCMake(allocator, "${nest${underscore}proxy}", "nest_underscore_proxy", value_map); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@@nest_underscore@underscore@proxy@@proxy@", "", value_map)); + try testReplaceVariablesCMake(allocator, "${nest${${nest_underscore${underscore}proxy}}proxy}", "nest_underscore_proxy", value_map); // invalid characters lead to an error - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str*ing}", "", values)); - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str$ing}", "", values)); - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str@ing}", "", values)); + try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str*ing}", "", value_map)); + try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str$ing}", "", value_map)); + try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str@ing}", "", value_map)); } test "expandVariablesCmake escaped characters" { const allocator = std.testing.allocator; - var values: std.array_hash_map.String(Value) = .init(allocator); - defer values.deinit(); + var value_map: std.array_hash_map.String(Value) = .init(allocator); + defer value_map.deinit(); - try values.putNoClobber("string", Value{ .string = "text" }); + try value_map.putNoClobber("string", Value{ .string = "text" }); // backslash is an invalid character for @ lookup - try testReplaceVariablesCMake(allocator, "\\@string\\@", "\\@string\\@", values); + try testReplaceVariablesCMake(allocator, "\\@string\\@", "\\@string\\@", value_map); // backslash is preserved, but doesn't affect ${} variable expansion - try testReplaceVariablesCMake(allocator, "\\${string}", "\\text", values); + try testReplaceVariablesCMake(allocator, "\\${string}", "\\text", value_map); // backslash breaks ${} opening bracket identification - try testReplaceVariablesCMake(allocator, "$\\{string}", "$\\{string}", values); + try testReplaceVariablesCMake(allocator, "$\\{string}", "$\\{string}", value_map); // backslash is skipped when checking for invalid characters, yet it mangles the key - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${string\\}", "", values)); + try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${string\\}", "", value_map)); } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index d19a357b50571ceb93b455e38a590241957414a7..5040a5f619f8ca868b648a99cc33c34d25e7b9ed 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1117,6 +1117,38 @@ pub const Step = extern struct { undef = max_u32 - 1, defined = max_u32, _, + + pub fn unpack(this: @This(), c: *const Configuration) Unpacked { + return switch (this) { + .int_0 => .{ .u64 = 0 }, + .int_1 => .{ .u64 = 1 }, + .bool_false => .{ .bool = false }, + .bool_true => .{ .bool = true }, + .undef => .undef, + .defined => .defined, + _ => { + const value = extraData(c, Value, @intFromEnum(this)); + return switch (value.flags.tag) { + .ident => .{ .ident = value.ident.value.?.slice(c) }, + .string => .{ .string = value.string.value.?.slice(c) }, + .small_unsigned => .{ .u64 = value.flags.small }, + .small_signed => .{ .i64 = @as(i29, @bitCast(value.flags.small)) }, + .i64 => .{ .i64 = value.i64.value.? }, + .u64 => .{ .u64 = value.u64.value.? }, + }; + }, + }; + } + }; + + pub const Unpacked = union(enum) { + bool: bool, + undef, + defined, + i64: i64, + u64: u64, + ident: []const u8, + string: []const u8, }; pub fn initSigned(x: i64) @This() { diff --git a/test/standalone/cmakedefine/build.zig b/test/standalone/cmakedefine/build.zig index a7a395b00c1a3c17495161185cd8118b4c839636..78fe1f1cbfc9b2ec07dba6dea6fe808190e37c7d 100644 --- a/test/standalone/cmakedefine/build.zig +++ b/test/standalone/cmakedefine/build.zig @@ -48,7 +48,6 @@ pub fn build(b: *std.Build) void { .include_path = "stack.h", }, .{ - .AT = "@", .UNDERSCORE = "_", .NEST_UNDERSCORE_PROXY = "UNDERSCORE", .NEST_PROXY = "NEST_UNDERSCORE_PROXY", -- 2.54.0 From 6d5fbb26dafae1f1d293e968a0183ae08a4e3992 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 00:46:05 -0700 Subject: [PATCH 139/179] Maker: enhance debuggability when resolveLazyPath fails --- lib/compiler/Maker.zig | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index b1b106969be24639dd23bc03b0fff317725dc094..f7bbe5686bf2e01f49557d5b17cb608eddd1bc4f 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1702,8 +1702,7 @@ pub fn resolveLazyPath( arena: Allocator, lazy_path: Configuration.LazyPath, asking_step_index: Configuration.Step.Index, -) Allocator.Error!Path { - _ = asking_step_index; // TODO use this to enhance debugability when this function fails +) error{ OutOfMemory, MakeFailed }!Path { const c = &maker.scanned_config.configuration; return switch (lazy_path) { .source_path => |sp| try packagePath(maker, arena, sp.owner, sp.sub_path.slice(c)), @@ -1712,8 +1711,10 @@ pub fn resolveLazyPath( const base = generatedPath(maker, gen.index); var file_path = base; for (0..gen.flags.up) |_| { - file_path.sub_path = Dir.path.dirname(file_path.sub_path) orelse - fatal("invalid LazyPath traversal: up {d} times from {f}", .{ gen.flags.up, base }); + file_path.sub_path = Dir.path.dirname(file_path.sub_path) orelse { + const s = stepByIndex(maker, asking_step_index); + return s.fail(maker, "invalid LazyPath traversal: up {d} times from {f}", .{ gen.flags.up, base }); + }; } return file_path.join(arena, gen.sub_path.slice(c)); }, @@ -1725,7 +1726,7 @@ pub fn resolveLazyPathIndex( arena: Allocator, lazy_path_index: Configuration.LazyPath.Index, asking_step_index: Configuration.Step.Index, -) Allocator.Error!Path { +) error{ OutOfMemory, MakeFailed }!Path { const c = &maker.scanned_config.configuration; return resolveLazyPath(maker, arena, lazy_path_index.get(c), asking_step_index); } @@ -1737,7 +1738,7 @@ pub fn resolveLazyPathAbs( arena: Allocator, lazy_path: Configuration.LazyPath, asking_step_index: Configuration.Step.Index, -) Allocator.Error![]const u8 { +) error{ OutOfMemory, MakeFailed }![]const u8 { const p = try resolveLazyPath(maker, arena, lazy_path, asking_step_index); const root_dir_path = p.root_dir.path orelse return p.subPathOrDot(); if (p.sub_path.len == 0) return root_dir_path; @@ -1751,7 +1752,7 @@ pub fn resolveLazyPathIndexAbs( arena: Allocator, lazy_path_index: Configuration.LazyPath.Index, asking_step_index: Configuration.Step.Index, -) Allocator.Error![]const u8 { +) error{ OutOfMemory, MakeFailed }![]const u8 { const c = &maker.scanned_config.configuration; return resolveLazyPathAbs(maker, arena, lazy_path_index.get(c), asking_step_index); } -- 2.54.0 From 2edeeb5a6476cc996367e547eb2c86e878784bf1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 00:55:21 -0700 Subject: [PATCH 140/179] zig build: add CLI usage for --print-configuration --- lib/compiler/Maker/ScannedConfig.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 748cc84e97fc52872056cc32102ecf382ca97038..379b8b209c83243a20769383f596e1f964b67e99 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -349,6 +349,7 @@ pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { \\ poisoned Don't cache the configuration \\ disallowed Panics when cache would be poisoned \\ ignored A little poison never hurt anybody + \\ --print-configuration Render configuration as .zon to stdout \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) -- 2.54.0 From 8b4e55372a51bd2d6869cd3cc1e90939de3b568f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 12:13:16 -0700 Subject: [PATCH 141/179] fix incr-check tests --- test/tests.zig | 2 +- tools/incr-check.zig | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/tests.zig b/test/tests.zig index d7b74073902989f873ac038ac1ca0f6be4968e18..b88dc8e3c57edb1d09497e587dd17f0823e218c4 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2970,7 +2970,7 @@ pub fn addIncrementalTests(b: *std.Build, test_step: *Step, test_filters: []cons run.addFileArg(b.path("test/incremental/").path(b, entry.path)); run.addArg("--zig-lib-dir"); - run.addFileArg(.zig_lib); + run.addDirectoryArg(.zig_lib); run.addArgs(&.{ "--target", target_str }); diff --git a/tools/incr-check.zig b/tools/incr-check.zig index 75906424b18cbfc7e9258906d38b7ac54032c026..89c14ce1e7f60a710e863349025d3803f2dc37bb 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -360,7 +360,10 @@ const Eval = struct { const bin_name = try std.zig.EmitArtifact.bin.cacheName(arena, .{ .root_name = "root", // corresponds to the module name "root" - .target = &eval.target, + .cpu_arch = eval.target.cpu.arch, + .os_tag = eval.target.os.tag, + .ofmt = eval.target.ofmt, + .abi = eval.target.abi, .output_mode = .Exe, }); const bin_path = try Dir.path.join(arena, &.{ result_dir, bin_name }); @@ -487,9 +490,12 @@ const Eval = struct { var argv_buf: [2][]const u8 = undefined; const argv: []const []const u8, const is_foreign: bool = sw: switch (std.zig.system.getExternalExecutor( io, - &eval.host, &eval.target, - .{ .link_libc = eval.backend == .cbe }, + .{ + .link_libc = eval.backend == .cbe, + .host_cpu_arch = eval.host.cpu.arch, + .host_os_tag = eval.host.os.tag, + }, )) { .bad_dl, .bad_os_or_cpu => { // This binary cannot be executed on this host. -- 2.54.0 From fa7433a924794b0e40e522680710c60859c65db1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 12:15:00 -0700 Subject: [PATCH 142/179] fix std-docs.zig compilation --- lib/compiler/std-docs.zig | 53 ++++++++++++--------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/lib/compiler/std-docs.zig b/lib/compiler/std-docs.zig index 213c0b826e2e6aeac2f2fcf4fcd6c032ed22a61d..db2d22a06f02b999e408b0609635277d95284a70 100644 --- a/lib/compiler/std-docs.zig +++ b/lib/compiler/std-docs.zig @@ -271,12 +271,16 @@ fn serveWasm( // Do the compilation every request, so that the user can edit the files // and see the changes without restarting the server. const wasm_base_path = try buildWasmBinary(arena, context, optimize_mode); + const target = std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ + .arch_os_abi = autodoc_arch_os_abi, + .cpu_features = autodoc_cpu_features, + }) catch unreachable) catch unreachable; const bin_name = try std.zig.binNameAlloc(arena, .{ .root_name = autodoc_root_name, - .target = &(std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ - .arch_os_abi = autodoc_arch_os_abi, - .cpu_features = autodoc_cpu_features, - }) catch unreachable) catch unreachable), + .cpu_arch = target.cpu.arch, + .os_tag = target.os.tag, + .ofmt = target.ofmt, + .abi = target.abi, .output_mode = .Exe, }); // std.http.Server does not have a sendfile API yet. @@ -406,51 +410,26 @@ fn buildWasmBinary( child.stdin.?.close(io); child.stdin = null; - switch (try child.wait(io)) { - .exited => |code| { - if (code != 0) { - std.log.err( - "the following command exited with error code {d}:\n{s}", - .{ code, try std.Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, - ); - return error.WasmCompilationFailed; - } - }, - .signal => |sig| { - std.log.err( - "the following command terminated with signal {t}:\n{s}", - .{ sig, try std.Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, - ); - return error.WasmCompilationFailed; - }, - .stopped => |sig| { - std.log.err( - "the following command stopped unexpectedly with signal {t}:\n{s}", - .{ sig, try std.Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, - ); - return error.WasmCompilationFailed; - }, - .unknown => { - std.log.err( - "the following command terminated unexpectedly:\n{s}", - .{try std.Build.Step.allocPrintCmd(arena, .inherit, null, argv.items)}, - ); - return error.WasmCompilationFailed; - }, + const term = try child.wait(io); + if (!term.success()) { + std.log.err("the following command {f}:\n{s}", .{ + term, try std.zig.allocPrintCmd(arena, argv.items, .{}), + }); + return error.WasmCompilationFailed; } if (result_error_bundle.errorMessageCount() > 0) { try result_error_bundle.renderToStderr(io, .{}, .auto); std.log.err("the following command failed with {d} compilation errors:\n{s}", .{ result_error_bundle.errorMessageCount(), - try std.Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), + try std.zig.allocPrintCmd(arena, argv.items, .{}), }); return error.WasmCompilationFailed; } return result orelse { std.log.err("child process failed to report result\n{s}", .{ - try std.Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), + try std.zig.allocPrintCmd(arena, argv.items, .{}), }); return error.WasmCompilationFailed; }; -- 2.54.0 From e7546f8dfd99e54832c0c68c53f3f03caeaafcc7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 12:16:42 -0700 Subject: [PATCH 143/179] CI: --debug-maker in ci/x86_64-linux-debug-llvm.sh in case any build system bugs are introduced this will make it quicker to find the cause. --- ci/x86_64-linux-debug-llvm.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/x86_64-linux-debug-llvm.sh b/ci/x86_64-linux-debug-llvm.sh index 750243130a3619998ed4ca5dc5fe81043fb251f7..ed24f7e460eb4f1a662f7d0d3ab4e6d6fe5e3c38 100755 --- a/ci/x86_64-linux-debug-llvm.sh +++ b/ci/x86_64-linux-debug-llvm.sh @@ -49,6 +49,7 @@ stage3-debug/bin/zig build \ -Dno-lib stage3-debug/bin/zig build test docs \ + --debug-maker \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dlldb=$HOME/deps/lldb-zig/Debug-e0a42bb34/bin/lldb \ -Dlibc-test-path=$HOME/deps/libc-test-f2bac77 \ -- 2.54.0 From 3e75a3e36b75cc8b85b4e7b1e825a11b0f1e8de1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 12:18:43 -0700 Subject: [PATCH 144/179] Configuration: add new target info --- lib/std/Build/Configuration.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 5040a5f619f8ca868b648a99cc33c34d25e7b9ed..2ac253c7a9b52682fe50ddc60d13747f169b95cc 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -2265,6 +2265,7 @@ pub const TargetQuery = struct { simulator, ohos, ohoseabi, + call0, default, @@ -2297,6 +2298,7 @@ pub const TargetQuery = struct { .simulator => .simulator, .ohos => .ohos, .ohoseabi => .ohoseabi, + .call0 => .call0, }; } @@ -2329,6 +2331,7 @@ pub const TargetQuery = struct { .simulator => .simulator, .ohos => .ohos, .ohoseabi => .ohoseabi, + .call0 => .call0, .default => null, }; } @@ -2357,6 +2360,7 @@ pub const TargetQuery = struct { loongarch32, loongarch64, m68k, + m88k, microblaze, microblazeel, mips, @@ -2421,6 +2425,7 @@ pub const TargetQuery = struct { .loongarch32 => .loongarch32, .loongarch64 => .loongarch64, .m68k => .m68k, + .m88k => .m88k, .microblaze => .microblaze, .microblazeel => .microblazeel, .mips => .mips, @@ -2485,6 +2490,7 @@ pub const TargetQuery = struct { .loongarch32 => .loongarch32, .loongarch64 => .loongarch64, .m68k => .m68k, + .m88k => .m88k, .microblaze => .microblaze, .microblazeel => .microblazeel, .mips => .mips, -- 2.54.0 From 05fbeb4ea7acef3e9cfcce650343b1ef73e12e29 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 13:11:50 -0700 Subject: [PATCH 145/179] Maker.Step.Run: fix wrong allocator used --- lib/compiler/Maker/Step/Run.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index c0bb82cdef3799653b45090045f2c3d6105a5da6..f5437180a9123824b8ad535c1691740f5eb5196c 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -2106,6 +2106,7 @@ fn spawnChildAndCollect( const graph = maker.graph; const io = graph.io; const arena = graph.arena; // TODO don't leak into process arena + const gpa = maker.gpa; const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); const conf_run = conf_step.extended.get(conf.extra).run; @@ -2122,7 +2123,7 @@ fn spawnChildAndCollect( // If an error occurs, it's caused by this command: assert(step.result_failed_command == null); - step.result_failed_command = try std.zig.allocPrintCmd(arena, argv, .{ + step.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{ .cwd = child_cwd, .child_env = environ_map, .parent_env = &graph.environ_map, -- 2.54.0 From b069a2eb21d4bdcbc87c566be013fa30f26d3a1e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 14:10:40 -0700 Subject: [PATCH 146/179] Maker: update macos file watching code to new api --- lib/compiler/Maker.zig | 2 +- lib/compiler/Maker/Watch.zig | 35 +++++++++-------- lib/compiler/Maker/Watch/FsEvents.zig | 55 ++++++++++++++++----------- 3 files changed, 52 insertions(+), 40 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index f7bbe5686bf2e01f49557d5b17cb608eddd1bc4f..5e5cb96df710f6276e26d8d6a25fc30709b82f2a 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -623,7 +623,7 @@ pub fn main(init: process.Init.Minimal) !void { // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. if (!Watch.have_impl) unreachable; - try w.update(gpa, maker.step_stack.keys()); + try w.update(maker.step_stack.keys()); // Wait until a file system notification arrives. Read all such events // until the buffer is empty. Then wait for a debounce interval, resetting diff --git a/lib/compiler/Maker/Watch.zig b/lib/compiler/Maker/Watch.zig index b59f616d995c1b7c77893714cdf441563ea7c1c6..99502df188bb14e651c15ed80deb80d9c6eb3dd8 100644 --- a/lib/compiler/Maker/Watch.zig +++ b/lib/compiler/Maker/Watch.zig @@ -177,8 +177,9 @@ const Os = switch (builtin.os.tag) { } } - fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + fn update(w: *Watch, steps: []const Configuration.Step.Index) !void { const maker = w.maker; + const gpa = maker.gpa; // Add missing marks and note persisted ones. for (steps) |step_index| { @@ -465,8 +466,7 @@ const Os = switch (builtin.os.tag) { }; }; - fn init(cwd_path: []const u8) !Watch { - _ = cwd_path; + fn init(maker: *Maker) !Watch { return .{ .dir_table = .{}, .dir_count = 0, @@ -478,6 +478,7 @@ const Os = switch (builtin.os.tag) { else => {}, }, .generation = 0, + .maker = maker, }; } @@ -546,7 +547,8 @@ const Os = switch (builtin.os.tag) { return any_dirty; } - fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + fn update(w: *Watch, steps: []const Configuration.Step.Index) !void { + const gpa = w.maker.gpa; // Add missing marks and note persisted ones. for (steps) |step| { for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { @@ -677,8 +679,7 @@ const Os = switch (builtin.os.tag) { const EV = std.c.EV; const NOTE = std.c.NOTE; - fn init(cwd_path: []const u8) !Watch { - _ = cwd_path; + fn init(maker: *Maker) !Watch { return .{ .dir_table = .{}, .dir_count = 0, @@ -687,10 +688,12 @@ const Os = switch (builtin.os.tag) { .handles = .empty, }, .generation = 0, + .maker = maker, }; } - fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + fn update(w: *Watch, steps: []const Configuration.Step.Index) !void { + const gpa = w.maker.gpa; const handles = &w.os.handles; for (steps) |step| { for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { @@ -860,21 +863,21 @@ const Os = switch (builtin.os.tag) { .macos => struct { fse: FsEvents, - fn init(cwd_path: []const u8) !Watch { + fn init(maker: *Maker) !Watch { return .{ - .os = .{ .fse = try .init(cwd_path) }, + .os = .{ .fse = try .init(maker.graph.cache.cwd) }, .dir_count = 0, .dir_table = undefined, .generation = undefined, + .maker = maker, }; } - fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { - try w.os.fse.setPaths(gpa, steps); + fn update(w: *Watch, steps: []const Configuration.Step.Index) !void { + try w.os.fse.setPaths(w.maker, steps); w.dir_count = w.os.fse.watch_roots.len; } - fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - _ = io; - return w.os.fse.wait(gpa, switch (timeout) { + fn wait(w: *Watch, timeout: Timeout) !WaitResult { + return w.os.fse.wait(w.maker, switch (timeout) { .none => null, .ms => |ms| @as(u64, ms) * std.time.ns_per_ms, }); @@ -938,8 +941,8 @@ fn markStepSetDirty(maker: *Maker, step_set: *StepSet, any_dirty: bool) bool { return any_dirty or this_any_dirty; } -pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { - return Os.update(w, gpa, steps); +pub fn update(w: *Watch, steps: []const Configuration.Step.Index) !void { + return Os.update(w, steps); } pub const Timeout = union(enum) { diff --git a/lib/compiler/Maker/Watch/FsEvents.zig b/lib/compiler/Maker/Watch/FsEvents.zig index 0a56ce182255176222ef4b9a7a7bbb22abc9350d..04833b2644c9cc53252a58d5e1098d9d29e7438a 100644 --- a/lib/compiler/Maker/Watch/FsEvents.zig +++ b/lib/compiler/Maker/Watch/FsEvents.zig @@ -17,6 +17,7 @@ //! the logic that would avoid them is currently disabled, because the build system kind //! of relies on them at the time of writing to avoid redundant work -- see the comment at //! the top of `wait` for details. +const FsEvents = @This(); const enable_debug_logs = false; @@ -30,7 +31,7 @@ paths_arena: std.heap.ArenaAllocator.State, watch_roots: [][:0]const u8, /// All of the paths being watched. Value is the set of steps which depend on the file/directory. /// Keys and values are in `paths_arena`, but this map is allocated into the GPA. -watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step), +watch_paths: std.array_hash_map.String([]const std.Build.Configuration.Step.Index), /// The semaphore we use to block the thread calling `wait` until the callback determines a relevant /// event has occurred. This is retained across `wait` calls for simplicity and efficiency. @@ -118,19 +119,22 @@ pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void { } } -pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) !void { +pub fn setPaths(fse: *FsEvents, maker: *Maker, steps: []const std.Build.Configuration.Step.Index) !void { + const gpa = maker.gpa; + var paths_arena_instance = fse.paths_arena.promote(gpa); defer fse.paths_arena = paths_arena_instance.state; const paths_arena = paths_arena_instance.allocator(); - var need_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; + var need_dirs: std.array_hash_map.String(void) = .empty; defer need_dirs.deinit(gpa); fse.watch_paths.clearRetainingCapacity(); - // We take `step` by pointer for a slight memory optimization in a moment. - for (steps) |*step| { - for (step.*.inputs.table.keys(), step.*.inputs.table.values()) |path, *files| { + // We take `step_index` by pointer for a slight memory optimization in a moment. + for (steps) |*step_index| { + const step = maker.stepByIndex(step_index.*); + for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { const resolved_dir = try std.fs.path.resolvePosix(paths_arena, &.{ fse.cwd_path, path.root_dir.path orelse ".", path.sub_path, }); @@ -143,14 +147,14 @@ pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) const gop = try fse.watch_paths.getOrPut(gpa, watch_path); if (gop.found_existing) { const old_steps = gop.value_ptr.*; - const new_steps = try paths_arena.alloc(*std.Build.Step, old_steps.len + 1); + const new_steps = try paths_arena.alloc(std.Build.Configuration.Step.Index, old_steps.len + 1); @memcpy(new_steps[0..old_steps.len], old_steps); - new_steps[old_steps.len] = step.*; + new_steps[old_steps.len] = step_index.*; gop.value_ptr.* = new_steps; } else { // This is why we captured `step` by pointer! We can avoid allocating a slice of one // step in the arena in the common case where a file is referenced by only one step. - gop.value_ptr.* = step[0..1]; + gop.value_ptr.* = step_index[0..1]; } } } @@ -206,8 +210,9 @@ pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) } } -pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory, StartFailed }!std.Build.Watch.WaitResult { +pub fn wait(fse: *FsEvents, maker: *Maker, timeout_ns: ?u64) error{ OutOfMemory, StartFailed }!Watch.WaitResult { if (fse.watch_roots.len == 0) @panic("nothing to watch"); + const gpa = maker.gpa; const rs = fse.resolved_symbols; @@ -253,7 +258,7 @@ pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory const callback_ctx: EventCallbackCtx = .{ .fse = fse, - .gpa = gpa, + .maker = maker, }; const event_stream = rs.FSEventStreamCreate( null, @@ -321,7 +326,7 @@ const cf_alloc_callbacks = struct { const EventCallbackCtx = struct { fse: *FsEvents, - gpa: Allocator, + maker: *Maker, }; fn eventCallback( @@ -333,8 +338,8 @@ fn eventCallback( events_ids_ptr: [*]const FSEventStreamEventId, ) callconv(.c) void { const ctx: *const EventCallbackCtx = @ptrCast(@alignCast(client_callback_info)); + const maker = ctx.maker; const fse = ctx.fse; - const gpa = ctx.gpa; const rs = fse.resolved_symbols; const events_paths_ptr_casted: [*]const [*:0]const u8 = @ptrCast(@alignCast(events_paths_ptr)); const events_paths = events_paths_ptr_casted[0..num_events]; @@ -349,17 +354,13 @@ fn eventCallback( false => { if (fse.watch_paths.get(event_path)) |steps| { assert(steps.len > 0); - for (steps) |s| { - if (s.invalidateResult(gpa)) any_dirty = true; - } + if (invalidateSteps(maker, steps)) any_dirty = true; } if (std.fs.path.dirname(event_path)) |event_dirname| { // Modifying '/foo/bar' triggers the watch on '/foo'. if (fse.watch_paths.get(event_dirname)) |steps| { assert(steps.len > 0); - for (steps) |s| { - if (s.invalidateResult(gpa)) any_dirty = true; - } + if (invalidateSteps(maker, steps)) any_dirty = true; } } }, @@ -372,9 +373,7 @@ fn eventCallback( const changed_path = std.fs.path.dirname(event_path) orelse event_path; for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| { if (dirStartsWith(watching_path, changed_path)) { - for (steps) |s| { - if (s.invalidateResult(gpa)) any_dirty = true; - } + if (invalidateSteps(maker, steps)) any_dirty = true; } } }, @@ -392,6 +391,15 @@ fn dirStartsWith(path: []const u8, prefix: []const u8) bool { return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar` } +fn invalidateSteps(maker: *Maker, steps: []const std.Build.Configuration.Step.Index) bool { + var any_dirty = false; + for (steps) |step_index| { + const step = maker.stepByIndex(step_index); + if (maker.invalidateResult(step)) any_dirty = true; + } + return any_dirty; +} + const CFAllocatorRef = ?*const opaque {}; const CFArrayRef = *const opaque {}; const CFStringRef = *const opaque {}; @@ -476,4 +484,5 @@ const Io = std.Io; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const watch_log = std.log.scoped(.watch); -const FsEvents = @This(); +const Maker = @import("../../Maker.zig"); +const Watch = @import("../Watch.zig"); -- 2.54.0 From fed031031dc671f961b5f1f04cffd1ea4753f3ba Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 14:16:44 -0700 Subject: [PATCH 147/179] Maker: fix compilation on windows --- lib/compiler/Maker/Step/FindProgram.zig | 2 +- lib/compiler/Maker/Watch.zig | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/compiler/Maker/Step/FindProgram.zig b/lib/compiler/Maker/Step/FindProgram.zig index d3b8d067388bd1a6ff875b27aa314aa0d764980a..b3396cc5c82f0c0484b3a992753b7a17a6c8ec30 100644 --- a/lib/compiler/Maker/Step/FindProgram.zig +++ b/lib/compiler/Maker/Step/FindProgram.zig @@ -93,7 +93,7 @@ fn checkCandidate( while (it.next()) |ext| { if (!supportedWindowsProgramExtension(ext)) continue; - const extended_path = try std.mem.concat(arena, &.{ full_path, ext }); + const extended_path = try std.mem.concat(arena, u8, &.{ full_path, ext }); if (Io.Dir.cwd().access(io, extended_path, .{ .execute = true })) |_| { maker.generatedPath(found_path).* = .initCwd(extended_path); diff --git a/lib/compiler/Maker/Watch.zig b/lib/compiler/Maker/Watch.zig index 99502df188bb14e651c15ed80deb80d9c6eb3dd8..05dace418b062c5d464758040543705b5c1f80d0 100644 --- a/lib/compiler/Maker/Watch.zig +++ b/lib/compiler/Maker/Watch.zig @@ -548,9 +548,11 @@ const Os = switch (builtin.os.tag) { } fn update(w: *Watch, steps: []const Configuration.Step.Index) !void { - const gpa = w.maker.gpa; + const maker = w.maker; + const gpa = maker.gpa; // Add missing marks and note persisted ones. - for (steps) |step| { + for (steps) |step_index| { + const step = maker.stepByIndex(step_index); for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { const dir = dir: { const gop = try w.dir_table.getOrPut(gpa, path); @@ -579,7 +581,7 @@ const Os = switch (builtin.os.tag) { for (files.items) |basename| { const gop = try dir.reaction_set.getOrPut(gpa, basename); if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step, w.generation); + try gop.value_ptr.put(gpa, step_index, w.generation); } } } -- 2.54.0 From bd84824d9f48a320572355c4e135cc54480c3e03 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 14:18:42 -0700 Subject: [PATCH 148/179] Maker: fix compilation on BSDs --- lib/compiler/Maker/Watch.zig | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/compiler/Maker/Watch.zig b/lib/compiler/Maker/Watch.zig index 05dace418b062c5d464758040543705b5c1f80d0..36eefcbaafd5810daaff591589cb54051c61964c 100644 --- a/lib/compiler/Maker/Watch.zig +++ b/lib/compiler/Maker/Watch.zig @@ -695,9 +695,11 @@ const Os = switch (builtin.os.tag) { } fn update(w: *Watch, steps: []const Configuration.Step.Index) !void { - const gpa = w.maker.gpa; + const maker = w.maker; + const gpa = maker.gpa; const handles = &w.os.handles; - for (steps) |step| { + for (steps) |step_index| { + const step = maker.stepByIndex(step_index); for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { const reaction_set = rs: { const gop = try w.dir_table.getOrPut(gpa, path); @@ -732,7 +734,7 @@ const Os = switch (builtin.os.tag) { for (files.items) |basename| { const gop = try reaction_set.getOrPut(gpa, basename); if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step, w.generation); + try gop.value_ptr.put(gpa, step_index, w.generation); } } } -- 2.54.0 From 9b6dd7ee5c4137942f7fbbdd27d8e06f6ce7c7b1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 14:30:07 -0700 Subject: [PATCH 149/179] zig build CLI: change --debug-maker to --maker-opt=[mode] the --debug- prefixed args are reserved for compiler debugging flags. --- ci/x86_64-linux-debug-llvm.sh | 2 +- lib/compiler/Maker/ScannedConfig.zig | 2 +- src/main.zig | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ci/x86_64-linux-debug-llvm.sh b/ci/x86_64-linux-debug-llvm.sh index ed24f7e460eb4f1a662f7d0d3ab4e6d6fe5e3c38..d4cbf3ce3cdd7b621e5031351257c0066d6eeedd 100755 --- a/ci/x86_64-linux-debug-llvm.sh +++ b/ci/x86_64-linux-debug-llvm.sh @@ -49,7 +49,7 @@ stage3-debug/bin/zig build \ -Dno-lib stage3-debug/bin/zig build test docs \ - --debug-maker \ + --maker-opt=Debug \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dlldb=$HOME/deps/lldb-zig/Debug-e0a42bb34/bin/lldb \ -Dlibc-test-path=$HOME/deps/libc-test-f2bac77 \ diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 379b8b209c83243a20769383f596e1f964b67e99..7bcdec5f64b25f20a43f7d36e7817989bfb57548 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -359,7 +359,7 @@ pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { \\ none (default) No build ID \\ --debug-log [scope] Enable debugging the compiler \\ --debug-pkg-config Fail if unknown pkg-config flags encountered - \\ --debug-maker[=mode] Change maker executable optimization mode + \\ --maker-opt=[mode] Change maker executable optimization mode (default: ReleaseSafe) \\ --verbose-link Enable compiler debug output for linking \\ --verbose-air Enable compiler debug output for Zig AIR \\ --verbose-llvm-ir Enable compiler debug output for LLVM IR diff --git a/src/main.zig b/src/main.zig index 5db4c1c95ff50d2f3faf1244801975c41f3dd3b4..a79ee7150f240e37bb3740e31c92843c52991da8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5116,10 +5116,7 @@ fn cmdBuild( }; } else if (mem.eql(u8, arg, "-fno-reference-trace")) { reference_trace = null; - } else if (mem.eql(u8, arg, "--debug-maker")) { - maker_optimize_mode = .Debug; - continue; - } else if (mem.cutPrefix(u8, arg, "--debug-maker=")) |rest| { + } else if (mem.cutPrefix(u8, arg, "--maker-opt=")) |rest| { maker_optimize_mode = parseOptimizeMode(rest); continue; } else if (mem.eql(u8, arg, "--debug-log")) { -- 2.54.0 From 642d017fea531d1347b8ade16c33065389a20192 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 15:34:31 -0700 Subject: [PATCH 150/179] Maker: fix resolveLazyPath accidental mutation --- lib/compiler/Maker.zig | 6 ++++-- lib/std/Build/Step/Run.zig | 26 ++++++++++++++++---------- test/standalone/dirname/build.zig | 16 ++++++---------- test/standalone/dirname/touch.zig | 19 +++++++++---------- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 5e5cb96df710f6276e26d8d6a25fc30709b82f2a..e908b7f39e5ffed1cc2348dd1b7aa2c8d4e96bcb 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1708,12 +1708,14 @@ pub fn resolveLazyPath( .source_path => |sp| try packagePath(maker, arena, sp.owner, sp.sub_path.slice(c)), .relative => |relative| relativePath(maker, relative), .generated => |gen| { - const base = generatedPath(maker, gen.index); + const base = generatedPath(maker, gen.index).*; var file_path = base; for (0..gen.flags.up) |_| { file_path.sub_path = Dir.path.dirname(file_path.sub_path) orelse { const s = stepByIndex(maker, asking_step_index); - return s.fail(maker, "invalid LazyPath traversal: up {d} times from {f}", .{ gen.flags.up, base }); + return s.fail(maker, "invalid LazyPath traversal: up {d} times from {f}", .{ + gen.flags.up, base, + }); }; } return file_path.join(arena, gen.sub_path.slice(c)); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 35699051fbfd511c399cb7033e613abebd9cf9d5..00b80ed5900425c40616b2afa783786ff7e753dc 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -242,21 +242,23 @@ pub fn addPrefixedArtifactArg(run: *Run, prefix: []const u8, artifact: *Step.Com /// Returns a `std.Build.LazyPath` which can be used as inputs to other APIs /// throughout the build system. /// +/// `sub_path` is the name of the generated output file which may have zero or +/// more path components. +/// /// Related: /// * `addPrefixedOutputFileArg` - same thing but prepends a string to the argument /// * `addFileArg` - for input files given to the child process -pub fn addOutputFileArg(run: *Run, basename: []const u8) std.Build.LazyPath { - return run.addPrefixedOutputFileArg("", basename); +pub fn addOutputFileArg(run: *Run, sub_path: []const u8) std.Build.LazyPath { + return run.addPrefixedOutputFileArg("", sub_path); } /// Provides a file path as a command line argument to the command being run. -/// Asserts `basename` is not empty. /// -/// For example, a prefix of "-o" and basename of "output.txt" will result in +/// For example, a prefix of "-o" and `sub_path` of "output.txt" will result in /// the child process seeing something like this: "-ozig-cache/.../output.txt" /// /// The child process will see a single argument, regardless of whether the -/// prefix or basename have spaces. +/// prefix or `sub_path` have spaces. /// /// The returned `std.Build.LazyPath` can be used as inputs to other APIs /// throughout the build system. @@ -267,23 +269,27 @@ pub fn addOutputFileArg(run: *Run, basename: []const u8) std.Build.LazyPath { pub fn addPrefixedOutputFileArg( run: *Run, prefix: []const u8, - basename: []const u8, + /// The name of the generated output file which may have zero or more path + /// components. + /// + /// Asserted to be non-empty. + sub_path: []const u8, ) std.Build.LazyPath { const b = run.step.owner; const graph = b.graph; const arena = graph.arena; - if (basename.len == 0) @panic("basename must not be empty"); + assert(sub_path.len != 0); - const output = arena.create(Output) catch @panic("OOM"); + const output = graph.create(Output); output.* = .{ .prefix = graph.dupeString(prefix), - .basename = graph.dupeString(basename), + .basename = graph.dupeString(sub_path), .generated_file = graph.addGeneratedFile(&run.step), }; run.argv.append(arena, .{ .output_file = output }) catch @panic("OOM"); if (run.rename_step_with_output_arg) { - run.setName(b.fmt("{s} ({s})", .{ run.step.name, basename })); + run.setName(b.fmt("{s} ({s})", .{ run.step.name, sub_path })); } return .{ .generated = .{ .index = output.generated_file } }; diff --git a/test/standalone/dirname/build.zig b/test/standalone/dirname/build.zig index dc3be15cf2a9591958ec317aa13547979db0678c..e3f2ebd72688abec98183469663edf32b3245967 100644 --- a/test/standalone/dirname/build.zig +++ b/test/standalone/dirname/build.zig @@ -27,22 +27,16 @@ pub fn build(b: *std.Build) void { }), }); - // Known path: - addTestRun(test_step, exists_in, touch_src.dirname(), &.{"touch.zig"}); - - // Generated file: - addTestRun(test_step, exists_in, generated.dirname(), &.{"generated.txt"}); - - // Generated file multiple levels: - addTestRun(test_step, exists_in, generated.dirname().dirname(), &.{ + addTestRun(test_step, exists_in, "run exists_in (known path)", touch_src.dirname(), &.{"touch.zig"}); + addTestRun(test_step, exists_in, "run exists_in (generated file)", generated.dirname(), &.{"generated.txt"}); + addTestRun(test_step, exists_in, "run exists_in (generated file multi level)", generated.dirname().dirname(), &.{ "subdir" ++ std.fs.path.sep_str ++ "generated.txt", }); - // Absolute path: const write_files = b.addWriteFiles(); _ = write_files.add("foo.txt", ""); const abs_path = write_files.getDirectory(); - addTestRun(test_step, exists_in, abs_path, &.{"foo.txt"}); + addTestRun(test_step, exists_in, "run exists_in (absolute path)", abs_path, &.{"foo.txt"}); } // Runs exe with the parameters [dirname, args...]. @@ -50,10 +44,12 @@ pub fn build(b: *std.Build) void { fn addTestRun( test_step: *std.Build.Step, exe: *std.Build.Step.Compile, + step_name: []const u8, dirname: std.Build.LazyPath, args: []const []const u8, ) void { const run = test_step.owner.addRunArtifact(exe); + run.setName(step_name); run.addDirectoryArg(dirname); run.addArgs(args); run.expectExitCode(0); diff --git a/test/standalone/dirname/touch.zig b/test/standalone/dirname/touch.zig index 86aef059362b3e088f58a813a2e0de8043e6ac5e..85563cc5293f50469fb1153986f9f382fd10dfb3 100644 --- a/test/standalone/dirname/touch.zig +++ b/test/standalone/dirname/touch.zig @@ -7,27 +7,26 @@ //! Path must be absolute. const std = @import("std"); +const Io = std.Io; pub fn main(init: std.process.Init) !void { + const io = init.io; + var args = try init.minimal.args.iterateAllocator(init.gpa); defer args.deinit(); - _ = args.next() orelse unreachable; // skip binary name + _ = args.next().?; // skip binary name const path = args.next() orelse { std.log.err("missing argument", .{}); return error.BadUsage; }; - const dir_path = std.Io.Dir.path.dirname(path) orelse unreachable; - const basename = std.Io.Dir.path.basename(path); + const dir_path = Io.Dir.path.dirname(path).?; + const basename = Io.Dir.path.basename(path); - const io = std.Io.Threaded.global_single_threaded.io(); - - var dir = try std.Io.Dir.cwd().openDir(io, dir_path, .{}); + var dir = try Io.Dir.cwd().openDir(io, dir_path, .{}); defer dir.close(io); - _ = dir.statFile(io, basename, .{}) catch { - var file = try dir.createFile(io, basename, .{}); - file.close(io); - }; + var file = try dir.createFile(io, basename, .{ .truncate = false }); + file.close(io); } -- 2.54.0 From 4ba2bcbbec68cfc78f1dc0c2031188519bc37a68 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 16:05:41 -0700 Subject: [PATCH 151/179] std.Build.Configuration: fix loading on big endian --- lib/std/Build/Configuration.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 2ac253c7a9b52682fe50ddc60d13747f169b95cc..5041fbe4eff7c93e0b63f8eb5f948e368c534782 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -3360,7 +3360,7 @@ pub fn loadFile(arena: Allocator, io: Io, file: Io.File) LoadFileError!Configura pub const LoadError = Io.Reader.Error || Allocator.Error; pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { - const header = try reader.takeStruct(Header, .little); + const header = try reader.takeStruct(Header, .native); const result: Configuration = .{ .string_bytes = try arena.alloc(u8, header.string_bytes_len), .steps = try arena.alloc(Step, header.steps_len), -- 2.54.0 From 626e4104131bd557ec0a3148a7c02a7125605b75 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 16:49:38 -0700 Subject: [PATCH 152/179] Maker: fix regressed dyn lib symlink logic --- lib/compiler/Maker.zig | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index e908b7f39e5ffed1cc2348dd1b7aa2c8d4e96bcb..c364af3c8bffc1288245286ef91043e14cb24182 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1998,17 +1998,19 @@ fn installSymLinksInner( ) !void { const io = maker.graph.io; const step = stepByIndex(maker, asking_step_index); + const out_basename = Io.Dir.path.basename(output_path.sub_path); + const out_dir = output_path.dirname().?; - // sym link for libfoo.so.1 to libfoo.so.1.2.3 const major_only_path = try out_dir.join(arena, filename_major_only); - output_path.root_dir.handle.symLinkAtomic(io, output_path.sub_path, major_only_path.sub_path, .{}) catch |err| { - return step.fail(maker, "unable to symlink {f} -> {f}: {t}", .{ output_path, major_only_path, err }); - }; - // sym link for libfoo.so to libfoo.so.1 const name_only_path = try out_dir.join(arena, filename_name_only); - major_only_path.root_dir.handle.symLinkAtomic(io, major_only_path.sub_path, name_only_path.sub_path, .{}) catch |err| { - return step.fail(maker, "unable to symlink {f} -> {s}: {t}", .{ name_only_path, filename_major_only, err }); - }; + + // libfoo.so.1 to libfoo.so.1.2.3 + major_only_path.root_dir.handle.symLinkAtomic(io, out_basename, major_only_path.sub_path, .{}) catch |err| + return step.fail(maker, "failed symlinking {f} to {s}: {t}", .{ output_path, out_basename, err }); + + // libfoo.so to libfoo.so.1 + name_only_path.root_dir.handle.symLinkAtomic(io, filename_major_only, name_only_path.sub_path, .{}) catch |err| + return step.fail(maker, "failed symlinking {f} to {s}: {t}", .{ name_only_path, filename_major_only, err }); } fn cleanExit(io: Io, scanned_config: *const ScannedConfig) void { -- 2.54.0 From ed492ff51c7867b04b3f2a1512df7642a5debe60 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 17:20:34 -0700 Subject: [PATCH 153/179] Maker.Step.WriteFile: fix not creating dir entries --- lib/compiler/Maker/Step/WriteFile.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index 4be41d526b16d2cff25ef40fbc825a5f2f3724ed..ca14f46fada5fd1106e7ad53d552c521ab36bd79 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -260,7 +260,7 @@ fn operate( dest_path.root_dir.handle, dest_path.sub_path, io, - .{}, + .{ .make_path = true }, // Directory entry may be filtered out above. ) catch |err| return step.fail(maker, "failed copying file from {f} to {f}: {t}", .{ src_entry_path, dest_path, err, }); -- 2.54.0 From 0aa613d9bfbc3904cbd62892bb4119f9104e7b67 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 20:03:32 -0700 Subject: [PATCH 154/179] Maker.Step.Run: fix passing -L to qemu --- lib/compiler/Maker/Step/Run.zig | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index f5437180a9123824b8ad535c1691740f5eb5196c..c76d6626eed8af0281763a6b3783ee9afe3ba489 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -1811,7 +1811,11 @@ fn runCommand( }; const need_cross_libc = link_libc and root_target.os.tag == .linux and - producer.flags2.linkage == .dynamic; + switch (producer.flags2.linkage) { + .static => false, + .dynamic => true, + .default => root_target.isGnuLibC(), + }; switch (std.zig.system.getExternalExecutor(io, &root_target, .{ .host_cpu_arch = host.cpu.arch, .host_os_tag = host.os.tag, -- 2.54.0 From 9a788dccf8bea7ebdd4e31e81496ccdf3e8a734a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 20:16:17 -0700 Subject: [PATCH 155/179] std.Build.lazyImport: fix compilation errors --- lib/std/Build.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 27c151fcacde135da78be692575219082df5ad17..3b31500f2cda7ce5c7d896dce507cd507f513895 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -2037,6 +2037,7 @@ fn findPkgHashOrFatal(b: *Build, name: []const u8) []const u8 { inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, comptime dep_name: []const u8) []const u8 { const build_runner = @import("root"); const deps = build_runner.dependencies; + const arena = b.graph.arena; const b_pkg_hash, const b_pkg_deps = comptime for (@typeInfo(deps.packages).@"struct".decls) |decl| { const pkg_hash = decl.name; @@ -2044,7 +2045,7 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c if (@hasDecl(pkg, "build_zig") and pkg.build_zig == asking_build_zig) break .{ pkg_hash, pkg.deps }; } else .{ "", deps.root_deps }; if (!std.mem.eql(u8, b_pkg_hash, b.pkg_hash)) { - const build_zig_path = b.root.join("build.zig") catch @panic("OOM"); + const build_zig_path = b.root.join(arena, "build.zig") catch @panic("OOM"); panic("{} is not the struct that corresponds to {f}", .{ asking_build_zig, build_zig_path, }); @@ -2053,7 +2054,7 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c if (std.mem.eql(u8, dep[0], dep_name)) return dep[1]; }; - const full_path = b.root.join("build.zig.zon") catch @panic("OOM"); + const full_path = b.root.join(arena, "build.zig.zon") catch @panic("OOM"); panic("no dependency named {s} in {f}. All packages used in build.zig must be declared in this file", .{ dep_name, full_path, }); -- 2.54.0 From 1c8d50e0624c7099e82a35c78b135c59e2dbeaa3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 20:25:08 -0700 Subject: [PATCH 156/179] ci: remove redundant comment it says it right there in the echo text afterwards --- ci/aarch64-freebsd-release.sh | 1 - ci/aarch64-linux-release.sh | 1 - ci/aarch64-macos-release.sh | 1 - ci/aarch64-netbsd-release.sh | 1 - ci/loongarch64-linux-release.sh | 1 - ci/powerpc64le-linux-release.sh | 1 - ci/s390x-linux-release.sh | 1 - ci/x86_64-freebsd-release.sh | 1 - ci/x86_64-linux-release.sh | 1 - ci/x86_64-netbsd-release.sh | 1 - ci/x86_64-openbsd-release.sh | 1 - 11 files changed, 11 deletions(-) diff --git a/ci/aarch64-freebsd-release.sh b/ci/aarch64-freebsd-release.sh index 456919a851245a6520313e3bb7f06a4917be5960..9d99e6e515911dfb7d9756d32ce08a0d1d1201f7 100755 --- a/ci/aarch64-freebsd-release.sh +++ b/ci/aarch64-freebsd-release.sh @@ -59,7 +59,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/aarch64-linux-release.sh b/ci/aarch64-linux-release.sh index bcdeade1179fb16fe4f52b9b71114b6eb89419bf..ae5203195d5a17d6fdd4b9970b3802c3d4115ca1 100755 --- a/ci/aarch64-linux-release.sh +++ b/ci/aarch64-linux-release.sh @@ -64,7 +64,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/aarch64-macos-release.sh b/ci/aarch64-macos-release.sh index f358713326c766bcbcbd77583933e6683f1a0fee..5f2012d268fd097215113c760320092aba529ad0 100755 --- a/ci/aarch64-macos-release.sh +++ b/ci/aarch64-macos-release.sh @@ -73,7 +73,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/aarch64-netbsd-release.sh b/ci/aarch64-netbsd-release.sh index cb7594b14594623b98377ef88c291e7dfde98d0d..db59d529e928f0059f664bd3557cde654d803f92 100755 --- a/ci/aarch64-netbsd-release.sh +++ b/ci/aarch64-netbsd-release.sh @@ -59,7 +59,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/loongarch64-linux-release.sh b/ci/loongarch64-linux-release.sh index bb7fdd1eb9177be3d314acacea12b06f44c3d8d8..0343259929fb06d39cfeda396c326f9b0cabbba9 100755 --- a/ci/loongarch64-linux-release.sh +++ b/ci/loongarch64-linux-release.sh @@ -61,7 +61,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/powerpc64le-linux-release.sh b/ci/powerpc64le-linux-release.sh index b1398bd630b36fbe66a71227abcd7de9836526f2..25150408a4ef2161eb171371796b229a5f768ed6 100755 --- a/ci/powerpc64le-linux-release.sh +++ b/ci/powerpc64le-linux-release.sh @@ -63,7 +63,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/s390x-linux-release.sh b/ci/s390x-linux-release.sh index 17f8cd33393a0f29a14863424e6c88bfab6fefe2..db70925e67b6afea80b483821e0a9a47d011f4a0 100755 --- a/ci/s390x-linux-release.sh +++ b/ci/s390x-linux-release.sh @@ -62,7 +62,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/x86_64-freebsd-release.sh b/ci/x86_64-freebsd-release.sh index 103fa5b3e06993f17013f89dd0038e795202a492..f3d50ee920e1ff5ba8cf9a128b1c4a8f32e5df0d 100755 --- a/ci/x86_64-freebsd-release.sh +++ b/ci/x86_64-freebsd-release.sh @@ -69,7 +69,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/x86_64-linux-release.sh b/ci/x86_64-linux-release.sh index 9a614001fba367a3aee8e5e995480d2e476c749e..d2627598e7cb30fba1241408c3cee4627393539e 100755 --- a/ci/x86_64-linux-release.sh +++ b/ci/x86_64-linux-release.sh @@ -85,7 +85,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/x86_64-netbsd-release.sh b/ci/x86_64-netbsd-release.sh index c9157219b4a501964c7ff55ed55228f5e7470211..2a3dde40bd6f28f4c2ccec953307a29178d79d9d 100755 --- a/ci/x86_64-netbsd-release.sh +++ b/ci/x86_64-netbsd-release.sh @@ -63,7 +63,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig diff --git a/ci/x86_64-openbsd-release.sh b/ci/x86_64-openbsd-release.sh index 504c4a143b4496d25ef33438f532bbbfbcea7387..ea9c44df11318118c0dbd7cc1a01838adc2a0116 100755 --- a/ci/x86_64-openbsd-release.sh +++ b/ci/x86_64-openbsd-release.sh @@ -64,7 +64,6 @@ stage3-release/bin/zig build \ -Duse-zig-libcxx \ -Dversion-string="$(stage3-release/bin/zig version)" -# diff returns an error code if the files differ. echo "If the following command fails, it means nondeterminism has been" echo "introduced, making stage3 and stage4 no longer byte-for-byte identical." diff stage3-release/bin/zig stage4-release/bin/zig -- 2.54.0 From c0504a8fa87f66c5ddb2c02c35c10c945f4bb8b9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2026 21:46:06 -0700 Subject: [PATCH 157/179] fuzzer: get it working again --- lib/compiler/Maker.zig | 2 + lib/compiler/Maker/Fuzz.zig | 187 ++++++++++++++++++---------- lib/compiler/Maker/Graph.zig | 1 + lib/compiler/Maker/Step/Compile.zig | 15 +-- lib/compiler/Maker/Step/Run.zig | 16 ++- 5 files changed, 146 insertions(+), 75 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index c364af3c8bffc1288245286ef91043e14cb24182..0547d5fa31043cfbe1ec8f8cfd65434684fda6b9 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -317,6 +317,7 @@ pub fn main(init: process.Init.Minimal) !void { if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; } else if (mem.eql(u8, arg, "--fuzz")) { fuzz = .{ .forever = undefined }; + graph.fuzzing = true; if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; } else if (mem.startsWith(u8, arg, "--fuzz=")) { const value = arg["--fuzz=".len..]; @@ -352,6 +353,7 @@ pub fn main(init: process.Init.Minimal) !void { .amount = normalized_amount, }, }; + graph.fuzzing = true; } else if (mem.eql(u8, arg, "-fincremental")) { graph.incremental = true; } else if (mem.eql(u8, arg, "-fno-incremental")) { diff --git a/lib/compiler/Maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig index def3b923150cb16240ef7041840b41265af29c1a..2160befc08df2fb7f896d0f37b522e838cf7d331 100644 --- a/lib/compiler/Maker/Fuzz.zig +++ b/lib/compiler/Maker/Fuzz.zig @@ -83,6 +83,7 @@ pub fn init( const graph = maker.graph; const gpa = graph.cache.gpa; const io = graph.io; + const conf = &maker.scanned_config.configuration; const run_steps: []const Configuration.Step.Index = steps: { var steps: std.ArrayList(Configuration.Step.Index) = .empty; @@ -92,13 +93,13 @@ pub fn init( var rebuild_group: Io.Group = .init; defer rebuild_group.cancel(io); - for (all_steps) |step| { - if (true) @panic("TODO update the fuzzer"); - const run = step.cast(std.Build.Step.Run) orelse continue; - if (run.producer == null) continue; + for (all_steps) |step_index| { + const conf_run = step_index.ptr(conf).extended.cast(conf, Configuration.Step.Run) orelse continue; + if (conf_run.producer.value == null) continue; + const run = &maker.stepByIndex(step_index).extended.run; if (run.fuzz_tests.items.len == 0) continue; - try steps.append(gpa, run); - rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node }); + try steps.append(gpa, step_index); + rebuild_group.async(io, rebuildTestsWorkerRun, .{ maker, step_index, rebuild_node }); } if (steps.items.len == 0) fatal("no fuzz tests found", .{}); @@ -109,10 +110,10 @@ pub fn init( }; errdefer gpa.free(run_steps); - for (run_steps) |run_step_index| { - if (true) @panic("TODO update the fuzzer"); - assert(run_step_index.fuzz_tests.items.len > 0); - if (run_step_index.rebuilt_executable == null) + for (run_steps) |run_index| { + const run = &maker.stepByIndex(run_index).extended.run; + assert(run.fuzz_tests.items.len > 0); + if (run.rebuilt_executable == null) fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); } @@ -144,11 +145,10 @@ pub fn start(fuzz: *Fuzz) void { fatal("unable to spawn coverage task: {t}", .{err}); } - if (true) @panic("TODO update the fuzzer"); - - for (fuzz.run_steps) |run| { + for (fuzz.run_steps) |run_index| { + const run = &maker.stepByIndex(run_index).extended.run; assert(run.rebuilt_executable != null); - fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run }); + fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run_index }); } } @@ -163,67 +163,111 @@ pub fn deinit(fuzz: *Fuzz) void { gpa.free(fuzz.run_steps); } -fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void { - rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { - const compile = run.producer.?; - log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); +fn rebuildTestsWorkerRun( + maker: *Maker, + run_index: Configuration.Step.Index, + parent_prog_node: std.Progress.Node, +) void { + rebuildTestsWorkerRunFallible(maker, run_index, parent_prog_node) catch |err| { + const conf = &maker.scanned_config.configuration; + const conf_run = run_index.ptr(conf).extended.cast(conf, Configuration.Step.Run).?; + const comp_index = conf_run.producer.value.?; + const step_name = comp_index.ptr(conf).name.slice(conf); + log.err("step {s}: failed to rebuild in fuzz mode: {t}", .{ step_name, err }); }; } -fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { - const graph = run.step.owner.graph; +fn rebuildTestsWorkerRunFallible( + maker: *Maker, + run_index: Configuration.Step.Index, + parent_prog_node: std.Progress.Node, +) !void { + const graph = maker.graph; const io = graph.io; - const compile = run.producer.?; - const prog_node = parent_prog_node.start(compile.step.name, 0); + const gpa = maker.gpa; + const conf = &maker.scanned_config.configuration; + const run = &maker.stepByIndex(run_index).extended.run; + const conf_run = run_index.ptr(conf).extended.cast(conf, Configuration.Step.Run).?; + const comp_index = conf_run.producer.value.?; + const comp_step = maker.stepByIndex(comp_index); + const comp = &comp_step.extended.compile; + const conf_comp_step = comp_index.ptr(conf); + const conf_comp = conf_comp_step.extended.cast(conf, Configuration.Step.Compile).?; + const root_module = conf_comp.root_module.get(conf); + const target = root_module.resolved_target.get(conf).?.result.get(conf); + + const prog_node = parent_prog_node.start(conf_comp_step.name.slice(conf), 0); defer prog_node.end(); - const result = compile.rebuildInFuzzMode(gpa, prog_node); + const result = comp.rebuildInFuzzMode(maker, comp_index, prog_node); - const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; - const show_error_msgs = compile.step.result_error_msgs.items.len > 0; - const show_stderr = compile.step.result_stderr.len > 0; + const show_compile_errors = comp_step.result_error_bundle.errorMessageCount() > 0; + const show_error_msgs = comp_step.result_error_msgs.items.len > 0; + const show_stderr = comp_step.result_stderr.len > 0; if (show_error_msgs or show_compile_errors or show_stderr) { var buf: [256]u8 = undefined; const stderr = try io.lockStderr(&buf, graph.stderr_mode); defer io.unlockStderr(); - Maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + maker.printErrorMessages(comp_index, .{}, stderr.terminal(), .verbose, .indent) catch {}; } const rebuilt_bin_path = result catch |err| switch (err) { error.MakeFailed => return, else => |other| return other, }; - run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); + const compile_filename = try std.zig.binNameAlloc(gpa, .{ + .root_name = conf_comp.root_name.slice(conf), + .cpu_arch = target.flags.cpu_arch.unwrap().?, + .os_tag = target.flags.os_tag.unwrap().?, + .ofmt = target.flags.object_format.unwrap().?, + .abi = target.flags.abi.unwrap().?, + .output_mode = switch (conf_comp.flags3.kind) { + .lib => .Lib, + .obj, .test_obj => .Obj, + .exe, .@"test" => .Exe, + }, + .link_mode = conf_comp.flags2.linkage.unwrap(), + .version = if (conf_comp.version.value) |v| + std.SemanticVersion.parse(v.slice(conf)) catch unreachable + else + null, + }); + defer gpa.free(compile_filename); + + run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile_filename); } -fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { - const owner = run.step.owner; - const gpa = owner.allocator; - const graph = owner.graph; +fn fuzzWorkerRun(fuzz: *Fuzz, run_index: Configuration.Step.Index) void { + const maker = fuzz.maker; + const graph = maker.graph; const io = graph.io; + const conf = &maker.scanned_config.configuration; + const run = &maker.stepByIndex(run_index).extended.run; - run.rerunInFuzzMode(run, fuzz, fuzz.prog_node) catch |err| switch (err) { + run.rerunInFuzzMode(run_index, fuzz, fuzz.prog_node) catch |err| switch (err) { error.MakeFailed => { var buf: [256]u8 = undefined; const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) { error.Canceled => return, }; defer io.unlockStderr(); - Maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + maker.printErrorMessages(run_index, .{}, stderr.terminal(), .verbose, .indent) catch {}; return; }, else => { - log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err }); + const step_name = run_index.ptr(conf).name.slice(conf); + log.err("step {s}: failed to rerun in fuzz mode: {t}", .{ step_name, err }); return; }, }; } pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { - if (true) @panic("TODO update the fuzzer"); assert(fuzz.mode == .forever); - const gpa = fuzz.maker.gpa; + const maker = fuzz.maker; + const gpa = maker.gpa; + const conf = &maker.scanned_config.configuration; var arena_state: std.heap.ArenaAllocator = .init(gpa); defer arena_state.deinit(); @@ -233,8 +277,11 @@ pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { var dedup_table: DedupTable = .empty; defer dedup_table.deinit(gpa); - for (fuzz.run_steps) |run_step| { - const compile_inputs = run_step.producer.?.step.inputs.table; + for (fuzz.run_steps) |run_index| { + const conf_run = run_index.ptr(conf).extended.cast(conf, Configuration.Step.Run) orelse continue; + const comp_index = conf_run.producer.value.?; + const comp_step = maker.stepByIndex(comp_index); + const compile_inputs = comp_step.inputs.table; for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { try dedup_table.ensureUnusedCapacity(gpa, file_list.items.len); for (file_list.items) |sub_path| { @@ -372,14 +419,15 @@ fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { fuzz.msg_queue.clearRetainingCapacity(); } } -fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { - if (true) @panic("TODO update the fuzzer"); +fn prepareTables(fuzz: *Fuzz, run_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { assert(fuzz.mode == .forever); const ws = fuzz.mode.forever.ws; const maker = fuzz.maker; const graph = maker.graph; const io = graph.io; const gpa = maker.gpa; + const conf = &maker.scanned_config.configuration; + const cache_root = graph.local_cache_root; try fuzz.coverage_mutex.lock(io); defer fuzz.coverage_mutex.unlock(io); @@ -405,37 +453,45 @@ fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage }; errdefer gop.value_ptr.coverage.deinit(gpa); - const rebuilt_exe_path = run_step_index.rebuilt_executable.?; - const target = run_step_index.producer.?.rootModuleTarget(); + const run_step = maker.stepByIndex(run_index); + const conf_run_step = run_index.ptr(conf); + const conf_run = conf_run_step.extended.cast(conf, Configuration.Step.Run).?; + const comp_index = conf_run.producer.value.?; + const conf_comp_step = comp_index.ptr(conf); + const conf_comp = conf_comp_step.extended.cast(conf, Configuration.Step.Compile).?; + const rebuilt_exe_path = run_step.extended.run.rebuilt_executable.?; + const root_module = conf_comp.root_module.get(conf); + const target = root_module.resolved_target.get(conf).?.result.get(conf); + var debug_info = std.debug.Info.load( gpa, io, rebuilt_exe_path, &gop.value_ptr.coverage, - target.ofmt, - target.cpu.arch, + target.flags.object_format.unwrap().?, + target.flags.cpu_arch.unwrap().?, ) catch |err| { - log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ - run_step_index.step.name, rebuilt_exe_path, err, + log.err("step {s}: failed to load debug information for {f}: {t}", .{ + conf_run_step.name.slice(conf), rebuilt_exe_path, err, }); return error.AlreadyReported; }; defer debug_info.deinit(gpa); const coverage_file_path: Build.Cache.Path = .{ - .root_dir = run_step_index.step.owner.cache_root, + .root_dir = cache_root, .sub_path = "v/" ++ std.fmt.hex(coverage_id), }; var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { - log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ - run_step_index.step.name, coverage_file_path, err, + log.err("step {s}: failed to load coverage file {f}: {t}", .{ + conf_run_step.name.slice(conf), coverage_file_path, err, }); return error.AlreadyReported; }; defer coverage_file.close(io); const file_size = coverage_file.length(io) catch |err| { - log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err }); + log.err("unable to check len of coverage file {f}: {t}", .{ coverage_file_path, err }); return error.AlreadyReported; }; @@ -447,7 +503,7 @@ fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage coverage_file.handle, 0, ) catch |err| { - log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err }); + log.err("failed to map coverage file {f}: {t}", .{ coverage_file_path, err }); return error.AlreadyReported; }; gop.value_ptr.mapped_memory = mapped_memory; @@ -538,11 +594,12 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte } pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { - if (true) @panic("TODO update the fuzzer"); assert(fuzz.mode == .limit); const maker = fuzz.maker; const graph = maker.graph; const io = graph.io; + const cache_root = graph.local_cache_root; + const conf = &maker.scanned_config.configuration; try fuzz.group.await(io); fuzz.group = .init; @@ -552,13 +609,15 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { if (msg != .coverage) continue; const cov = msg.coverage; + const run_step_name = cov.run.ptr(conf).name.slice(conf); + const run = &maker.stepByIndex(cov.run).extended.run; const coverage_file_path: std.Build.Cache.Path = .{ - .root_dir = cov.run.step.owner.cache_root, + .root_dir = cache_root, .sub_path = "v/" ++ std.fmt.hex(cov.id), }; var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { - fatal("step '{s}': failed to load coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, + fatal("step {s}: failed to load coverage file {f}: {t}", .{ + run_step_name, coverage_file_path, err, }); }; defer coverage_file.close(io); @@ -569,14 +628,14 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { var header: fuzz_abi.SeenPcsHeader = undefined; r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { - fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, + fatal("step {s}: failed to read from coverage file {f}: {t}", .{ + run_step_name, coverage_file_path, err, }); }; if (header.pcs_len == 0) { - fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{ - cov.run.step.name, coverage_file_path, + fatal("step {s}: corrupted coverage file {f}: pcs_len was zero", .{ + run_step_name, coverage_file_path, }); } @@ -584,8 +643,8 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len); for (0..chunk_count) |_| { const seen = r.interface.takeInt(usize, .little) catch |err| { - fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, + fatal("step {s}: failed to read from coverage file {f}: {t}", .{ + run_step_name, coverage_file_path, err, }); }; seen_count += @popCount(seen); @@ -602,8 +661,8 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { \\Coverage: {}/{} -> {}/{} ({:.02}%) \\ , .{ - cov.run.step.name, - cov.run.fuzz_tests.items[0], + run_step_name, + run.fuzz_tests.items[0], cov.id, cov.cumulative.runs, header.n_runs, diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index 667ce6d06d2b23999affbcde5524f48bc7bcf39e..bca008ffc04ceb84cf49b8937b393950b1689080 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -31,6 +31,7 @@ reference_trace: ?u32 = null, debug_log_scopes: std.ArrayList([]const u8) = .empty, debug_compile_errors: bool = false, debug_incremental: bool = false, +fuzzing: bool = false, verbose: bool = false, verbose_air: bool = false, verbose_cc: bool = false, diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 859a24d4c4d7e44e9a7f212047e58af6a524db84..4f7d89ba8ecc552e91127e7f18a2ba7cbe60413e 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -947,11 +947,11 @@ fn lowerZigArgs( pub fn rebuildInFuzzMode( compile: *Compile, maker: *Maker, - step_index: Configuration.Step.Index, + compile_index: Configuration.Step.Index, progress_node: std.Progress.Node, ) !Path { - const gpa = maker.graph.gpa; - const step = maker.stepByIndex(step_index); + const gpa = maker.gpa; + const step = maker.stepByIndex(compile_index); step.result_error_msgs.clearRetainingCapacity(); step.result_stderr = ""; @@ -966,8 +966,8 @@ pub fn rebuildInFuzzMode( const zig_args = &compile.zig_args; zig_args.clearRetainingCapacity(); - try lowerZigArgs(compile, maker, progress_node, zig_args, true); - const maybe_output_bin_path = try step.evalZigProcess(zig_args.items, progress_node, false, maker); + try lowerZigArgs(compile, compile_index, maker, progress_node, zig_args, true); + const maybe_output_bin_path = try Step.evalZigProcess(compile_index, maker, zig_args.items, progress_node, false); return maybe_output_bin_path.?; } @@ -980,10 +980,7 @@ fn addFlag(gpa: Allocator, args: *std.ArrayList([]const u8), comptime name: []co try args.append(gpa, if (cond) "-f" ++ name else "-fno-" ++ name); } -fn checkCompileErrors( - maker: *Maker, - step_index: Configuration.Step.Index, -) Step.ExtendedMakeError!void { +fn checkCompileErrors(maker: *Maker, step_index: Configuration.Step.Index) Step.ExtendedMakeError!void { const step = maker.stepByIndex(step_index); const graph = maker.graph; const arena = graph.arena; // TODO don't leak into the process arena diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index c76d6626eed8af0281763a6b3783ee9afe3ba489..c72db4eb2603eb5f3cbf98eb3b27b409713ebef9 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -67,6 +67,7 @@ pub fn make( } } + man.hash.add(graph.fuzzing); man.hash.add(conf_run.flags.color); man.hash.add(conf_run.flags.disable_zig_progress); @@ -234,7 +235,8 @@ pub fn make( } // Whether the Run step has side effects *other than* updating the output arguments. - const has_side_effects = conf_run.flags.has_side_effects or any_cli_positionals or + // When fuzzing we need to always run the test runner to populate fuzz_tests. + const has_side_effects = graph.fuzzing or conf_run.flags.has_side_effects or any_cli_positionals or switch (conf_run.flags.stdio) { .infer_from_args => !any_output_args and conf_run.captured_stdout.value == null and @@ -1511,7 +1513,7 @@ const IndexedOutput = struct { pub fn rerunInFuzzMode( run: *Run, run_index: Configuration.Step.Index, - fuzz: *std.Build.Fuzz, + fuzz: *Fuzz, prog_node: std.Progress.Node, ) !void { const maker = fuzz.maker; @@ -1524,6 +1526,7 @@ pub fn rerunInFuzzMode( const conf_step = run_index.ptr(conf); const conf_run = conf_step.extended.get(conf.extra).run; const argv_list = &run.argv; + const cache_root = graph.local_cache_root; argv_list.clearRetainingCapacity(); @@ -1599,6 +1602,15 @@ pub fn rerunInFuzzMode( } } + if (conf_run.flags.test_runner_mode) { + const cache_dir_string = try convertPathArg(run_index, maker, .{ .root_dir = cache_root }); + + try argv_list.ensureUnusedCapacity(gpa, 3); + argv_list.appendAssumeCapacity(try allocPrint(arena, "--cache-dir={s}", .{cache_dir_string})); + argv_list.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{graph.random_seed})); + argv_list.appendAssumeCapacity("--listen=-"); + } + if (step.result_failed_command) |cmd| { gpa.free(cmd); step.result_failed_command = null; -- 2.54.0 From c5517102e75eb834acca8593711cd96536d3a271 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 24 May 2026 12:17:52 -0700 Subject: [PATCH 158/179] Maker: delete ConfigHeader unit tests these will work better as standalone tests that exercise the std.Build API. --- lib/compiler/Maker/Step/ConfigHeader.zig | 310 ----------------------- 1 file changed, 310 deletions(-) diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index 762b0ff483856df1aaa2a5ddad3d19b5c47199e7..8f4f677ee3bd4cef5f42541673823e9377185c86 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -608,313 +608,3 @@ fn expandVariablesCmake( return result.toOwnedSliceAssert(); } - -fn testReplaceVariablesAutoconfAt( - arena: Allocator, - contents: []const u8, - expected: []const u8, - value_map: *const ValueMap, -) !void { - var aw: Writer.Allocating = .init(arena); - defer aw.deinit(); - - const used = try arena.alloc(bool, value_map.count()); - for (used) |*u| u.* = false; - - try expandVariablesAutoconfAt(&aw.writer, contents, value_map, used); - - for (used) |u| if (!u) return error.UnusedValue; - try std.testing.expectEqualStrings(expected, aw.written()); -} - -fn testReplaceVariablesCMake( - arena: Allocator, - contents: []const u8, - expected: []const u8, - value_map: *const ValueMap, -) !void { - const actual = try expandVariablesCmake(arena, contents, value_map); - - try std.testing.expectEqualStrings(expected, actual); -} - -test "expandVariablesAutoconfAt simple cases" { - const allocator = std.testing.allocator; - var value_map: ValueMap = .empty; - defer value_map.deinit(); - - // empty strings are preserved - try testReplaceVariablesAutoconfAt(allocator, "", "", value_map); - - // line with misc content is preserved - try testReplaceVariablesAutoconfAt(allocator, "no substitution", "no substitution", value_map); - - // empty @ sigils are preserved - try testReplaceVariablesAutoconfAt(allocator, "@", "@", value_map); - try testReplaceVariablesAutoconfAt(allocator, "@@", "@@", value_map); - try testReplaceVariablesAutoconfAt(allocator, "@@@", "@@@", value_map); - try testReplaceVariablesAutoconfAt(allocator, "@@@@", "@@@@", value_map); - - // simple substitution - try value_map.putNoClobber("undef", .undef); - try testReplaceVariablesAutoconfAt(allocator, "@undef@", "", value_map); - value_map.clearRetainingCapacity(); - - try value_map.putNoClobber("defined", .defined); - try testReplaceVariablesAutoconfAt(allocator, "@defined@", "", value_map); - value_map.clearRetainingCapacity(); - - try value_map.putNoClobber("true", Value{ .boolean = true }); - try testReplaceVariablesAutoconfAt(allocator, "@true@", "1", value_map); - value_map.clearRetainingCapacity(); - - try value_map.putNoClobber("false", Value{ .boolean = false }); - try testReplaceVariablesAutoconfAt(allocator, "@false@", "0", value_map); - value_map.clearRetainingCapacity(); - - try value_map.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@", "42", value_map); - value_map.clearRetainingCapacity(); - - try value_map.putNoClobber("ident", Value{ .string = "value" }); - try testReplaceVariablesAutoconfAt(allocator, "@ident@", "value", value_map); - value_map.clearRetainingCapacity(); - - try value_map.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@", "text", value_map); - value_map.clearRetainingCapacity(); - - // double packed substitution - try value_map.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@@string@", "texttext", value_map); - value_map.clearRetainingCapacity(); - - // triple packed substitution - try value_map.putNoClobber("int", Value{ .int = 42 }); - try value_map.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@string@@int@@string@", "text42text", value_map); - value_map.clearRetainingCapacity(); - - // double separated substitution - try value_map.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@.@int@", "42.42", value_map); - value_map.clearRetainingCapacity(); - - // triple separated substitution - try value_map.putNoClobber("true", Value{ .boolean = true }); - try value_map.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "@int@.@true@.@int@", "42.1.42", value_map); - value_map.clearRetainingCapacity(); - - // misc prefix is preserved - try value_map.putNoClobber("false", Value{ .boolean = false }); - try testReplaceVariablesAutoconfAt(allocator, "false is @false@", "false is 0", value_map); - value_map.clearRetainingCapacity(); - - // misc suffix is preserved - try value_map.putNoClobber("true", Value{ .boolean = true }); - try testReplaceVariablesAutoconfAt(allocator, "@true@ is true", "1 is true", value_map); - value_map.clearRetainingCapacity(); - - // surrounding content is preserved - try value_map.putNoClobber("int", Value{ .int = 42 }); - try testReplaceVariablesAutoconfAt(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", value_map); - value_map.clearRetainingCapacity(); - - // incomplete key is preserved - try testReplaceVariablesAutoconfAt(allocator, "@undef", "@undef", value_map); - - // unknown key leads to an error - try std.testing.expectError(error.MissingValue, testReplaceVariablesAutoconfAt(allocator, "@bad@", "", value_map)); - - // unused key leads to an error - try value_map.putNoClobber("int", Value{ .int = 42 }); - try value_map.putNoClobber("false", Value{ .boolean = false }); - try std.testing.expectError(error.UnusedValue, testReplaceVariablesAutoconfAt(allocator, "@int", "", value_map)); - value_map.clearRetainingCapacity(); -} - -test "expandVariablesAutoconfAt edge cases" { - const allocator = std.testing.allocator; - var value_map: std.array_hash_map.String(Value) = .init(allocator); - defer value_map.deinit(); - - // @-vars resolved only when they wrap valid characters, otherwise considered literals - try value_map.putNoClobber("string", Value{ .string = "text" }); - try testReplaceVariablesAutoconfAt(allocator, "@@string@@", "@text@", value_map); - value_map.clearRetainingCapacity(); - - // expanded variables are considered strings after expansion - try value_map.putNoClobber("string_at", Value{ .string = "@string@" }); - try testReplaceVariablesAutoconfAt(allocator, "@string_at@", "@string@", value_map); - value_map.clearRetainingCapacity(); -} - -test "expandVariablesCmake simple cases" { - const allocator = std.testing.allocator; - var value_map: std.array_hash_map.String(Value) = .init(allocator); - defer value_map.deinit(); - - try value_map.putNoClobber("undef", .undef); - try value_map.putNoClobber("defined", .defined); - try value_map.putNoClobber("true", Value{ .boolean = true }); - try value_map.putNoClobber("false", Value{ .boolean = false }); - try value_map.putNoClobber("int", Value{ .int = 42 }); - try value_map.putNoClobber("ident", Value{ .string = "value" }); - try value_map.putNoClobber("string", Value{ .string = "text" }); - - // empty strings are preserved - try testReplaceVariablesCMake(allocator, "", "", value_map); - - // line with misc content is preserved - try testReplaceVariablesCMake(allocator, "no substitution", "no substitution", value_map); - - // empty ${} wrapper leads to an error - try std.testing.expectError(error.MissingKey, testReplaceVariablesCMake(allocator, "${}", "", value_map)); - - // empty @ sigils are preserved - try testReplaceVariablesCMake(allocator, "@", "@", value_map); - try testReplaceVariablesCMake(allocator, "@@", "@@", value_map); - try testReplaceVariablesCMake(allocator, "@@@", "@@@", value_map); - try testReplaceVariablesCMake(allocator, "@@@@", "@@@@", value_map); - - // simple substitution - try testReplaceVariablesCMake(allocator, "@undef@", "", value_map); - try testReplaceVariablesCMake(allocator, "${undef}", "", value_map); - try testReplaceVariablesCMake(allocator, "@defined@", "", value_map); - try testReplaceVariablesCMake(allocator, "${defined}", "", value_map); - try testReplaceVariablesCMake(allocator, "@true@", "1", value_map); - try testReplaceVariablesCMake(allocator, "${true}", "1", value_map); - try testReplaceVariablesCMake(allocator, "@false@", "0", value_map); - try testReplaceVariablesCMake(allocator, "${false}", "0", value_map); - try testReplaceVariablesCMake(allocator, "@int@", "42", value_map); - try testReplaceVariablesCMake(allocator, "${int}", "42", value_map); - try testReplaceVariablesCMake(allocator, "@ident@", "value", value_map); - try testReplaceVariablesCMake(allocator, "${ident}", "value", value_map); - try testReplaceVariablesCMake(allocator, "@string@", "text", value_map); - try testReplaceVariablesCMake(allocator, "${string}", "text", value_map); - - // double packed substitution - try testReplaceVariablesCMake(allocator, "@string@@string@", "texttext", value_map); - try testReplaceVariablesCMake(allocator, "${string}${string}", "texttext", value_map); - - // triple packed substitution - try testReplaceVariablesCMake(allocator, "@string@@int@@string@", "text42text", value_map); - try testReplaceVariablesCMake(allocator, "@string@${int}@string@", "text42text", value_map); - try testReplaceVariablesCMake(allocator, "${string}@int@${string}", "text42text", value_map); - try testReplaceVariablesCMake(allocator, "${string}${int}${string}", "text42text", value_map); - - // double separated substitution - try testReplaceVariablesCMake(allocator, "@int@.@int@", "42.42", value_map); - try testReplaceVariablesCMake(allocator, "${int}.${int}", "42.42", value_map); - - // triple separated substitution - try testReplaceVariablesCMake(allocator, "@int@.@true@.@int@", "42.1.42", value_map); - try testReplaceVariablesCMake(allocator, "@int@.${true}.@int@", "42.1.42", value_map); - try testReplaceVariablesCMake(allocator, "${int}.@true@.${int}", "42.1.42", value_map); - try testReplaceVariablesCMake(allocator, "${int}.${true}.${int}", "42.1.42", value_map); - - // misc prefix is preserved - try testReplaceVariablesCMake(allocator, "false is @false@", "false is 0", value_map); - try testReplaceVariablesCMake(allocator, "false is ${false}", "false is 0", value_map); - - // misc suffix is preserved - try testReplaceVariablesCMake(allocator, "@true@ is true", "1 is true", value_map); - try testReplaceVariablesCMake(allocator, "${true} is true", "1 is true", value_map); - - // surrounding content is preserved - try testReplaceVariablesCMake(allocator, "what is 6*7? @int@!", "what is 6*7? 42!", value_map); - try testReplaceVariablesCMake(allocator, "what is 6*7? ${int}!", "what is 6*7? 42!", value_map); - - // incomplete key is preserved - try testReplaceVariablesCMake(allocator, "@undef", "@undef", value_map); - try testReplaceVariablesCMake(allocator, "${undef", "${undef", value_map); - try testReplaceVariablesCMake(allocator, "{undef}", "{undef}", value_map); - try testReplaceVariablesCMake(allocator, "undef@", "undef@", value_map); - try testReplaceVariablesCMake(allocator, "undef}", "undef}", value_map); - - // unknown key leads to an error - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@bad@", "", value_map)); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${bad}", "", value_map)); -} - -test "expandVariablesCmake edge cases" { - const allocator = std.testing.allocator; - var value_map: std.array_hash_map.String(Value) = .init(allocator); - defer value_map.deinit(); - - // special symbols - try value_map.putNoClobber("at", Value{ .string = "@" }); - try value_map.putNoClobber("dollar", Value{ .string = "$" }); - try value_map.putNoClobber("underscore", Value{ .string = "_" }); - - // basic value - try value_map.putNoClobber("string", Value{ .string = "text" }); - - // proxy case value_map - try value_map.putNoClobber("string_proxy", Value{ .string = "string" }); - try value_map.putNoClobber("string_at", Value{ .string = "@string@" }); - try value_map.putNoClobber("string_curly", Value{ .string = "{string}" }); - try value_map.putNoClobber("string_var", Value{ .string = "${string}" }); - - // stack case value_map - try value_map.putNoClobber("nest_underscore_proxy", Value{ .string = "underscore" }); - try value_map.putNoClobber("nest_proxy", Value{ .string = "nest_underscore_proxy" }); - - // @-vars resolved only when they wrap valid characters, otherwise considered literals - try testReplaceVariablesCMake(allocator, "@@string@@", "@text@", value_map); - try testReplaceVariablesCMake(allocator, "@${string}@", "@text@", value_map); - - // @-vars are resolved inside ${}-vars - try testReplaceVariablesCMake(allocator, "${@string_proxy@}", "text", value_map); - - // expanded variables are considered strings after expansion - try testReplaceVariablesCMake(allocator, "@string_at@", "@string@", value_map); - try testReplaceVariablesCMake(allocator, "${string_at}", "@string@", value_map); - try testReplaceVariablesCMake(allocator, "$@string_curly@", "${string}", value_map); - try testReplaceVariablesCMake(allocator, "$${string_curly}", "${string}", value_map); - try testReplaceVariablesCMake(allocator, "${string_var}", "${string}", value_map); - try testReplaceVariablesCMake(allocator, "@string_var@", "${string}", value_map); - try testReplaceVariablesCMake(allocator, "${dollar}{${string}}", "${text}", value_map); - try testReplaceVariablesCMake(allocator, "@dollar@{${string}}", "${text}", value_map); - try testReplaceVariablesCMake(allocator, "@dollar@{@string@}", "${text}", value_map); - - // when expanded variables contain invalid characters, they prevent further expansion - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${${string_var}}", "", value_map)); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${@string_var@}", "", value_map)); - - // nested expanded variables are expanded from the inside out - try testReplaceVariablesCMake(allocator, "${string${underscore}proxy}", "string", value_map); - try testReplaceVariablesCMake(allocator, "${string@underscore@proxy}", "string", value_map); - - // nested vars are only expanded when ${} is closed - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@underscore@proxy@", "", value_map)); - try testReplaceVariablesCMake(allocator, "${nest${underscore}proxy}", "nest_underscore_proxy", value_map); - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "@nest@@nest_underscore@underscore@proxy@@proxy@", "", value_map)); - try testReplaceVariablesCMake(allocator, "${nest${${nest_underscore${underscore}proxy}}proxy}", "nest_underscore_proxy", value_map); - - // invalid characters lead to an error - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str*ing}", "", value_map)); - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str$ing}", "", value_map)); - try std.testing.expectError(error.InvalidCharacter, testReplaceVariablesCMake(allocator, "${str@ing}", "", value_map)); -} - -test "expandVariablesCmake escaped characters" { - const allocator = std.testing.allocator; - var value_map: std.array_hash_map.String(Value) = .init(allocator); - defer value_map.deinit(); - - try value_map.putNoClobber("string", Value{ .string = "text" }); - - // backslash is an invalid character for @ lookup - try testReplaceVariablesCMake(allocator, "\\@string\\@", "\\@string\\@", value_map); - - // backslash is preserved, but doesn't affect ${} variable expansion - try testReplaceVariablesCMake(allocator, "\\${string}", "\\text", value_map); - - // backslash breaks ${} opening bracket identification - try testReplaceVariablesCMake(allocator, "$\\{string}", "$\\{string}", value_map); - - // backslash is skipped when checking for invalid characters, yet it mangles the key - try std.testing.expectError(error.MissingValue, testReplaceVariablesCMake(allocator, "${string\\}", "", value_map)); -} -- 2.54.0 From 2a8e15bc067a02b979bc06006f423a6a700ad36f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 24 May 2026 13:30:40 -0700 Subject: [PATCH 159/179] disable passing zig progress pipe to nsz libc test however, this does not actually address the test case that started failing --- test/src/Libc.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/src/Libc.zig b/test/src/Libc.zig index e06651bb55c7e6ef2efb940feba81fe9922068b1..a7cb01c8252edf4ce8b10bab454f95913483a13c 100644 --- a/test/src/Libc.zig +++ b/test/src/Libc.zig @@ -35,7 +35,7 @@ pub fn addLibcTestCase( const arena = graph.arena; const name = arena.dupe(u8, path[0 .. path.len - std.fs.path.extension(path).len]) catch @panic("OOM"); std.mem.replaceScalar(u8, name, '/', '.'); - libc.test_cases.append(libc.b.allocator, .{ + libc.test_cases.append(arena, .{ .name = name, .src_file = libc.libc_test_src_path.path(libc.b, path), .additional_src_file = if (options.additional_src_file) |additional_src_file| libc.libc_test_src_path.path(libc.b, additional_src_file) else null, @@ -114,6 +114,7 @@ pub fn addTarget(libc: *const Libc, target: std.Build.ResolvedTarget) void { const run = libc.b.addRunArtifact(exe); run.setName(annotated_case_name); run.skip_foreign_checks = true; + run.disable_zig_progress = true; // can interfere with fd count assumptions run.expectStdErrEqual(""); run.expectStdOutEqual(""); run.expectExitCode(0); -- 2.54.0 From bc031bedaa470d14348a6be1da09a8ba5fd31c5d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 24 May 2026 13:35:18 -0700 Subject: [PATCH 160/179] std.Build.Step: delete legacy Maker fields --- lib/std/Build/Step.zig | 61 +++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 4332737c16ed6e7250cde394b15b285c938ac542..b35697a2a1bba5896737df3d4375d433934795fb 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -1,20 +1,15 @@ const Step = @This(); -const builtin = @import("builtin"); const std = @import("../std.zig"); -const Io = std.Io; const Build = std.Build; -const Allocator = std.mem.Allocator; const assert = std.debug.assert; -const Cache = Build.Cache; -const Path = Cache.Path; -const ArrayList = std.ArrayList; +const Configuration = std.Build.Configuration; -tag: std.Build.Configuration.Step.Tag, +tag: Configuration.Step.Tag, name: []const u8, owner: *Build, -dependencies: ArrayList(*Step), +dependencies: std.ArrayList(*Step), /// Set this field to declare an upper bound on the amount of bytes of memory it will /// take to run the step. Zero means no limit. @@ -37,43 +32,30 @@ dependencies: ArrayList(*Step), /// total system memory available. max_rss: usize, -state: State, - /// The return address associated with creation of this step that can be useful /// to print along with debugging messages. debug_stack_trace: std.debug.StackTrace, -pub const State = enum { - precheck_unstarted, - precheck_started, - /// This is also used to indicate "dirty" steps that have been modified - /// after a previous build completed, in which case, the step may or may - /// not have been completed before. Either way, one or more of its direct - /// file system inputs have been modified, meaning that the step needs to - /// be re-evaluated. - precheck_done, - dependency_failure, -}; - -pub const Tag = std.Build.Configuration.Step.Tag; +pub const Tag = Configuration.Step.Tag; pub fn Type(comptime tag: Tag) type { return switch (tag) { - .top_level => Build.TopLevelStep, + .check_file => CheckFile, .compile => Compile, - .install_artifact => InstallArtifact, - .install_file => InstallFile, - .install_dir => InstallDir, + .config_header => ConfigHeader, .fail => Fail, + .find_program => FindProgram, .fmt => Fmt, - .translate_c => TranslateC, - .write_file => WriteFile, - .update_source_files => UpdateSourceFiles, - .run => Run, - .check_file => CheckFile, - .config_header => ConfigHeader, + .install_artifact => InstallArtifact, + .install_dir => InstallDir, + .install_file => InstallFile, .obj_copy => ObjCopy, .options => Options, + .run => Run, + .top_level => TopLevel, + .translate_c => TranslateC, + .update_source_files => UpdateSourceFiles, + .write_file => WriteFile, }; } @@ -116,7 +98,6 @@ pub fn init(options: StepOptions) Step { .name = arena.dupe(u8, options.name) catch @panic("OOM"), .owner = options.owner, .dependencies = .empty, - .state = .precheck_unstarted, .max_rss = options.max_rss, .debug_stack_trace = blk: { const addr_buf = arena.alloc(usize, options.owner.debug_stack_frames_count) catch @panic("OOM"); @@ -132,14 +113,12 @@ pub fn dependOn(step: *Step, other: *Step) void { } pub fn cast(step: *Step, comptime T: type) ?*T { - if (step.tag == T.base_tag) { - return @fieldParentPtr("step", step); - } + if (step.tag == T.base_tag) return @fieldParentPtr("step", step); return null; } /// For debugging purposes, prints identifying information about this Step. -pub fn dump(step: *Step, t: Io.Terminal) void { +pub fn dump(step: *Step, t: std.Io.Terminal) void { const w = t.writer; if (step.debug_stack_trace.return_addresses.len > 0) { w.print("name: '{s}'. creation stack trace:\n", .{step.name}) catch {}; @@ -155,16 +134,18 @@ pub fn dump(step: *Step, t: Io.Terminal) void { test { _ = CheckFile; + _ = Compile; + _ = ConfigHeader; _ = Fail; + _ = FindProgram; _ = Fmt; _ = InstallArtifact; _ = InstallDir; _ = InstallFile; _ = ObjCopy; - _ = Compile; _ = Options; _ = Run; _ = TranslateC; - _ = WriteFile; _ = UpdateSourceFiles; + _ = WriteFile; } -- 2.54.0 From f79ae3214c41183c90ca4d7acb0ac0ded3d99f61 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 24 May 2026 13:57:35 -0700 Subject: [PATCH 161/179] zig build: close open file handles before spawning child ever so slightly reduces rlimit pressure in the child. plus it avoids accounting problems in libc-test --- src/main.zig | 1064 +++++++++++++++++++++++++------------------------- 1 file changed, 533 insertions(+), 531 deletions(-) diff --git a/src/main.zig b/src/main.zig index a79ee7150f240e37bb3740e31c92843c52991da8..e15ab41c1f56c05a72e6c7ab866797d656f5de93 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5202,562 +5202,564 @@ fn cmdBuild( .build_file = build_file, }); - // This `init` calls `fatal` on error. - var dirs: Compilation.Directories = .init( - arena, - io, - override_lib_dir, - override_global_cache_dir, - .{ .override = path: { - if (override_local_cache_dir) |d| break :path d; - break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename}); - } }, - .empty, - self_exe_path, - environ_map, - cwd_path, - ); - defer dirs.deinit(io); - - const thread_limit = @min( - @max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), - std.math.maxInt(Zcu.PerThread.IdBacking), - ); - try setThreadLimit(arena, thread_limit); - - // Cache lookup for configure options. If we get a match, we can skip - // execution of the configure script. If not, we get the file path to pass - // to the configure process. - var local_cache: Cache = .{ - .gpa = gpa, - .io = io, - .manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}), - .cwd = cwd_path, - }; - local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); - local_cache.addPrefix(dirs.zig_lib); - local_cache.addPrefix(dirs.local_cache); - local_cache.addPrefix(dirs.global_cache); - defer local_cache.manifest_dir.close(io); - - var config_man = local_cache.obtain(); - defer config_man.deinit(); - config_man.hash.addBytes(build_options.version); - - for (cached_passthru_configure.items) |i| - config_man.hash.addBytes(configure_argv.items[i]); - - // Prevents a `zig build` from getting a false positive cache hit following - // a `zig build --cache-poison=ignored`. - config_man.hash.add(cache_poison == .ignored); - - // Normally the build runner is compiled for the host target but here is - // some code to help when debugging edits to the build runner so that you - // can make sure it compiles successfully on other targets. - const resolved_target: Package.Module.ResolvedTarget = t: { - if (build_options.enable_debug_extensions) { - if (debug_target) |triple| { - const target_query = try std.Target.Query.parse(.{ - .arch_os_abi = triple, - }); - config_man.hash.addBytes(triple); - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, target_query), - .is_native_os = false, - .is_native_abi = false, - .is_explicit_dynamic_linker = false, - }; - } - } - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, .{}), - .is_native_os = true, - .is_native_abi = true, - .is_explicit_dynamic_linker = false, - }; - }; - - // Likewise, `--debug-libc` allows overriding the libc installation. - const libc_installation: ?*const LibCInstallation = lci: { - const paths_file = debug_libc_paths_file orelse break :lci null; - if (!build_options.enable_debug_extensions) unreachable; - const lci = try arena.create(LibCInstallation); - lci.* = try .parse(arena, io, paths_file, &resolved_target.result); - LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi); - break :lci lci; - }; - - // Kick off an optimized compilation of the make runner. - var make_runner_task = io.async(compileMakeRunner, .{ gpa, arena, io, .{ - .dirs = .{ - .cwd = dirs.cwd, - .zig_lib = dirs.zig_lib, - .global_cache = dirs.global_cache, - .local_cache = dirs.global_cache, - }, - .environ_map = environ_map, - .parent_prog_node = root_prog_node, - .resolved_target = resolved_target, - .libc_installation = libc_installation, - .thread_limit = thread_limit, - .self_exe_path = self_exe_path, - .color = color, - .reference_trace = reference_trace, - .optimize_mode = maker_optimize_mode, - } }); - defer _ = make_runner_task.cancel(io) catch {}; - - const pkg_root: Path = if (override_pkg_dir) |p| - .initCwd(p) - else if (system_pkg_dir_path) |p| - .initCwd(p) - else - .{ - .root_dir = build_root.directory, - .sub_path = "zig-pkg", - }; - - make_argv.items[make_argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; - make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path; - make_argv.items[make_argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; - make_argv.items[make_argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; - - configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; - - // Dummy http client that is not actually used when fetch_command is unsupported. - // Prevents bootstrap from depending on a bunch of unnecessary stuff. - var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { - allocator: Allocator, - io: Io, - fn deinit(_: @This()) void {} - } = .{ .allocator = gpa, .io = io }; - defer http_client.deinit(); - - var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; - var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; - { - // Populate fork_set. - var group: Io.Group = .init; - defer group.cancel(io); - - for (forks.items) |*fork| - group.async(io, Fork.load, .{ io, gpa, fork, color }); - - try group.await(io); - - for (forks.items) |*fork| { - if (fork.failed) process.exit(1); - try fork_set.put(arena, .{ - .path = fork.path, - .manifest_ast = fork.manifest_ast, - .manifest = fork.manifest, - .uses = 0, - }, {}); - } - } - defer Fork.deinitList(forks.items); - - // This loop is re-evaluated when the build script exits with an indication that it - // could not continue due to missing lazy dependencies. - const configuration_path: Path, const poisoned: bool = cp: while (true) { - // We want to release all the locks before executing the child process, so we make a nice - // big block here to ensure the cleanup gets run when we extract out our argv. + // This `init` calls `fatal` on error. + var dirs: Compilation.Directories = .init( + arena, + io, + override_lib_dir, + override_global_cache_dir, + .{ .override = path: { + if (override_local_cache_dir) |d| break :path d; + break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename}); + } }, + .empty, + self_exe_path, + environ_map, + cwd_path, + ); + defer dirs.deinit(io); + + const thread_limit = @min( + @max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), + std.math.maxInt(Zcu.PerThread.IdBacking), + ); + try setThreadLimit(arena, thread_limit); + + // Cache lookup for configure options. If we get a match, we can skip + // execution of the configure script. If not, we get the file path to pass + // to the configure process. + var local_cache: Cache = .{ + .gpa = gpa, + .io = io, + .manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}), + .cwd = cwd_path, + }; + local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + local_cache.addPrefix(dirs.zig_lib); + local_cache.addPrefix(dirs.local_cache); + local_cache.addPrefix(dirs.global_cache); + defer local_cache.manifest_dir.close(io); + + var config_man = local_cache.obtain(); + defer config_man.deinit(); + config_man.hash.addBytes(build_options.version); + + for (cached_passthru_configure.items) |i| + config_man.hash.addBytes(configure_argv.items[i]); + + // Prevents a `zig build` from getting a false positive cache hit following + // a `zig build --cache-poison=ignored`. + config_man.hash.add(cache_poison == .ignored); + + // Normally the build runner is compiled for the host target but here is + // some code to help when debugging edits to the build runner so that you + // can make sure it compiles successfully on other targets. + const resolved_target: Package.Module.ResolvedTarget = t: { + if (build_options.enable_debug_extensions) { + if (debug_target) |triple| { + const target_query = try std.Target.Query.parse(.{ + .arch_os_abi = triple, + }); + config_man.hash.addBytes(triple); + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(io, target_query), + .is_native_os = false, + .is_native_abi = false, + .is_explicit_dynamic_linker = false, + }; + } + } + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(io, .{}), + .is_native_os = true, + .is_native_abi = true, + .is_explicit_dynamic_linker = false, + }; + }; + + // Likewise, `--debug-libc` allows overriding the libc installation. + const libc_installation: ?*const LibCInstallation = lci: { + const paths_file = debug_libc_paths_file orelse break :lci null; + if (!build_options.enable_debug_extensions) unreachable; + const lci = try arena.create(LibCInstallation); + lci.* = try .parse(arena, io, paths_file, &resolved_target.result); + LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi); + break :lci lci; + }; + + // Kick off an optimized compilation of the make runner. + var make_runner_task = io.async(compileMakeRunner, .{ gpa, arena, io, .{ + .dirs = .{ + .cwd = dirs.cwd, + .zig_lib = dirs.zig_lib, + .global_cache = dirs.global_cache, + .local_cache = dirs.global_cache, + }, + .environ_map = environ_map, + .parent_prog_node = root_prog_node, + .resolved_target = resolved_target, + .libc_installation = libc_installation, + .thread_limit = thread_limit, + .self_exe_path = self_exe_path, + .color = color, + .reference_trace = reference_trace, + .optimize_mode = maker_optimize_mode, + } }); + defer _ = make_runner_task.cancel(io) catch {}; + + const pkg_root: Path = if (override_pkg_dir) |p| + .initCwd(p) + else if (system_pkg_dir_path) |p| + .initCwd(p) + else + .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", + }; + + make_argv.items[make_argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; + make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path; + make_argv.items[make_argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; + make_argv.items[make_argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; + + configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; + + // Dummy http client that is not actually used when fetch_command is unsupported. + // Prevents bootstrap from depending on a bunch of unnecessary stuff. + var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { + allocator: Allocator, + io: Io, + fn deinit(_: @This()) void {} + } = .{ .allocator = gpa, .io = io }; + defer http_client.deinit(); + + var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; + var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; + { - const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_make_runner) |runner| .{ - .root = try .fromUnresolved(arena, dirs, &.{fs.path.dirname(runner) orelse "."}), - .root_src_path = fs.path.basename(runner), - } else .{ - .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), - .root_src_path = "configurer.zig", - }; + // Populate fork_set. + var group: Io.Group = .init; + defer group.cancel(io); - const config = try Compilation.Config.resolve(.{ - .output_mode = .Exe, - .resolved_target = resolved_target, - .have_zcu = true, - .emit_bin = true, - .is_test = false, - }); + for (forks.items) |*fork| + group.async(io, Fork.load, .{ io, gpa, fork, color }); - const root_mod = try Package.Module.create(arena, .{ - .paths = main_mod_paths, - .fully_qualified_name = "root", - .cc_argv = &.{}, - .inherited = .{ + try group.await(io); + + for (forks.items) |*fork| { + if (fork.failed) process.exit(1); + try fork_set.put(arena, .{ + .path = fork.path, + .manifest_ast = fork.manifest_ast, + .manifest = fork.manifest, + .uses = 0, + }, {}); + } + } + defer Fork.deinitList(forks.items); + + // This loop is re-evaluated when the build script exits with an indication that it + // could not continue due to missing lazy dependencies. + const configuration_path: Path, const poisoned: bool = cp: while (true) { + // We want to release all the locks before executing the child process, so we make a nice + // big block here to ensure the cleanup gets run when we extract out our argv. + { + const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_make_runner) |runner| .{ + .root = try .fromUnresolved(arena, dirs, &.{fs.path.dirname(runner) orelse "."}), + .root_src_path = fs.path.basename(runner), + } else .{ + .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), + .root_src_path = "configurer.zig", + }; + + const config = try Compilation.Config.resolve(.{ + .output_mode = .Exe, .resolved_target = resolved_target, - .single_threaded = true, - }, - .global = config, - .parent = null, - }); - - const build_mod = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromUnresolved(arena, dirs, &.{build_root.directory.path orelse "."}), - .root_src_path = build_root.build_zig_basename, - }, - .fully_qualified_name = "root.@build", - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - }); - - if (dev.env.supports(.fetch_command)) { - const fetch_prog_node = root_prog_node.start("Fetch Packages", 0); - defer fetch_prog_node.end(); - - // Reset fork match counts. - for (fork_set.keys()) |*fork| fork.uses = 0; - - var job_queue: Package.Fetch.JobQueue = .{ - .io = io, - .http_client = &http_client, - .global_cache = dirs.global_cache, - .local_storage = &.{ - .cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" }, - .pkg_root = pkg_root, + .have_zcu = true, + .emit_bin = true, + .is_test = false, + }); + + const root_mod = try Package.Module.create(arena, .{ + .paths = main_mod_paths, + .fully_qualified_name = "root", + .cc_argv = &.{}, + .inherited = .{ + .resolved_target = resolved_target, + .single_threaded = true, }, - .recursive = true, - .debug_hash = false, - .unlazy_set = unlazy_set, - .fork_set = fork_set, - .mode = fetch_mode, - .prog_node = fetch_prog_node, - .read_only = system_pkg_dir_path != null, - }; - defer job_queue.deinit(); - - if (system_pkg_dir_path == null) { - try http_client.initDefaultProxies(arena, environ_map); - } - - try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); - try job_queue.table.ensureUnusedCapacity(gpa, 1); - - const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory }; - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .relative_path = phantom_package_root }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .remote_package_root = phantom_package_root, - .parent_package_root = phantom_package_root, - .parent_manifest_ast = null, - .prog_node = fetch_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .use_latest_commit = false, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = true, - .oom_flag = false, - .latest_commit = null, - - .module = build_mod, - }; - - job_queue.all_fetches.appendAssumeCapacity(&fetch); - - job_queue.table.putAssumeCapacityNoClobber( - Package.Fetch.relativePathDigest(phantom_package_root, dirs.global_cache), - &fetch, - ); - - job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); - try job_queue.group.await(io); - - { - // Ensure that forks were actually used. This is done - // before printing manifest errors because using a fork can - // prevent them. - var any_unused = false; - for (fork_set.keys()) |*fork| { - if (fork.uses == 0) { - std.log.err("fork {f} matched no {s} packages", .{ - fork.path, fork.manifest.name, - }); - any_unused = true; - } else { - std.log.info("fork {f} matched {d} {s} packages", .{ - fork.path, fork.uses, fork.manifest.name, - }); + .global = config, + .parent = null, + }); + + const build_mod = try Package.Module.create(arena, .{ + .paths = .{ + .root = try .fromUnresolved(arena, dirs, &.{build_root.directory.path orelse "."}), + .root_src_path = build_root.build_zig_basename, + }, + .fully_qualified_name = "root.@build", + .cc_argv = &.{}, + .inherited = .{}, + .global = config, + .parent = root_mod, + }); + + if (dev.env.supports(.fetch_command)) { + const fetch_prog_node = root_prog_node.start("Fetch Packages", 0); + defer fetch_prog_node.end(); + + // Reset fork match counts. + for (fork_set.keys()) |*fork| fork.uses = 0; + + var job_queue: Package.Fetch.JobQueue = .{ + .io = io, + .http_client = &http_client, + .global_cache = dirs.global_cache, + .local_storage = &.{ + .cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" }, + .pkg_root = pkg_root, + }, + .recursive = true, + .debug_hash = false, + .unlazy_set = unlazy_set, + .fork_set = fork_set, + .mode = fetch_mode, + .prog_node = fetch_prog_node, + .read_only = system_pkg_dir_path != null, + }; + defer job_queue.deinit(); + + if (system_pkg_dir_path == null) { + try http_client.initDefaultProxies(arena, environ_map); + } + + try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + try job_queue.table.ensureUnusedCapacity(gpa, 1); + + const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory }; + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .relative_path = phantom_package_root }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .remote_package_root = phantom_package_root, + .parent_package_root = phantom_package_root, + .parent_manifest_ast = null, + .prog_node = fetch_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .use_latest_commit = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = true, + .oom_flag = false, + .latest_commit = null, + + .module = build_mod, + }; + + job_queue.all_fetches.appendAssumeCapacity(&fetch); + + job_queue.table.putAssumeCapacityNoClobber( + Package.Fetch.relativePathDigest(phantom_package_root, dirs.global_cache), + &fetch, + ); + + job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); + try job_queue.group.await(io); + + { + // Ensure that forks were actually used. This is done + // before printing manifest errors because using a fork can + // prevent them. + var any_unused = false; + for (fork_set.keys()) |*fork| { + if (fork.uses == 0) { + std.log.err("fork {f} matched no {s} packages", .{ + fork.path, fork.manifest.name, + }); + any_unused = true; + } else { + std.log.info("fork {f} matched {d} {s} packages", .{ + fork.path, fork.uses, fork.manifest.name, + }); + } } + if (any_unused) process.exit(1); } - if (any_unused) process.exit(1); - } - try job_queue.consolidateErrors(); + try job_queue.consolidateErrors(); - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - process.exit(1); - } + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, color) catch {}; + process.exit(1); + } - if (fetch_only) return cleanExit(io); + if (fetch_only) return cleanExit(io); + + var source_buf = std.array_list.Managed(u8).init(gpa); + defer source_buf.deinit(); + try job_queue.createDependenciesSource(&source_buf); + const deps_mod = try createDependenciesModule( + arena, + io, + source_buf.items, + root_mod, + dirs, + config, + ); + + { + // We need a Module for each package's build.zig. + const hashes = job_queue.table.keys(); + const fetches = job_queue.table.values(); + try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); + for (hashes, fetches) |*hash, f| { + if (f == &fetch) { + // The first one is a dummy package for the current project. + continue; + } + if (!f.has_build_zig) + continue; + const hash_slice = hash.toSlice(); + const mod_root_path = try f.package_root.toString(arena); + const m = try Package.Module.create(arena, .{ + .paths = .{ + .root = try .fromUnresolved(arena, dirs, &.{mod_root_path}), + .root_src_path = Package.build_zig_basename, + }, + .fully_qualified_name = try std.fmt.allocPrint( + arena, + "root.@dependencies.{s}", + .{hash_slice}, + ), + .cc_argv = &.{}, + .inherited = .{}, + .global = config, + .parent = root_mod, + }); + const hash_cloned = try arena.dupe(u8, hash_slice); + deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); + f.module = m; + } - var source_buf = std.array_list.Managed(u8).init(gpa); - defer source_buf.deinit(); - try job_queue.createDependenciesSource(&source_buf); - const deps_mod = try createDependenciesModule( + // Each build.zig module needs access to each of its + // dependencies' build.zig modules by name. + for (fetches) |f| { + const mod = f.module orelse continue; + if (!f.have_manifest) continue; + const man = &f.manifest; + const dep_names = man.dependencies.keys(); + try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); + for (dep_names, man.dependencies.values()) |name, dep| { + const dep_digest = Package.Fetch.depDigest( + f.package_root, + dirs.global_cache, + dep, + ) orelse continue; + const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; + const name_cloned = try arena.dupe(u8, name); + mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); + } + } + } + } else try createEmptyDependenciesModule( arena, io, - source_buf.items, root_mod, dirs, config, ); - { - // We need a Module for each package's build.zig. - const hashes = job_queue.table.keys(); - const fetches = job_queue.table.values(); - try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); - for (hashes, fetches) |*hash, f| { - if (f == &fetch) { - // The first one is a dummy package for the current project. - continue; - } - if (!f.has_build_zig) - continue; - const hash_slice = hash.toSlice(); - const mod_root_path = try f.package_root.toString(arena); - const m = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromUnresolved(arena, dirs, &.{mod_root_path}), - .root_src_path = Package.build_zig_basename, + const compile_prog_node = root_prog_node.start("Compile Configure Script", 0); + defer compile_prog_node.end(); + + try root_mod.deps.put(arena, "@build", build_mod); + + var create_diag: Compilation.CreateDiagnostic = undefined; + const comp = Compilation.create(gpa, arena, io, &create_diag, .{ + .libc_installation = libc_installation, + .dirs = dirs, + .root_name = "configure", + .config = config, + .root_mod = root_mod, + .main_mod = build_mod, + .emit_bin = .yes_cache, + .self_exe_path = self_exe_path, + .thread_limit = thread_limit, + .verbose_cc = verbose_cc, + .verbose_link = verbose_link, + .verbose_air = verbose_air, + .verbose_intern_pool = verbose_intern_pool, + .verbose_generic_instances = verbose_generic_instances, + .verbose_llvm_ir = verbose_llvm_ir, + .verbose_llvm_bc = verbose_llvm_bc, + .verbose_llvm_cpu_features = verbose_llvm_cpu_features, + .cache_mode = .whole, + .reference_trace = reference_trace, + .debug_compile_errors = debug_compile_errors, + .environ_map = environ_map, + }) catch |err| switch (err) { + error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), + else => |e| fatal("failed to create compilation: {t}", .{e}), + }; + defer comp.destroy(); + + updateModule(comp, color, compile_prog_node) catch |err| switch (err) { + error.CompileErrorsReported => process.exit(2), + else => |e| return e, + }; + + // Since incremental compilation isn't done yet, we use cache_mode = whole + // above, and thus the output file is already closed. + //try comp.makeBinFileExecutable(); + const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); + const exe_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), + }; + _ = try config_man.addFilePath(exe_path, null); + configure_argv.items[0] = try exe_path.toString(arena); + + switch (cache_poison) { + .pure, .disallowed, .ignored => if (try config_man.hit()) { + const digest = config_man.final(); + break :cp .{ + .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), }, - .fully_qualified_name = try std.fmt.allocPrint( - arena, - "root.@dependencies.{s}", - .{hash_slice}, - ), - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - }); - const hash_cloned = try arena.dupe(u8, hash_slice); - deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); - f.module = m; - } - - // Each build.zig module needs access to each of its - // dependencies' build.zig modules by name. - for (fetches) |f| { - const mod = f.module orelse continue; - if (!f.have_manifest) continue; - const man = &f.manifest; - const dep_names = man.dependencies.keys(); - try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); - for (dep_names, man.dependencies.values()) |name, dep| { - const dep_digest = Package.Fetch.depDigest( - f.package_root, - dirs.global_cache, - dep, - ) orelse continue; - const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; - const name_cloned = try arena.dupe(u8, name); - mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); - } - } + false, + }; + }, + .poisoned => {}, // Don't bother checking for cache hit. } - } else try createEmptyDependenciesModule( - arena, + } + + if (!process.can_spawn) { + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); + } + + const rand_int = randInt(io, u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); + const config_tmp_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = tmp_dir_sub_path, + }; + const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( io, - root_mod, - dirs, - config, - ); - - const compile_prog_node = root_prog_node.start("Compile Configure Script", 0); - defer compile_prog_node.end(); - - try root_mod.deps.put(arena, "@build", build_mod); - - var create_diag: Compilation.CreateDiagnostic = undefined; - const comp = Compilation.create(gpa, arena, io, &create_diag, .{ - .libc_installation = libc_installation, - .dirs = dirs, - .root_name = "configure", - .config = config, - .root_mod = root_mod, - .main_mod = build_mod, - .emit_bin = .yes_cache, - .self_exe_path = self_exe_path, - .thread_limit = thread_limit, - .verbose_cc = verbose_cc, - .verbose_link = verbose_link, - .verbose_air = verbose_air, - .verbose_intern_pool = verbose_intern_pool, - .verbose_generic_instances = verbose_generic_instances, - .verbose_llvm_ir = verbose_llvm_ir, - .verbose_llvm_bc = verbose_llvm_bc, - .verbose_llvm_cpu_features = verbose_llvm_cpu_features, - .cache_mode = .whole, - .reference_trace = reference_trace, - .debug_compile_errors = debug_compile_errors, - .environ_map = environ_map, - }) catch |err| switch (err) { - error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - else => |e| fatal("failed to create compilation: {t}", .{e}), - }; - defer comp.destroy(); - - updateModule(comp, color, compile_prog_node) catch |err| switch (err) { - error.CompileErrorsReported => process.exit(2), - else => |e| return e, - }; - - // Since incremental compilation isn't done yet, we use cache_mode = whole - // above, and thus the output file is already closed. - //try comp.makeBinFileExecutable(); - const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); - const exe_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), - }; - _ = try config_man.addFilePath(exe_path, null); - configure_argv.items[0] = try exe_path.toString(arena); - - switch (cache_poison) { - .pure, .disallowed, .ignored => if (try config_man.hit()) { - const digest = config_man.final(); - break :cp .{ - .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), - }, - false, - }; - }, - .poisoned => {}, // Don't bother checking for cache hit. - } - } - - if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); - } - - const rand_int = randInt(io, u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - const config_tmp_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = tmp_dir_sub_path, - }; - const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( - io, - config_tmp_path.sub_path, - .{ .read = true, .exclusive = true }, - ); - defer config_tmp_file.close(io); - - const term = term: { - const child_node = root_prog_node.start("Run Configure Script", 0); - defer child_node.end(); - var child = std.process.spawn(io, .{ - .argv = configure_argv.items, - .stdout = .{ .file = config_tmp_file }, - .progress_node = child_node, - }) catch |err| fatal("failed to spawn configure script {s}: {t}", .{ configure_argv.items[0], err }); - defer child.kill(io); - break :term child.wait(io) catch |err| - fatal("failed to wait configure script {s}: {t}", .{ configure_argv.items[0], err }); - }; - if (!term.success()) { - // Failure to produce the configuration file. - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following configure command {f}:\n{s}", .{ term, cmd }); - } - // Even though the file is designed to be sent directly to make - // runner, we must load it now because: - // * If it contains additional file dependencies, we need to - // add them to `config_man` before obtaining the final digest. - // * If it contains a set of lazy packages that need to be - // fetched, we need to fetch those now and re-run configure. - var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| - fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); - - if (configuration.unlazy_deps.len != 0) { - if (!dev.env.supports(.fetch_command)) process.exit(1); - var any_errors = false; - for (configuration.unlazy_deps) |hash_string| { - const hash = hash_string.slice(&configuration); - assert(hash.len != 0); - if (hash.len > Package.Hash.max_len) { - std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ hash.len, hash }); - any_errors = true; - continue; - } - try unlazy_set.put(arena, .fromSlice(hash), {}); - } - if (any_errors) process.exit(1); - if (system_pkg_dir_path) |p| { - // In this mode, the system needs to provide these packages; they - // cannot be fetched by Zig. - const s = fs.path.sep_str; - for (unlazy_set.keys()) |*hash| { - std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); - } - std.log.info("remote package fetching disabled due to --system mode", .{}); - std.log.info("dependencies might be avoidable depending on build configuration", .{}); - process.exit(1); - } - continue :cp; - } - - for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { - const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; - try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); - } - - // If it is poisoned, there is no point in moving it to cached - // location. Just leave it in the tmp directory. - if (configuration.poisoned) { - break :cp .{ config_tmp_path, true }; - } else { - const digest = config_man.final(); - const final_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), - }; - Io.Dir.rename( - config_tmp_path.root_dir.handle, config_tmp_path.sub_path, - final_path.root_dir.handle, - final_path.sub_path, - io, - ) catch |err| { - fatal("failed to rename configuration file from {f} into {f}: {t}", .{ - config_tmp_path, final_path, err, - }); + .{ .read = true, .exclusive = true }, + ); + defer config_tmp_file.close(io); + + const term = term: { + const child_node = root_prog_node.start("Run Configure Script", 0); + defer child_node.end(); + var child = std.process.spawn(io, .{ + .argv = configure_argv.items, + .stdout = .{ .file = config_tmp_file }, + .progress_node = child_node, + }) catch |err| fatal("failed to spawn configure script {s}: {t}", .{ configure_argv.items[0], err }); + defer child.kill(io); + break :term child.wait(io) catch |err| + fatal("failed to wait configure script {s}: {t}", .{ configure_argv.items[0], err }); }; - config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); - break :cp .{ final_path, false }; + if (!term.success()) { + // Failure to produce the configuration file. + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following configure command {f}:\n{s}", .{ term, cmd }); + } + // Even though the file is designed to be sent directly to make + // runner, we must load it now because: + // * If it contains additional file dependencies, we need to + // add them to `config_man` before obtaining the final digest. + // * If it contains a set of lazy packages that need to be + // fetched, we need to fetch those now and re-run configure. + var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); + + if (configuration.unlazy_deps.len != 0) { + if (!dev.env.supports(.fetch_command)) process.exit(1); + var any_errors = false; + for (configuration.unlazy_deps) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); + if (hash.len > Package.Hash.max_len) { + std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ hash.len, hash }); + any_errors = true; + continue; + } + try unlazy_set.put(arena, .fromSlice(hash), {}); + } + if (any_errors) process.exit(1); + if (system_pkg_dir_path) |p| { + // In this mode, the system needs to provide these packages; they + // cannot be fetched by Zig. + const s = fs.path.sep_str; + for (unlazy_set.keys()) |*hash| { + std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); + } + std.log.info("remote package fetching disabled due to --system mode", .{}); + std.log.info("dependencies might be avoidable depending on build configuration", .{}); + process.exit(1); + } + continue :cp; + } + + for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { + const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; + try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); + } + + // If it is poisoned, there is no point in moving it to cached + // location. Just leave it in the tmp directory. + if (configuration.poisoned) { + break :cp .{ config_tmp_path, true }; + } else { + const digest = config_man.final(); + const final_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + }; + Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + ) catch |err| { + fatal("failed to rename configuration file from {f} into {f}: {t}", .{ + config_tmp_path, final_path, err, + }); + }; + config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + break :cp .{ final_path, false }; + } + }; + + { + // Release all file system locks just before running the maker process. + var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; + defer if (configuration_lock) |*l| l.release(io); + + const make_runner = make_runner_task.await(io) catch |err| fatal("failed compiling maker: {t}", .{err}); + + make_argv.items[0] = try make_runner.exe_path.toString(arena); + make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); } - }; - - { - // Release all file system locks just before running the maker process. - var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; - defer if (configuration_lock) |*l| l.release(io); - - const make_runner = make_runner_task.await(io) catch |err| fatal("failed compiling maker: {t}", .{err}); - - make_argv.items[0] = try make_runner.exe_path.toString(arena); - make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); } if (!process.can_spawn) { -- 2.54.0 From d1a446203222bb5bbd1d46139b93a13367c1e8bb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 24 May 2026 22:51:14 -0700 Subject: [PATCH 162/179] Maker.Step.InstallArtifact: use gpa for fs walking --- lib/compiler/Maker/Step/InstallArtifact.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/compiler/Maker/Step/InstallArtifact.zig b/lib/compiler/Maker/Step/InstallArtifact.zig index 00d787e3eb554724ca42226397d1f260e91dfeff..6f90157c38683adc7512a643f813e8ef1dc9d315 100644 --- a/lib/compiler/Maker/Step/InstallArtifact.zig +++ b/lib/compiler/Maker/Step/InstallArtifact.zig @@ -19,6 +19,7 @@ pub fn make( const step = maker.stepByIndex(step_index); const conf = &maker.scanned_config.configuration; const graph = maker.graph; + const gpa = maker.gpa; const arena = graph.arena; // TODO don't leak into process arena const io = graph.io; const conf_step = step_index.ptr(conf); @@ -100,7 +101,8 @@ pub fn make( }; defer src_dir.close(io); - var it = try src_dir.walk(arena); + var it = try src_dir.walk(gpa); + defer it.deinit(); next_entry: while (it.next(io) catch |err| switch (err) { error.Canceled, error.OutOfMemory => |e| return e, else => |e| return step.fail(maker, "failed to iterate directory {f}: {t}", .{ src_dir_path, e }), -- 2.54.0 From 65c96c403589607f882468891c4e2c329a28fde9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 24 May 2026 23:23:24 -0700 Subject: [PATCH 163/179] Maker.Step.Run: add missing PATH entries for DLLs --- lib/compiler/Maker/Step/Run.zig | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index c72db4eb2603eb5f3cbf98eb3b27b409713ebef9..e8c90424b1fbfa2a4ec1a4f4890cbc1edc5ef1ed 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -1782,6 +1782,20 @@ fn runCommand( } else { try environ_map.putAll(&graph.environ_map); } + + // Now that we have the environ map, we might need to mutate it to insert + // .dll search paths because Windows doesn't have rpaths. + const arg0 = conf_run.args.slice[0].get(conf); + if (arg0.producer.value) |producer_index| { + const producer_step = producer_index.ptr(conf); + const producer = producer_step.extended.get(conf.extra).compile; + const root_module = producer.root_module.get(conf); + const root_module_target = root_module.resolved_target.get(conf).?.result.get(conf); + if (root_module_target.flags.os_tag == .windows) { + try addPathForDynLibs(maker, producer_index, environ_map, argv[0]); + } + } + try graph.handleVerbose(cwd, environ_map, argv); const opt_generic_result = spawnChildAndCollect( @@ -1802,7 +1816,6 @@ fn runCommand( // relying on it being a Compile step. This will make this logic // work even for the edge case that the binary was produced by a // third party. - const arg0 = conf_run.args.slice[0].get(conf); const producer_index = arg0.producer.value orelse break :interpret; const producer_step = producer_index.ptr(conf); const producer = producer_step.extended.get(conf.extra).compile; @@ -1926,11 +1939,6 @@ fn runCommand( }, } - if (root_target.os.tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - try addPathForDynLibs(maker, producer_index, environ_map, argv[0]); - } - gpa.free(step.result_failed_command.?); step.result_failed_command = null; try graph.handleVerbose(cwd, environ_map, interp_argv.items); -- 2.54.0 From ea151030bc4a1366ef3ca0bd1f59d4867a215762 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 12:07:21 -0700 Subject: [PATCH 164/179] std.Io.Writer: implement {q} formatter This is intended to replace the common pattern: print("invalid foobar: '{s}': {t}", ...) The idea is to not invent imaginary syntax. If the string contained single quotes for example, this would be a nonsensical error message. On the other hand with the new pattern: print("invalid foobar: {q}: {t}", ...) It's both easier on the eyes at the print site, and also it will allow the user to copy paste a properly escaped string, should the quoted text contain any odd characters, including invisible ones like null bytes. --- lib/std/Io/Writer.zig | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index c489a5c93d6ef3b4adcf94a2eca16daff8a56eee..2aef4f8826aaa7a6657c4e41bba675f6eb400179 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -1172,23 +1172,12 @@ pub fn printValue( }, else => invalidFmtError(fmt, value), }, - // TODO make this print double quotes and quote-escape the string - // according to zig string syntax rules 'q' => switch (@typeInfo(T)) { .pointer => |info| switch (info.size) { - .one, .slice => { - const slice: []const u8 = value; - return w.alignBufferOptions(slice, options); - }, - .many, .c => { - const slice: [:0]const u8 = std.mem.span(value); - return w.alignBufferOptions(slice, options); - }, - }, - .array => { - const slice: []const u8 = &value; - return w.alignBufferOptions(slice, options); + .one, .slice => return printStringEscaped(w, value), + .many, .c => return printStringEscaped(w, std.mem.span(value)), }, + .array => return printStringEscaped(w, &value), else => invalidFmtError(fmt, value), }, 'B' => switch (@typeInfo(T)) { @@ -1467,6 +1456,14 @@ fn printEnumNonexhaustive(w: *Writer, value: anytype) Error!void { try w.writeByte(')'); } +/// Prints a double quote, then escapes a string according to Zig string +/// literal rules, then a double quote. +pub fn printStringEscaped(w: *Writer, bytes: []const u8) Error!void { + try w.writeByte('"'); + try std.zig.stringEscape(bytes, w); + try w.writeByte('"'); +} + pub fn printVector( w: *Writer, comptime fmt: []const u8, @@ -2121,6 +2118,11 @@ test "printFloat with comptime_float" { try testing.expectFmt("1", "{}", .{1.0}); } +test "{q} format string" { + const data: []const u8 = "i\tlike\"cheese\x00\x05cheese"; + try testing.expectFmt("hello \"i\\tlike\\\"cheese\\x00\\x05cheese\" world", "hello {q} world", .{data}); +} + fn testPrintIntCase(expected: []const u8, value: anytype, base: u8, case: std.fmt.Case, options: std.fmt.Options) !void { var buffer: [100]u8 = undefined; var w: Writer = .fixed(&buffer); -- 2.54.0 From 9eb85c4e5eb65ae987463af33ca24225f12b3a8b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 12:51:23 -0700 Subject: [PATCH 165/179] build system: track TODOs outside source code related to #363 --- build.zig | 2 -- lib/compiler/Maker.zig | 8 +++----- lib/compiler/Maker/Graph.zig | 2 +- lib/compiler/Maker/Step.zig | 4 ++-- lib/compiler/Maker/Step/InstallDir.zig | 1 - lib/compiler/Maker/Step/Run.zig | 20 ++++++++++++-------- lib/compiler/Maker/WebServer.zig | 14 -------------- lib/std/Build.zig | 6 +++++- lib/std/Build/Configuration.zig | 4 ++-- lib/std/zig.zig | 8 +++----- 10 files changed, 28 insertions(+), 41 deletions(-) diff --git a/build.zig b/build.zig index bbfb24c4b60579b4a0f78cb572b16a488060b530..2f00911197098320c4fcdbb69547d381b1492343 100644 --- a/build.zig +++ b/build.zig @@ -1525,8 +1525,6 @@ fn generateLangRef(b: *std.Build) !std.Build.LazyPath { cmd.addArg("--zig"); cmd.addFileArg(.zig_exe); - // TODO: enhance doctest to use "--listen=-" rather than operating in a - // temporary directory cmd.addArg("--cache-root"); cmd.addDirectoryArg(.cache_root); diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 0547d5fa31043cfbe1ec8f8cfd65434684fda6b9..643df6b68fa576c3cd75e46ff570e1c5920ebc85 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -184,7 +184,6 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--sysroot")) { graph.sysroot = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--maxrss")) { - // TODO refactor and reuse the fuzz number parsing here const max_rss_text = nextArgOrFatal(args, &arg_idx); max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| fatal("invalid byte size {q}: {t}", .{ max_rss_text, err }); @@ -269,7 +268,6 @@ pub fn main(init: process.Init.Minimal) !void { graph.build_id = std.zig.BuildId.parse(style) catch |err| fatal("unable to parse --build-id style {q}: {t}", .{ style, err }); } else if (mem.eql(u8, arg, "--debounce")) { - // TODO refactor and reuse the timeout parsing code also here const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected u16 after {q}", .{arg}); debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { @@ -1887,7 +1885,7 @@ pub fn truncatePath( ) Step.ExtendedMakeError!void { const graph = maker.graph; const io = graph.io; - if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ + if (graph.verbose) try graph.handleVerbose(null, null, &.{ "truncate", try dest_path.toString(arena), }); const err = e: { @@ -1924,7 +1922,7 @@ pub fn installPath( ) Step.ExtendedMakeError!Dir.PrevStatus { const graph = maker.graph; const io = graph.io; - if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ + if (graph.verbose) try graph.handleVerbose(null, null, &.{ "install", "-C", try src_path.toString(arena), try dest_path.toString(arena), }); return Dir.updateFile( @@ -1949,7 +1947,7 @@ pub fn installDir( ) Step.ExtendedMakeError!Dir.CreatePathStatus { const graph = maker.graph; const io = graph.io; - if (graph.verbose) try graph.handleVerbose(.inherit, null, &.{ + if (graph.verbose) try graph.handleVerbose(null, null, &.{ "install", "-d", try dest_path.toString(arena), }); return dest_path.root_dir.handle.createDirPathStatus(io, dest_path.sub_path, .default_dir) catch |err| { diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index bca008ffc04ceb84cf49b8937b393950b1689080..116132c614e740ed7159c276141759d2f0e8f1ab 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -69,7 +69,7 @@ enable_rosetta: bool = false, /// before spawning them. pub fn handleVerbose( graph: *const Graph, - cwd: std.process.Child.Cwd, + cwd: ?[]const u8, opt_env: ?*const std.process.Environ.Map, argv: []const []const u8, ) error{OutOfMemory}!void { diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 66e6079ab3950edd5aa9d4f0d358aeed6a2d18e2..bb65abea0e9ac29c6c089f62c9d3ddfeaa8a109a 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -341,7 +341,7 @@ pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcess s.result_failed_command = try std.zig.allocPrintCmd(gpa, options.argv, .{}); try handleChildProcUnsupported(s, maker); - try graph.handleVerbose(.inherit, null, options.argv); + try graph.handleVerbose(null, null, options.argv); const result = std.process.run(arena, io, .{ .argv = options.argv, @@ -459,7 +459,7 @@ pub fn evalZigProcess( assert(argv.len != 0); try handleChildProcUnsupported(s, maker); - try graph.handleVerbose(.inherit, null, argv); + try graph.handleVerbose(null, null, argv); const zp = try gpa.create(ZigProcess); defer if (!watch) gpa.destroy(zp); diff --git a/lib/compiler/Maker/Step/InstallDir.zig b/lib/compiler/Maker/Step/InstallDir.zig index 484bebf329c791d65e121bcfa2297d8c79843283..d8b7b7ed1881f23ed502f85009a3cec428ff43b2 100644 --- a/lib/compiler/Maker/Step/InstallDir.zig +++ b/lib/compiler/Maker/Step/InstallDir.zig @@ -81,7 +81,6 @@ pub fn make( .file => { for (blank_extensions) |ext| { if (endsWith(u8, entry.path, ext.slice(conf))) { - // TODO check if the file was already there and length 0 try maker.truncatePath(arena, dest_path, step_index); continue :next_entry; } diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index e8c90424b1fbfa2a4ec1a4f4890cbc1edc5ef1ed..5c17a637a41b146349bf469b21762f91d3a71dee 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -1796,7 +1796,12 @@ fn runCommand( } } - try graph.handleVerbose(cwd, environ_map, argv); + const cwd_string = switch (cwd) { + .path => |p| p, + .dir => unreachable, + .inherit => null, + }; + try graph.handleVerbose(cwd_string, environ_map, argv); const opt_generic_result = spawnChildAndCollect( run_index, @@ -1812,10 +1817,6 @@ fn runCommand( error.InvalidExe, // cpu arch mismatch error.FileNotFound, // can happen with a wrong dynamic linker path => interpret: { - // TODO: learn the target from the binary directly rather than from - // relying on it being a Compile step. This will make this logic - // work even for the edge case that the binary was produced by a - // third party. const producer_index = arg0.producer.value orelse break :interpret; const producer_step = producer_index.ptr(conf); const producer = producer_step.extended.get(conf.extra).compile; @@ -1829,7 +1830,6 @@ fn runCommand( const root_target = std.zig.system.resolveTargetQuery(io, other_target_query) catch unreachable; const link_libc = maker.stepByIndex(producer_index).extended.compile.is_linking_libc; - // TODO get this from the parent process instead const host: std.Target = std.zig.system.resolveTargetQuery(io, .{}) catch |he| switch (he) { error.Canceled => |e| return e, else => builtin.target, @@ -1941,7 +1941,7 @@ fn runCommand( gpa.free(step.result_failed_command.?); step.result_failed_command = null; - try graph.handleVerbose(cwd, environ_map, interp_argv.items); + try graph.handleVerbose(cwd_string, environ_map, interp_argv.items); break :term spawnChildAndCollect( run_index, @@ -2148,7 +2148,11 @@ fn spawnChildAndCollect( // If an error occurs, it's caused by this command: assert(step.result_failed_command == null); step.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{ - .cwd = child_cwd, + .cwd = switch (child_cwd) { + .path => |p| p, + .dir => unreachable, + .inherit => null, + }, .child_env = environ_map, .parent_env = &graph.environ_map, }); diff --git a/lib/compiler/Maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig index de30ca30730b0d859dd599ce6165344d606e1281..943265c9397eb1ce20738171557e9f53134477c7 100644 --- a/lib/compiler/Maker/WebServer.zig +++ b/lib/compiler/Maker/WebServer.zig @@ -225,7 +225,6 @@ pub fn updateStepStatus( ) void { const maker = ws.maker; const all_steps = maker.step_stack.keys(); - // TODO don't do linear search, especially in a hot loop like this const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == step_index) break @intCast(i); } else unreachable; @@ -569,16 +568,6 @@ pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []cons var read_buffer: [1024]u8 = undefined; var file_reader: Io.File.Reader = .initSize(file, io, &read_buffer, stat.size); - // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can - // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI: - // it turns out the WASM treats the first path component as the module name, typically - // resulting in modules named "" and "src". The compiler needs to tell the build system - // about the module graph so that the build system can correctly encode this information in - // the tar file. - // - // Additionally, this needs to ensure that all path separators for both prefix and - // sub_path are using the POSIX-style `/` on platforms that don't use it as their native - // path separator. archiver.prefix = path.root_dir.path orelse graph.cache.cwd; try archiver.writeFile(path.sub_path, &file_reader, @intCast(stat.mtime.toSeconds())); } @@ -799,7 +788,6 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { const io = maker.graph.io; const all_steps = maker.step_stack.keys(); - // TODO don't do linear search const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == opts.compile_step) break @intCast(i); } else unreachable; @@ -843,7 +831,6 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.In const io = maker.graph.io; const all_steps = maker.step_stack.keys(); - // TODO don't do linear search const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == step_index) break @intCast(i); } else unreachable; @@ -882,7 +869,6 @@ pub fn updateTimeReportRunTest( const io = maker.graph.io; const all_steps = maker.step_stack.keys(); - // TODO don't do linear search const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == run_step_index) break @intCast(i); } else unreachable; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 3b31500f2cda7ce5c7d896dce507cd507f513895..d8948380de3edb59703ea2a3dc90b95fa2e89f6f 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1890,7 +1890,11 @@ pub fn runFallible(b: *Build, argv: []const []const u8, options: RunOptions) Run const arena = graph.arena; const print_opts: std.zig.AllocPrintCmdOptions = .{ - .cwd = options.cwd, + .cwd = switch (options.cwd) { + .inherit => null, + .path => |p| p, + .dir => null, // Unknown without changing function signature of runFallible. + }, .child_env = options.environ_map, .parent_env = &graph.environ_map, }; diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 5041fbe4eff7c93e0b63f8eb5f948e368c534782..a8934415755bed46f7c17bd9addd925ffaddde22 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -3099,7 +3099,7 @@ pub const Storage = enum { .value = if (match) dataField(buffer, i, container, Field.Value) else null, }; }, - .extended => @compileError("TODO"), + .extended => @compileError("unimplemented"), .length_prefixed_list => { const n = @divExact(@sizeOf(Field.Elem), @sizeOf(u32)); const data_start = i.* + 1; @@ -3260,7 +3260,7 @@ pub const Storage = enum { .flag_union => return switch (value.u) { inline else => |x| setExtraField(buffer, i, @TypeOf(x), x), }, - .extended => @compileError("TODO"), + .extended => @compileError("unimplemented"), .flag_length_prefixed_list => { const len: u32 = @intCast(value.slice.len); if (len == 0) return 0; // Flag bit hides the length prefix. diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 2a3b4792a826e5fc28ffe9373c73bb7b6f63f6b1..db5e20c8d172c5d3701a077e2a913bd465653da6 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -1168,7 +1168,7 @@ pub const ClangCliParam = struct { }; pub const AllocPrintCmdOptions = struct { - cwd: std.process.Child.Cwd = .inherit, + cwd: ?[]const u8 = null, parent_env: ?*const std.process.Environ.Map = null, child_env: ?*const std.process.Environ.Map = null, }; @@ -1212,10 +1212,8 @@ pub fn allocPrintCmd(gpa: Allocator, argv: []const []const u8, options: AllocPri var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); const writer = &aw.writer; - switch (options.cwd) { - .inherit => {}, - .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, - .dir => @panic("TODO"), + if (options.cwd) |path| { + writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory; } if (options.child_env) |child_env| { for (child_env.keys(), child_env.values()) |key, value| { -- 2.54.0 From 1aa65d094ec737a140d7ab60e1b2af62144edd6a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 13:23:10 -0700 Subject: [PATCH 166/179] Maker.Step.Run: leak into global arena less There are still some uses: - fuzzing - generated paths (will require adjusting all step logic) - Step.result_stderr --- lib/compiler/Maker/Step.zig | 2 +- lib/compiler/Maker/Step/Run.zig | 148 +++++++++++++++----------------- 2 files changed, 70 insertions(+), 80 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index bb65abea0e9ac29c6c089f62c9d3ddfeaa8a109a..e9669a9118a30f91a1eee73ac95b90e2e0cce53f 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -73,7 +73,7 @@ comptime { // Common cache line size is 128. This check prevents accidentally crossing // an additional cache line. In the future it might be nice to try to fit // this struct in 128 bytes or less. - assert(@sizeOf(@This()) <= 128 * 4); + assert(@sizeOf(@This()) <= 128 * 3); } pub const Extended = union(enum) { diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 5c17a637a41b146349bf469b21762f91d3a71dee..ca3bca666556032378f371e66c9d1ecc199e5a95 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -13,6 +13,7 @@ const assert = std.debug.assert; const mem = std.mem; const process = std.process; const allocPrint = std.fmt.allocPrint; +const Allocator = std.mem.Allocator; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); @@ -28,13 +29,6 @@ cached_test_metadata: ?CachedTestMetadata = null, /// executable that contains fuzz tests. rebuilt_executable: ?Path = null, -/// Persisted to reuse memory on subsequent calls to `make`. -argv: std.ArrayList([]const u8) = .empty, -/// Persisted to reuse memory on subsequent calls to `make`. -output_placeholders: std.ArrayList(IndexedOutput) = .empty, -/// Persisted to reuse memory on subsequent calls to `make`. -environ_map: std.process.Environ.Map = .{ .array_hash_map = .empty, .allocator = undefined }, - pub fn make( run: *Run, run_index: Configuration.Step.Index, @@ -45,16 +39,17 @@ pub fn make( const gpa = maker.gpa; const step = maker.stepByIndex(run_index); const io = graph.io; - const arena = graph.arena; // TODO don't leak into the process arena const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); const conf_run = conf_step.extended.get(conf.extra).run; - const argv_list = &run.argv; - const output_placeholders = &run.output_placeholders; const cache_root = graph.local_cache_root; - argv_list.clearRetainingCapacity(); - output_placeholders.clearRetainingCapacity(); + var arena_allocator: std.heap.ArenaAllocator = .init(gpa); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); + + var argv_list: std.ArrayList([]const u8) = .empty; + var output_placeholders: std.ArrayList(IndexedOutput) = .empty; var man = graph.cache.obtain(); defer man.deinit(); @@ -89,7 +84,7 @@ pub fn make( const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ - prefix, try convertPathArg(run_index, maker, file_path), suffix, + prefix, try convertPathArg(arena, run_index, maker, file_path), suffix, })); man.hash.addBytesZ(prefix); man.hash.addBytesZ(suffix); @@ -100,7 +95,7 @@ pub fn make( const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); const resolved_arg = try mem.concat(arena, u8, &.{ - prefix, try convertPathArg(run_index, maker, file_path), suffix, + prefix, try convertPathArg(arena, run_index, maker, file_path), suffix, }); argv_list.appendAssumeCapacity(resolved_arg); man.hash.addBytes(resolved_arg); @@ -144,7 +139,7 @@ pub fn make( const file_path = producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?).*; argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ - prefix, try convertPathArg(run_index, maker, file_path), suffix, + prefix, try convertPathArg(arena, run_index, maker, file_path), suffix, })); _ = try man.addFilePath(file_path, null); @@ -183,7 +178,7 @@ pub fn make( man.hash.add(conf_run.flags.test_runner_mode); if (conf_run.flags.test_runner_mode) { - const cache_dir_string = try convertPathArg(run_index, maker, .{ .root_dir = cache_root }); + const cache_dir_string = try convertPathArg(arena, run_index, maker, .{ .root_dir = cache_root }); try argv_list.ensureUnusedCapacity(gpa, 3); argv_list.appendAssumeCapacity(try allocPrint(arena, "--cache-dir={s}", .{cache_dir_string})); @@ -259,8 +254,8 @@ pub fn make( const digest = if (has_side_effects) man.hash.final() else man.final(); const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); - try populateGeneratedPathsCreateDirs(run, run_index, maker, output_dir_path); - try runCommand(run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); + try populateGeneratedPathsCreateDirs(arena, run_index, maker, output_dir_path, output_placeholders.items, argv_list.items); + try runCommand(arena, run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); return; } @@ -270,8 +265,8 @@ pub fn make( io.random(@ptrCast(&rand_int)); const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - try populateGeneratedPathsCreateDirs(run, run_index, maker, tmp_dir_path); - try runCommand(run, run_index, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null); + try populateGeneratedPathsCreateDirs(arena, run_index, maker, tmp_dir_path, output_placeholders.items, argv_list.items); + try runCommand(arena, run, run_index, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null); for (output_placeholders.items) |placeholder| { const arg = placeholder.arg_index.get(conf); @@ -341,6 +336,7 @@ pub fn make( /// * A test (or a response from the test runner) times out /// * The wait fails, indicating the child closed stdout and stderr fn waitZigTest( + arena: Allocator, run: *Run, run_index: Configuration.Step.Index, maker: *Maker, @@ -363,7 +359,6 @@ fn waitZigTest( const graph = maker.graph; const gpa = maker.gpa; const io = graph.io; - const arena = graph.arena; // TODO don't leak into the process arena const step = maker.stepByIndex(run_index); var sub_prog_node: ?std.Progress.Node = null; @@ -715,7 +710,7 @@ const FuzzTestRunner = struct { } } - fn listen(f: *FuzzTestRunner) !void { + fn listen(f: *FuzzTestRunner, arena: Allocator) !void { const maker = f.ctx.fuzz.maker; const graph = maker.graph; const io = graph.io; @@ -739,7 +734,7 @@ const FuzzTestRunner = struct { else => |read_e| return read_e, }), 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { - error.EndOfStream => return f.instanceEos(id), + error.EndOfStream => return f.instanceEos(arena, id), else => |read_e| return read_e, }), else => unreachable, @@ -901,7 +896,7 @@ const FuzzTestRunner = struct { } }); } - fn instanceEos(f: *FuzzTestRunner, id: u32) !void { + fn instanceEos(f: *FuzzTestRunner, arena: Allocator, id: u32) !void { const maker = f.ctx.fuzz.maker; const instance = &f.instances[id]; const run_index = f.run_index; @@ -914,7 +909,7 @@ const FuzzTestRunner = struct { instance.child.stdin = null; const term = try instance.child.wait(io); if (!termMatches(.{ .exited = 0 }, term)) { - step.result_stderr = try f.mergedStderr(); + step.result_stderr = try f.mergedStderr(arena); try f.saveCrash(id, term); return step.fail(maker, "test process unexpectedly {f}", .{fmtTerm(term)}); } @@ -1057,11 +1052,7 @@ const FuzzTestRunner = struct { } } - fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { - const maker = f.ctx.fuzz.maker; - const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into the process arena - + fn mergedStderr(f: *FuzzTestRunner, arena: Allocator) Allocator.Error![]const u8 { // Collect any available stderr while (f.batch.next()) |completion| { if (completion.index % 3 != 2) continue; @@ -1092,12 +1083,13 @@ fn evalFuzzTest( var f: FuzzTestRunner = try .init(run, run_index, fuzz_context, progress_node, spawn_options); defer f.deinit(); try f.startInstances(); - try f.listen(); + try f.listen(fuzz_context.fuzz.maker.graph.arena); } const StdioPollEnum = enum { stdout, stderr }; fn evalZigTest( + arena: Allocator, run: *Run, run_index: Configuration.Step.Index, maker: *Maker, @@ -1113,7 +1105,6 @@ fn evalZigTest( const graph = maker.graph; const gpa = maker.gpa; const io = graph.io; - const arena = graph.arena; // TODO don't leak into the process arena const step = maker.stepByIndex(run_index); // We will update this every time a child runs. @@ -1146,6 +1137,7 @@ fn evalZigTest( }; switch (try waitZigTest( + arena, run, run_index, maker, @@ -1380,13 +1372,13 @@ fn sendRunFuzzTestMessage( } fn evalGeneric( + arena: Allocator, run_index: Configuration.Step.Index, maker: *Maker, spawn_options: process.SpawnOptions, ) !EvalGenericResult { const graph = maker.graph; const io = graph.io; - const arena = graph.arena; // TODO don't leak into the process arena const gpa = maker.gpa; const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); @@ -1520,15 +1512,17 @@ pub fn rerunInFuzzMode( const graph = maker.graph; const step = maker.stepByIndex(run_index); const io = graph.io; - const arena = graph.arena; // TODO don't leak into the process arena const gpa = maker.gpa; const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); const conf_run = conf_step.extended.get(conf.extra).run; - const argv_list = &run.argv; const cache_root = graph.local_cache_root; - argv_list.clearRetainingCapacity(); + var arena_allocator: std.heap.ArenaAllocator = .init(gpa); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); + + var argv_list: std.ArrayList([]const u8) = .empty; for (conf_run.args.slice) |arg_index| { const arg = arg_index.get(conf); @@ -1543,7 +1537,7 @@ pub fn rerunInFuzzMode( const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ - prefix, try convertPathArg(run_index, maker, file_path), suffix, + prefix, try convertPathArg(arena, run_index, maker, file_path), suffix, })); }, .path_directory => { @@ -1551,7 +1545,7 @@ pub fn rerunInFuzzMode( const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); const resolved_arg = try mem.concat(arena, u8, &.{ - prefix, try convertPathArg(run_index, maker, file_path), suffix, + prefix, try convertPathArg(arena, run_index, maker, file_path), suffix, }); argv_list.appendAssumeCapacity(resolved_arg); }, @@ -1593,7 +1587,7 @@ pub fn rerunInFuzzMode( producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?).*; argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ - prefix, try convertPathArg(run_index, maker, file_path), suffix, + prefix, try convertPathArg(arena, run_index, maker, file_path), suffix, })); }, .output_file => unreachable, @@ -1603,7 +1597,7 @@ pub fn rerunInFuzzMode( } if (conf_run.flags.test_runner_mode) { - const cache_dir_string = try convertPathArg(run_index, maker, .{ .root_dir = cache_root }); + const cache_dir_string = try convertPathArg(arena, run_index, maker, .{ .root_dir = cache_root }); try argv_list.ensureUnusedCapacity(gpa, 3); argv_list.appendAssumeCapacity(try allocPrint(arena, "--cache-dir={s}", .{cache_dir_string})); @@ -1620,7 +1614,7 @@ pub fn rerunInFuzzMode( var rand_int: u64 = undefined; io.random(@ptrCast(&rand_int)); const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - try runCommand(run, run_index, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{ + try runCommand(arena, run, run_index, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{ .fuzz = fuzz, }); } @@ -1633,13 +1627,12 @@ fn populateGeneratedPaths( ) !void { const conf = &maker.scanned_config.configuration; const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into the process arena for (output_placeholders) |placeholder| { const arg = placeholder.arg_index.get(conf); maker.generatedPath(arg.generated.value.?).* = .{ .root_dir = cache_root, - .sub_path = try Dir.path.join(arena, &.{ + .sub_path = try Dir.path.join(graph.arena, &.{ "o", digest, arg.basename.value.?.slice(conf), }), }; @@ -1647,20 +1640,20 @@ fn populateGeneratedPaths( } fn populateGeneratedPathsCreateDirs( - run: *Run, + arena: Allocator, run_index: Configuration.Step.Index, maker: *Maker, output_dir_path: []const u8, + output_placeholders: []const IndexedOutput, + argv: [][]const u8, ) !void { const step = maker.stepByIndex(run_index); const conf = &maker.scanned_config.configuration; const graph = maker.graph; const io = graph.io; - const arena = graph.arena; // TODO don't leak into the process arena const cache_root = graph.local_cache_root; - const argv = run.argv.items; - for (run.output_placeholders.items) |placeholder| { + for (output_placeholders) |placeholder| { const arg = placeholder.arg_index.get(conf); const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; @@ -1668,7 +1661,7 @@ fn populateGeneratedPathsCreateDirs( const generated_path: Path = .{ .root_dir = cache_root, - .sub_path = try Dir.path.join(arena, &.{ output_dir_path, basename }), + .sub_path = try Dir.path.join(graph.arena, &.{ output_dir_path, basename }), }; const create_path: Path = .{ .root_dir = cache_root, @@ -1683,7 +1676,7 @@ fn populateGeneratedPathsCreateDirs( maker.generatedPath(arg.generated.value.?).* = generated_path; - const arg_output_path = try convertPathArg(run_index, maker, generated_path); + const arg_output_path = try convertPathArg(arena, run_index, maker, generated_path); argv[placeholder.index] = try mem.concat(arena, u8, &.{ prefix, arg_output_path, suffix }); } } @@ -1696,12 +1689,11 @@ fn populateGeneratedStdIo( ) !void { const conf = &maker.scanned_config.configuration; const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into the process arena if (conf_run.captured_stdout.value) |captured| { maker.generatedPath(captured.generated_file).* = .{ .root_dir = cache_root, - .sub_path = try Dir.path.join(arena, &.{ + .sub_path = try Dir.path.join(graph.arena, &.{ "o", digest, captured.basename.slice(conf), }), }; @@ -1710,7 +1702,7 @@ fn populateGeneratedStdIo( if (conf_run.captured_stderr.value) |captured| { maker.generatedPath(captured.generated_file).* = .{ .root_dir = cache_root, - .sub_path = try Dir.path.join(arena, &.{ + .sub_path = try Dir.path.join(graph.arena, &.{ "o", digest, captured.basename.slice(conf), }), }; @@ -1736,6 +1728,7 @@ const FuzzContext = struct { }; fn runCommand( + arena: Allocator, run: *Run, run_index: Configuration.Step.Index, maker: *Maker, @@ -1746,7 +1739,6 @@ fn runCommand( fuzz_context: ?FuzzContext, ) Step.ExtendedMakeError!void { const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into process arena const gpa = maker.gpa; const step = maker.stepByIndex(run_index); const io = graph.io; @@ -1754,7 +1746,6 @@ fn runCommand( const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); const conf_run = conf_step.extended.get(conf.extra).run; - const environ_map = &run.environ_map; const cwd: process.Child.Cwd = if (conf_run.cwd.value) |lazy_cwd| .{ .path = try maker.resolveLazyPathIndexAbs(arena, lazy_cwd, run_index) } @@ -1768,12 +1759,11 @@ fn runCommand( var interp_argv: std.ArrayList([]const u8) = .empty; - // `environ_map` is initialized with an undefined `allocator` field; lazily - // initialize it here. - environ_map.allocator = gpa; + var environ_map: std.process.Environ.Map = .init(gpa); + defer environ_map.deinit(); + // In either case we add to this mutatable data structure so that we can // tweak the environment below. - environ_map.clearRetainingCapacity(); if (conf_run.environ_map.value) |env_map_index| { const conf_env_map = env_map_index.get(conf); for (conf_env_map.keys.slice(conf), conf_env_map.values.slice(conf)) |k, v| { @@ -1792,7 +1782,7 @@ fn runCommand( const root_module = producer.root_module.get(conf); const root_module_target = root_module.resolved_target.get(conf).?.result.get(conf); if (root_module_target.flags.os_tag == .windows) { - try addPathForDynLibs(maker, producer_index, environ_map, argv[0]); + try addPathForDynLibs(maker, arena, producer_index, &environ_map, argv[0]); } } @@ -1801,15 +1791,16 @@ fn runCommand( .dir => unreachable, .inherit => null, }; - try graph.handleVerbose(cwd_string, environ_map, argv); + try graph.handleVerbose(cwd_string, &environ_map, argv); const opt_generic_result = spawnChildAndCollect( + arena, run_index, run, maker, progress_node, argv, - environ_map, + &environ_map, has_side_effects, fuzz_context, ) catch |err| term: { @@ -1863,7 +1854,7 @@ fn runCommand( try environ_map.put("WINEDEBUG", "-all"); } } else { - return failForeign(&conf_run, maker, run_index, "-fwine", argv[0], &root_target, &host); + return failForeign(arena, &conf_run, maker, run_index, "-fwine", argv[0], &root_target, &host); } }, .qemu => |bin_name| { @@ -1887,11 +1878,11 @@ fn runCommand( root_target.abi, ) else unreachable, })); - } else return failForeign(&conf_run, maker, run_index, "--libc-runtimes", argv[0], &root_target, &host); + } else return failForeign(arena, &conf_run, maker, run_index, "--libc-runtimes", argv[0], &root_target, &host); } interp_argv.appendSliceAssumeCapacity(argv); - } else return failForeign(&conf_run, maker, run_index, "-fqemu", argv[0], &root_target, &host); + } else return failForeign(arena, &conf_run, maker, run_index, "-fqemu", argv[0], &root_target, &host); }, .darling => |bin_name| { if (graph.enable_darling) { @@ -1899,7 +1890,7 @@ fn runCommand( interp_argv.appendAssumeCapacity(bin_name); interp_argv.appendSliceAssumeCapacity(argv); } else { - return failForeign(&conf_run, maker, run_index, "-fdarling", argv[0], &root_target, &host); + return failForeign(arena, &conf_run, maker, run_index, "-fdarling", argv[0], &root_target, &host); } }, .wasmtime => |bin_name| { @@ -1912,7 +1903,7 @@ fn runCommand( interp_argv.appendAssumeCapacity("-Sinherit-env"); interp_argv.appendSliceAssumeCapacity(argv); } else { - return failForeign(&conf_run, maker, run_index, "-fwasmtime", argv[0], &root_target, &host); + return failForeign(arena, &conf_run, maker, run_index, "-fwasmtime", argv[0], &root_target, &host); } }, .bad_dl => |foreign_dl| { @@ -1941,15 +1932,16 @@ fn runCommand( gpa.free(step.result_failed_command.?); step.result_failed_command = null; - try graph.handleVerbose(cwd_string, environ_map, interp_argv.items); + try graph.handleVerbose(cwd_string, &environ_map, interp_argv.items); break :term spawnChildAndCollect( + arena, run_index, run, maker, progress_node, interp_argv.items, - environ_map, + &environ_map, has_side_effects, fuzz_context, ) catch |e| { @@ -1996,7 +1988,7 @@ fn runCommand( if (stream.captured) |captured| { const output_path: Path = .{ .root_dir = cache_root, - .sub_path = try Dir.path.join(arena, &.{ + .sub_path = try Dir.path.join(graph.arena, &.{ output_dir_path, captured.basename.slice(conf), }), }; @@ -2117,6 +2109,7 @@ const EvalGenericResult = struct { }; fn spawnChildAndCollect( + arena: Allocator, run_index: Configuration.Step.Index, run: *Run, maker: *Maker, @@ -2129,7 +2122,6 @@ fn spawnChildAndCollect( const step = maker.stepByIndex(run_index); const graph = maker.graph; const io = graph.io; - const arena = graph.arena; // TODO don't leak into process arena const gpa = maker.gpa; const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); @@ -2189,7 +2181,7 @@ fn spawnChildAndCollect( if (conf_run.flags.stdio == .zig_test) { const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalZigTest(run, run_index, maker, progress_node, spawn_options, fuzz_context) catch |err| switch (err) { + const result = evalZigTest(graph.arena, run, run_index, maker, progress_node, spawn_options, fuzz_context) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| e, }; @@ -2209,7 +2201,7 @@ fn spawnChildAndCollect( try setColorEnvironmentVariables(&conf_run, environ_map, terminal_mode); const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalGeneric(run_index, maker, spawn_options) catch |err| switch (err) { + const result = evalGeneric(arena, run_index, maker, spawn_options) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| e, }; @@ -2287,12 +2279,11 @@ fn checksContainStderr(conf_run: *const Configuration.Step.Run) bool { /// /// Whenever a path is included in the argv of a child, it should be put through this function first /// to make sure the child doesn't see paths relative to a cwd other than its own. -fn convertPathArg(run_index: Configuration.Step.Index, maker: *Maker, path: Path) ![]const u8 { +fn convertPathArg(arena: Allocator, run_index: Configuration.Step.Index, maker: *Maker, path: Path) ![]const u8 { const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); const conf_run = conf_step.extended.get(conf.extra).run; const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into process arena const path_str = try path.toString(arena); if (Dir.path.isAbsolute(path_str)) { @@ -2319,13 +2310,13 @@ fn convertPathArg(run_index: Configuration.Step.Index, maker: *Maker, path: Path fn addPathForDynLibs( maker: *Maker, + arena: Allocator, artifact: Configuration.Step.Index, environ_map: *process.Environ.Map, argv0: []const u8, ) !void { const conf = &maker.scanned_config.configuration; const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into process arena const use_wine = graph.enable_wine and builtin.os.tag != .windows and std.ascii.endsWithIgnoreCase(argv0, ".exe"); const path_key = if (use_wine) "WINEPATH" else "PATH"; const path_delimiter: u8 = if (builtin.os.tag == .windows or use_wine) @@ -2355,6 +2346,7 @@ fn addPathForDynLibs( } fn failForeign( + arena: Allocator, conf_run: *const Configuration.Step.Run, maker: *Maker, step_index: Configuration.Step.Index, @@ -2368,10 +2360,8 @@ fn failForeign( .check, .zig_test => { if (conf_run.flags.skip_foreign_checks) return error.MakeSkipped; - const graph = maker.graph; - const process_arena = graph.arena; // TODO don't leak into process arena - const host_name = try host_target.zigTriple(process_arena); - const foreign_name = try artifact_target.zigTriple(process_arena); + const host_name = try host_target.zigTriple(arena); + const foreign_name = try artifact_target.zigTriple(arena); return step.fail(maker, \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) -- 2.54.0 From bd1b47733bc7565187d4a7c806afec03401bbf25 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 13:54:20 -0700 Subject: [PATCH 167/179] zig build: remove --build-runner CLI parameter There is no concept of a "build runner" any more; it has been split in two: configurer and maker. Users of this feature will likely want to no longer override any logic, and instead consume the configuration file produced by configurer. If overriding one or the other of these is desired, the feature will need to be re-introduced (as --override-maker or --override-configurer). --- lib/compiler/Maker/ScannedConfig.zig | 1 - lib/std/zig.zig | 1 - src/main.zig | 11 +---------- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index 7bcdec5f64b25f20a43f7d36e7817989bfb57548..e9345f001f45f78ea66fabc87751c0591833a1a7 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -342,7 +342,6 @@ pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory \\ --zig-lib-dir [arg] Override path to Zig lib directory - \\ --build-runner [file] Override path to build runner \\ --seed [integer] For shuffling dependency traversal order (default: random) \\ --cache-poison[=mode] Override configuration caching behavior \\ pure (default) Avoid false positive cache hits diff --git a/lib/std/zig.zig b/lib/std/zig.zig index db5e20c8d172c5d3701a077e2a913bd465653da6..f667aa13b3d692b13b158773bd5eef7a249cfae0 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -756,7 +756,6 @@ pub const EnvVar = enum { ZIG_LOCAL_PKG_DIR, ZIG_LIB_DIR, ZIG_LIBC, - ZIG_BUILD_RUNNER, ZIG_BUILD_ERROR_STYLE, ZIG_BUILD_MULTILINE_ERRORS, ZIG_VERBOSE_LINK, diff --git a/src/main.zig b/src/main.zig index e15ab41c1f56c05a72e6c7ab866797d656f5de93..a8180265239d98b6054e995de2d2c6869168c4e3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4944,7 +4944,6 @@ fn cmdBuild( var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); - var override_make_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map); var maker_optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map)) .Debug else @@ -5074,11 +5073,6 @@ fn cmdBuild( i += 1; override_lib_dir = args[i]; continue; - } else if (mem.eql(u8, arg, "--build-runner")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - override_make_runner = args[i]; - continue; } else if (mem.eql(u8, arg, "--cache-dir")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; @@ -5365,10 +5359,7 @@ fn cmdBuild( // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. { - const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_make_runner) |runner| .{ - .root = try .fromUnresolved(arena, dirs, &.{fs.path.dirname(runner) orelse "."}), - .root_src_path = fs.path.basename(runner), - } else .{ + const main_mod_paths: Package.Module.CreateOptions.Paths = .{ .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), .root_src_path = "configurer.zig", }; -- 2.54.0 From 35ee3747ebe344b619c66b23053d3f2495956423 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 14:11:04 -0700 Subject: [PATCH 168/179] Maker.Step: avoid unnecessary compilation failure On s390x the cache line size is 256 so this check was failing. That's not useful, just check only for more common cache line sizes. --- lib/compiler/Maker/Step.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index e9669a9118a30f91a1eee73ac95b90e2e0cce53f..b4fd26fe66c7ab94dba29b35bafec882fd2b8723 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -73,7 +73,7 @@ comptime { // Common cache line size is 128. This check prevents accidentally crossing // an additional cache line. In the future it might be nice to try to fit // this struct in 128 bytes or less. - assert(@sizeOf(@This()) <= 128 * 3); + if (std.atomic.cache_line <= 128) assert(@sizeOf(@This()) <= 128 * 3); } pub const Extended = union(enum) { -- 2.54.0 From 860d5ab9c41fd64c5150bd7d9f5b87cbcda9f281 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 14:22:28 -0700 Subject: [PATCH 169/179] Maker: implement relativePath with non-empty subpath for zig_exe --- lib/compiler/Maker.zig | 10 ++++++---- lib/compiler/Maker/Step.zig | 14 ++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 643df6b68fa576c3cd75e46ff570e1c5920ebc85..4420b98eca6255901fb4c060592a01dab3df0958 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1706,7 +1706,7 @@ pub fn resolveLazyPath( const c = &maker.scanned_config.configuration; return switch (lazy_path) { .source_path => |sp| try packagePath(maker, arena, sp.owner, sp.sub_path.slice(c)), - .relative => |relative| relativePath(maker, relative), + .relative => |relative| relativePath(maker, arena, relative), .generated => |gen| { const base = generatedPath(maker, gen.index).*; var file_path = base; @@ -1785,11 +1785,10 @@ pub fn packagePath( }; } -pub fn relativePath(maker: *const Maker, relative: Configuration.LazyPath.Relative) Path { +pub fn relativePath(maker: *const Maker, arena: Allocator, relative: Configuration.LazyPath.Relative) Allocator.Error!Path { const graph = maker.graph; const c = &maker.scanned_config.configuration; const sub_path = relative.sub_path.slice(c); - if (relative.flags.base == .zig_exe and sub_path.len != 0) @panic("TODO relativePath zig_exe"); return switch (relative.flags.base) { .cwd => .{ .root_dir = .cwd(), @@ -1809,7 +1808,10 @@ pub fn relativePath(maker: *const Maker, relative: Configuration.LazyPath.Relati }, .zig_exe => .{ .root_dir = .cwd(), - .sub_path = graph.zig_exe, + .sub_path = if (sub_path.len == 0) + graph.zig_exe + else + try Io.Dir.path.join(arena, &.{ graph.zig_exe, sub_path }), }, .zig_lib => .{ .root_dir = graph.zig_lib_directory, diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index b4fd26fe66c7ab94dba29b35bafec882fd2b8723..2a42dd6959dd0510c0d6ff3887ef6e2b5eb5b411 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -785,7 +785,10 @@ pub fn addWatchInput(step: *Step, maker: *Maker, arena: Allocator, lazy_file: La const pkg_path = try maker.packagePath(arena, source_path.owner, sub_path); try addWatchInputPath(step, maker, pkg_path); }, - .relative => |relative| try addWatchInputPath(step, maker, maker.relativePath(relative)), + .relative => |relative| { + const resolved_path = try maker.relativePath(arena, relative); + try addWatchInputPath(step, maker, resolved_path); + }, // Nothing to watch because this dependency edge is modeled instead via `dependants`. .generated => {}, } @@ -799,16 +802,19 @@ pub fn addWatchInput(step: *Step, maker: *Maker, arena: Allocator, lazy_file: La /// `addDirectoryWatchInputFromPath` if and only if this function returns /// `true`. pub fn addDirectoryWatchInput(step: *Step, maker: *Maker, lazy_directory: LazyPath) Allocator.Error!bool { + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena switch (lazy_directory) { .source_path => |source_path| { const conf = &maker.scanned_config.configuration; - const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into the process arena const sub_path = source_path.sub_path.slice(conf); const pkg_path = try maker.packagePath(arena, source_path.owner, sub_path); try addDirectoryWatchInputFromPath(step, maker, pkg_path); }, - .relative => |relative| try addDirectoryWatchInputFromPath(step, maker, maker.relativePath(relative)), + .relative => |relative| { + const resolved_path = try maker.relativePath(arena, relative); + try addDirectoryWatchInputFromPath(step, maker, resolved_path); + }, // Nothing to watch because this dependency edge is modeled instead via `dependants`. .generated => return false, } -- 2.54.0 From 19c63406d4acded72966a66ac369ccceb69173a2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 14:50:56 -0700 Subject: [PATCH 170/179] LazyPath: store relative paths as actual strings --- lib/compiler/configurer.zig | 2 +- lib/std/Build.zig | 85 +++++++++++++---------------- lib/std/Build/Step/ConfigHeader.zig | 4 +- lib/std/Build/Step/InstallDir.zig | 2 +- lib/std/Build/Step/InstallFile.zig | 2 +- lib/std/Build/Step/ObjCopy.zig | 2 +- 6 files changed, 45 insertions(+), 52 deletions(-) diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index b81ee9de0d351115befc777ec102403a41cb49b7..308bf55fe72f4ab881f1168c96b1fc9ae9476bc3 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -208,7 +208,7 @@ const Serialize = struct { .relative => |relative| i: { break :i try wc.addExtraErased(Configuration.LazyPath.Relative, .{ .flags = .{ .base = relative.base }, - .sub_path = relative.sub_path, + .sub_path = try wc.addString(relative.sub_path), }); }, .dependency => |dependency| i: { diff --git a/lib/std/Build.zig b/lib/std/Build.zig index d8948380de3edb59703ea2a3dc90b95fa2e89f6f..bc77290f131a15926eb9e561bd030658b91ce60c 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -166,10 +166,9 @@ pub const Graph = struct { /// A path whose components and contents are known at some point during /// `Step` resolution, relative to the provided base directory. pub fn path(graph: *Graph, base: Configuration.Path.Base, sub_path: []const u8) LazyPath { - const wc = &graph.wip_configuration; return .{ .relative = .{ .base = base, - .sub_path = wc.addString(sub_path) catch @panic("OOM"), + .sub_path = @This().dupePath(graph, sub_path), } }; } @@ -974,12 +973,12 @@ pub fn dupe(b: *Build, bytes: []const u8) []const u8 { return b.graph.dupeString(bytes); } -/// Duplicates an array of strings without the need to handle out of memory. +/// Deprecated, call `Graph.dupeStrings` instead. pub fn dupeStrings(b: *Build, strings: []const []const u8) []const []const u8 { return b.graph.dupeStrings(strings); } -/// Duplicates a path, canonicalizing path separators. +/// Deprecated, call `Graph.dupePath` instead. pub fn dupePath(b: *Build, bytes: []const u8) []const u8 { return b.graph.dupePath(bytes); } @@ -1536,7 +1535,7 @@ pub fn addUserInputFlag(b: *Build, name_raw: []const u8) error{OutOfMemory}!bool return true; }, .lazy_path => |lp| { - log.err("Flag '-D{s}' conflicts with option '-D{s}={f}'.", .{ name, name, lp.fmt(graph) }); + log.err("Flag '-D{s}' conflicts with option '-D{s}={f}'.", .{ name, name, lp }); return true; }, @@ -2381,10 +2380,10 @@ pub const LazyPath = union(enum) { relative: struct { base: Configuration.Path.Base, - sub_path: Configuration.String = .empty, + sub_path: []const u8 = "", pub fn eql(a: @This(), b: @This()) bool { - return a.base == b.base and a.sub_path == b.sub_path; + return a.base == b.base and mem.eql(u8, a.sub_path, b.sub_path); } }, @@ -2398,11 +2397,11 @@ pub const LazyPath = union(enum) { /// Returns a lazy path referring to the directory containing this path. /// - /// The dirname is not allowed to escape the logical root for underlying path. - /// For example, if the path is relative to the build root, - /// the dirname is not allowed to traverse outside of the build root. - /// Similarly, if the path is a generated file inside zig-cache, - /// the dirname is not allowed to traverse outside of zig-cache. + /// The dirname is not allowed to escape the logical root for underlying + /// path. For example, if the path is relative to the build root, the + /// dirname is not allowed to traverse outside of the build root. + /// Similarly, if the path is a generated file inside zig-cache, the + /// dirname is not allowed to traverse outside of zig-cache. pub fn dirname(lazy_path: LazyPath) LazyPath { return switch (lazy_path) { .src_path => |sp| .{ .src_path = .{ @@ -2444,9 +2443,13 @@ pub const LazyPath = union(enum) { } }, }, - .relative => .{ - .relative = @panic("TODO"), - }, + .relative => |r| .{ .relative = .{ + .base = r.base, + .sub_path = dirnameAllowEmpty(r.sub_path) orelse { + dumpBadDirnameHelp(null, null, "dirname() attempted to traverse outside the base path\n", .{}) catch {}; + @panic("misconfigured build script"); + }, + } }, .dependency => |dep| .{ .dependency = .{ .dependency = dep.dependency, .sub_path = dirnameAllowEmpty(dep.sub_path) orelse { @@ -2480,9 +2483,10 @@ pub const LazyPath = union(enum) { .cwd_relative => |cwd_relative| .{ .cwd_relative = try fs.path.resolve(arena, &.{ cwd_relative, sub_path }), }, - .relative => .{ - .relative = @panic("TODO"), - }, + .relative => |r| .{ .relative = .{ + .base = r.base, + .sub_path = try fs.path.resolve(arena, &.{ r.sub_path, sub_path }), + } }, .dependency => |dep| .{ .dependency = .{ .dependency = dep.dependency, .sub_path = try fs.path.resolve(arena, &.{ dep.sub_path, sub_path }), @@ -2490,27 +2494,25 @@ pub const LazyPath = union(enum) { }; } - pub const Format = struct { - graph: *const Graph, - lazy_path: *const LazyPath, + /// Deprecated, use `format` instead. + pub fn getDisplayName(lazy_path: LazyPath) []const u8 { + return switch (lazy_path) { + .src_path => |sp| sp.sub_path, + .cwd_relative => |p| p, + .generated => "generated", + .dependency => "dependency", + .relative => |r| @tagName(r.base), + }; + } - pub fn format(f: Format, w: *Io.Writer) Io.Writer.Error!void { - switch (f.lazy_path.*) { - .src_path => |sp| try w.writeAll(sp.sub_path), - .cwd_relative => |p| try w.writeAll(p), - .generated => try w.writeAll("generated"), - .dependency => try w.writeAll("dependency"), - .relative => |r| { - const wc = &f.graph.wip_configuration; - try w.writeAll(@tagName(r.base)); - try w.writeAll(wc.stringSlice(r.sub_path)); - }, - } + pub fn format(lp: LazyPath, w: *Io.Writer) Io.Writer.Error!void { + switch (lp) { + .src_path => |sp| try w.writeAll(sp.sub_path), + .cwd_relative => |p| try w.writeAll(p), + .generated => try w.writeAll("generated"), + .dependency => try w.writeAll("dependency"), + .relative => |r| try w.print("{t} {s}", .{ r.base, r.sub_path }), } - }; - - pub fn fmt(lp: *const LazyPath, graph: *const Graph) Format { - return .{ .graph = graph, .lazy_path = lp }; } /// Adds dependencies this file source implies to the given step. @@ -2525,15 +2527,6 @@ pub const LazyPath = union(enum) { } } - pub fn basename(lazy_path: LazyPath) []const u8 { - return fs.path.basename(switch (lazy_path) { - .src_path => |sp| sp.sub_path, - .cwd_relative => |sub_path| sub_path, - .generated => |gen| gen.sub_path, - .dependency => |dep| dep.sub_path, - }); - } - /// Copies the internal strings. /// /// The `graph` parameter is only used for the global arena allocator. diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index 55b9a617f2e1670e3c32140b43f2aba3c450000a..291e83975eb123d589f73e0e3bc61b6b2afaccd8 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -73,7 +73,7 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { .src_path => |sp| sp.sub_path, .generated => break :default, .cwd_relative => |sub_path| sub_path, - .relative => |r| wc.stringSlice(r.sub_path), + .relative => |r| r.sub_path, .dependency => |dependency| dependency.sub_path, }; const basename = Io.Dir.path.basename(sub_path); @@ -85,7 +85,7 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { const name = if (options.style.getPath()) |s| allocPrint(arena, "configure {t} header {f} to {s}", .{ - options.style, s.fmt(graph), include_path, + options.style, s, include_path, }) catch @panic("OOM") else allocPrint(arena, "configure {t} header to {s}", .{ diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index 02a436e3a752b965ef4fa6eb41ae64a809a68d87..3aa0b16aa16417ada982f13b85a352ea52819732 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -47,7 +47,7 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir { install_dir.* = .{ .step = Step.init(.{ .tag = base_tag, - .name = owner.fmt("install {f}/", .{options.source_dir.fmt(graph)}), + .name = owner.fmt("install {f}/", .{options.source_dir}), .owner = owner, }), .options = options.dupe(graph), diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index 7d1d11cf5ad4b1ca8d33cbdf6fece43075e62f6d..d63cd6408c3b09c0e378a66f10d843fb950e035e 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -26,7 +26,7 @@ pub fn create( install_file.* = .{ .step = Step.init(.{ .tag = base_tag, - .name = owner.fmt("install {f} to {s}", .{ source.fmt(graph), dest_rel_path }), + .name = owner.fmt("install {f} to {s}", .{ source, dest_rel_path }), .owner = owner, }), .source = source.dupe(graph), diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index d5a38ee91e2bfe9460aa625d56b0e9e1019a9903..e02974b5701cdf49865ad8b178b0f2c5b3e50440 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -63,7 +63,7 @@ pub fn create(owner: *std.Build, input_file: std.Build.LazyPath, options: Option oc.* = .{ .step = .init(.{ .tag = base_tag, - .name = owner.fmt("objcopy {f}", .{input_file.fmt(graph)}), + .name = owner.fmt("objcopy {f}", .{input_file}), .owner = owner, }), .input_file = input_file, -- 2.54.0 From 198f35c98c8a8216a120f146605bb729e7b8dd66 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 15:00:56 -0700 Subject: [PATCH 171/179] Maker: restore commit: clear step inputs when resetting the step I'm a bit confused because my understanding is that each step is supposed to make its own decision about whether to clear its watch inputs and start over, or retain them from the previous update. However, without this fix in place, the problem from #35224 manifests itself again. Since this fix is in place in master branch, I'll leave it for now and audit the file watching logic later. --- lib/compiler/Maker/Step.zig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 2a42dd6959dd0510c0d6ff3887ef6e2b5eb5b411..1193b2c2a514021ff8c7db0a0053fdec6e08ac7f 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -312,8 +312,7 @@ pub fn reset(step: *Step, maker: *Maker) void { step.result_duration_ns = null; step.result_peak_rss = 0; step.test_results = .{}; - // We do not clearWatchInputs here because each step manages that choice - // independently. + clearWatchInputs(step, maker); step.result_error_bundle.deinit(gpa); step.result_error_bundle = std.zig.ErrorBundle.empty; -- 2.54.0 From a7d1edae8f3d673ecbdd88b13044d7edc51b28af Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 17:20:42 -0700 Subject: [PATCH 172/179] zig build: save configurations to .zig-cache c/, not o/ keeps the cache directory hierarchy more uniform and avoids too many files in one directory --- src/main.zig | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main.zig b/src/main.zig index a8180265239d98b6054e995de2d2c6869168c4e3..bd73f5a08a4d36cf65517b161d13b548d6ee19d6 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5629,7 +5629,7 @@ fn cmdBuild( break :cp .{ .{ .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + .sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}), }, false, }; @@ -5723,7 +5723,7 @@ fn cmdBuild( const digest = config_man.final(); const final_path: Path = .{ .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + .sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}), }; Io.Dir.rename( config_tmp_path.root_dir.handle, @@ -5731,9 +5731,24 @@ fn cmdBuild( final_path.root_dir.handle, final_path.sub_path, io, - ) catch |err| { + ) catch |err| retry: { + const e = switch (err) { + error.FileNotFound => e: { + const dir_path = final_path.dirname().?; + dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| + fatal("failed to create directory {f}: {t}", .{ dir_path, e }); + if (Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + )) |_| break :retry else |e| break :e e; + }, + else => |e| e, + }; fatal("failed to rename configuration file from {f} into {f}: {t}", .{ - config_tmp_path, final_path, err, + config_tmp_path, final_path, e, }); }; config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); -- 2.54.0 From cb1f3e0ac416ccc3a4cf5fead70807bd7a66d9b8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 17:45:53 -0700 Subject: [PATCH 173/179] Maker.Step.Compile: leak into the global arena less --- lib/compiler/Maker/Step.zig | 8 +-- lib/compiler/Maker/Step/Compile.zig | 74 +++++++++++++------------- lib/compiler/Maker/Step/Run.zig | 10 ++-- lib/compiler/Maker/Step/TranslateC.zig | 2 +- 4 files changed, 46 insertions(+), 48 deletions(-) diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 1193b2c2a514021ff8c7db0a0053fdec6e08ac7f..2367329a77b6cdab2858bc07c0d32e7b1c68fbf9 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -360,9 +360,11 @@ pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcess return result; } -fn clearFailedCommand(s: *Step, gpa: Allocator) void { - if (s.result_failed_command) |cmd| gpa.free(cmd); - s.result_failed_command = null; +pub fn clearFailedCommand(s: *Step, gpa: Allocator) void { + if (s.result_failed_command) |cmd| { + gpa.free(cmd); + s.result_failed_command = null; + } } pub const FailError = error{ OutOfMemory, MakeFailed }; diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 4f7d89ba8ecc552e91127e7f18a2ba7cbe60413e..75814a1e7bfdae19a12d6279978a6e6523a046ff 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -19,8 +19,6 @@ const PkgConfig = @import("../PkgConfig.zig"); /// Populated when there is compiler process that lives across multiple calls /// to `make`. zig_process: ?*Step.ZigProcess = null, -/// Persisted to reuse memory on subsequent calls to `make`. -zig_args: std.ArrayList([]const u8) = .empty, /// Populated by InstallArtifact. installed_path: ?Path = null, /// Populated by `make`, used by `Run`. @@ -33,25 +31,29 @@ pub fn make( progress_node: std.Progress.Node, ) Step.ExtendedMakeError!void { const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into process arena + const gpa = maker.gpa; const conf = &maker.scanned_config.configuration; const conf_step = compile_index.ptr(conf); const conf_comp = conf_step.extended.get(conf.extra).compile; - // Reset / repopulate persistent state. - compile.zig_args.clearRetainingCapacity(); + var arena_allocator: std.heap.ArenaAllocator = .init(gpa); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); - try lowerZigArgs(compile, compile_index, maker, progress_node, &compile.zig_args, false); + var argv: std.ArrayList([]const u8) = .empty; + defer argv.deinit(gpa); + + try lowerZigArgs(arena, compile, compile_index, maker, progress_node, &argv, false); const maybe_output_dir = Step.evalZigProcess( compile_index, maker, - compile.zig_args.items, + argv.items, progress_node, (graph.incremental == true) and (maker.watch or maker.web_server != null), ) catch |err| switch (err) { error.NeedCompileErrorCheck => { - try checkCompileErrors(maker, compile_index); + try checkCompileErrors(arena, maker, compile_index); return; }, else => |e| return e, @@ -63,14 +65,14 @@ pub fn make( // Update generated files if (maybe_output_dir) |output_dir| { if (conf_comp.emit_directory.value) |gf| maker.generatedPath(gf).* = output_dir; - try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_bin.value, .bin); - try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_pdb.value, .pdb); - try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_implib.value, .implib); - try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_h.value, .h); - try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_docs.value, .docs); - try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_asm.value, .@"asm"); - try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_llvm_ir.value, .llvm_ir); - try updateGeneratedFile(&conf_comp, maker, output_dir, &target, conf_comp.generated_llvm_bc.value, .llvm_bc); + try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_bin.value, .bin); + try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_pdb.value, .pdb); + try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_implib.value, .implib); + try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_h.value, .h); + try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_docs.value, .docs); + try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_asm.value, .@"asm"); + try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_llvm_ir.value, .llvm_ir); + try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_llvm_bc.value, .llvm_bc); } if (conf_comp.flags3.kind == .lib and conf_comp.flags2.linkage == .dynamic and @@ -84,8 +86,9 @@ pub fn make( } fn updateGeneratedFile( + maker: *Maker, + arena: Allocator, conf_comp: *const Configuration.Step.Compile, - maker: *Maker, out_path: std.Build.Cache.Path, target: *const Configuration.TargetQuery, opt_gf: ?Configuration.GeneratedFileIndex, @@ -94,7 +97,6 @@ fn updateGeneratedFile( const gf = opt_gf orelse return; const graph = maker.graph; const conf = &maker.scanned_config.configuration; - const arena = graph.arena; // TODO don't leak into process arena const name = try ea.cacheName(arena, .{ .root_name = conf_comp.root_name.slice(conf), .cpu_arch = target.flags.cpu_arch.unwrap().?, @@ -112,7 +114,7 @@ fn updateGeneratedFile( else null, }); - maker.generatedPath(gf).* = try out_path.join(arena, name); + maker.generatedPath(gf).* = try out_path.join(graph.arena, name); } /// List of importable modules in a compilation's module graph, including @@ -147,6 +149,7 @@ const ModuleListContext = struct { }; fn lowerZigArgs( + arena: Allocator, compile: *Compile, compile_index: Configuration.Step.Index, maker: *Maker, @@ -156,7 +159,6 @@ fn lowerZigArgs( ) Step.ExtendedMakeError!void { const step = maker.stepByIndex(compile_index); const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into the process arena const gpa = maker.gpa; const conf = &maker.scanned_config.configuration; const conf_step = compile_index.ptr(conf); @@ -508,7 +510,7 @@ fn lowerZigArgs( if (cli_named_modules.modules.getIndex(mod_index)) |module_cli_index| { const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; const module_index = cli_named_modules.modules.keys()[module_cli_index]; - try appendModuleFlags(module_index, zig_args, compile_index, maker); + try appendModuleFlags(arena, module_index, zig_args, compile_index, maker); const imports = mod.import_table.get(conf).imports.mal; @@ -953,21 +955,23 @@ pub fn rebuildInFuzzMode( const gpa = maker.gpa; const step = maker.stepByIndex(compile_index); + var arena_allocator: std.heap.ArenaAllocator = .init(gpa); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); + step.result_error_msgs.clearRetainingCapacity(); step.result_stderr = ""; step.result_error_bundle.deinit(gpa); step.result_error_bundle = std.zig.ErrorBundle.empty; - if (step.result_failed_command) |cmd| { - gpa.free(cmd); - step.result_failed_command = null; - } + step.clearFailedCommand(gpa); - const zig_args = &compile.zig_args; - zig_args.clearRetainingCapacity(); - try lowerZigArgs(compile, compile_index, maker, progress_node, zig_args, true); - const maybe_output_bin_path = try Step.evalZigProcess(compile_index, maker, zig_args.items, progress_node, false); + var argv: std.ArrayList([]const u8) = .empty; + defer argv.deinit(gpa); + + try lowerZigArgs(arena, compile, compile_index, maker, progress_node, &argv, true); + const maybe_output_bin_path = try Step.evalZigProcess(compile_index, maker, argv.items, progress_node, false); return maybe_output_bin_path.?; } @@ -980,10 +984,8 @@ fn addFlag(gpa: Allocator, args: *std.ArrayList([]const u8), comptime name: []co try args.append(gpa, if (cond) "-f" ++ name else "-fno-" ++ name); } -fn checkCompileErrors(maker: *Maker, step_index: Configuration.Step.Index) Step.ExtendedMakeError!void { +fn checkCompileErrors(arena: Allocator, maker: *Maker, step_index: Configuration.Step.Index) Step.ExtendedMakeError!void { const step = maker.stepByIndex(step_index); - const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into the process arena const conf = &maker.scanned_config.configuration; const conf_step = step_index.ptr(conf); const conf_comp = conf_step.extended.get(conf.extra).compile; @@ -1226,14 +1228,13 @@ fn getModuleList( } fn appendModuleFlags( + arena: Allocator, module_index: Configuration.Module.Index, zig_args: *std.ArrayList([]const u8), asking_step: Configuration.Step.Index, maker: *const Maker, ) !void { const gpa = maker.gpa; - const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into the process arena const conf = &maker.scanned_config.configuration; const m = module_index.get(conf); @@ -1317,7 +1318,7 @@ fn appendModuleFlags( try zig_args.ensureUnusedCapacity(gpa, 2 * m.include_dirs.len); for (0..m.include_dirs.len) |i| - try appendIncludeDirFlags(m.include_dirs.get(conf.extra, i), zig_args, asking_step, maker); + try appendIncludeDirFlags(arena, m.include_dirs.get(conf.extra, i), zig_args, asking_step, maker); try zig_args.ensureUnusedCapacity(gpa, m.c_macros.slice.len); for (m.c_macros.slice) |c_macro| @@ -1344,13 +1345,12 @@ fn appendModuleFlags( /// Assumes unused capacity for at least 2 items. pub fn appendIncludeDirFlags( + arena: Allocator, include_dir: Configuration.Module.IncludeDir, zig_args: *std.ArrayList([]const u8), asking_step: Configuration.Step.Index, maker: *const Maker, ) !void { - const graph = maker.graph; - const arena = graph.arena; // TODO don't leak into the process arena const conf = &maker.scanned_config.configuration; switch (include_dir) { diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index ca3bca666556032378f371e66c9d1ecc199e5a95..dacafac739dffbddf9b21a154d61f2e4efd40cd7 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -1605,10 +1605,7 @@ pub fn rerunInFuzzMode( argv_list.appendAssumeCapacity("--listen=-"); } - if (step.result_failed_command) |cmd| { - gpa.free(cmd); - step.result_failed_command = null; - } + step.clearFailedCommand(gpa); const has_side_effects = false; var rand_int: u64 = undefined; @@ -1930,8 +1927,7 @@ fn runCommand( }, } - gpa.free(step.result_failed_command.?); - step.result_failed_command = null; + step.clearFailedCommand(gpa); try graph.handleVerbose(cwd_string, &environ_map, interp_argv.items); break :term spawnChildAndCollect( @@ -2138,7 +2134,7 @@ fn spawnChildAndCollect( .inherit; // If an error occurs, it's caused by this command: - assert(step.result_failed_command == null); + step.clearFailedCommand(gpa); step.result_failed_command = try std.zig.allocPrintCmd(gpa, argv, .{ .cwd = switch (child_cwd) { .path => |p| p, diff --git a/lib/compiler/Maker/Step/TranslateC.zig b/lib/compiler/Maker/Step/TranslateC.zig index 68b32b16efab98711bab0c8b497359676f811f14..84176dde10f0fde70a9175803c3a325888920d82 100644 --- a/lib/compiler/Maker/Step/TranslateC.zig +++ b/lib/compiler/Maker/Step/TranslateC.zig @@ -53,7 +53,7 @@ pub fn make( try argv.ensureUnusedCapacity(arena, conf_tc.include_dirs.len * 2); for (0..conf_tc.include_dirs.len) |i| - try Step.Compile.appendIncludeDirFlags(conf_tc.include_dirs.get(conf.extra, i), &argv, step_index, maker); + try Step.Compile.appendIncludeDirFlags(arena, conf_tc.include_dirs.get(conf.extra, i), &argv, step_index, maker); for (conf_tc.c_macros.slice) |c_macro| { (try argv.addManyAsArray(arena, 2)).* = .{ "-D", c_macro.slice(conf) }; -- 2.54.0 From 66f0564c3a3f122bb50eabaa1bf9a2bb0fa0562d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 18:09:46 -0700 Subject: [PATCH 174/179] Maker.Watch: clean up a couple error logs --- lib/compiler/Maker/Watch.zig | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/compiler/Maker/Watch.zig b/lib/compiler/Maker/Watch.zig index 36eefcbaafd5810daaff591589cb54051c61964c..603b9c911d696ad94f0ea5d250df07236cec78d6 100644 --- a/lib/compiler/Maker/Watch.zig +++ b/lib/compiler/Maker/Watch.zig @@ -234,9 +234,8 @@ const Os = switch (builtin.os.tag) { posix.fanotify_mark(fan_fd, .{ .ADD = true, .ONLYDIR = true, - }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| { - fatal("unable to watch {f}: {s}", .{ path, @errorName(err) }); - }; + }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| + fatal("unable to watch {f}: {t}", .{ path, err }); } break :rs &dh_gop.value_ptr.reaction_set; } @@ -289,7 +288,7 @@ const Os = switch (builtin.os.tag) { .ONLYDIR = true, }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| switch (err) { error.FileNotFound => {}, // Expected, harmless. - else => |e| std.log.warn("unable to unwatch '{f}': {s}", .{ path, @errorName(e) }), + else => |e| std.log.warn("unable to unwatch {f}: {t}", .{ path, e }), }; w.dir_table.swapRemoveAt(i); -- 2.54.0 From a9e0eb5340bb60547fb8badc3591d9ffca415ac7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 19:20:40 -0700 Subject: [PATCH 175/179] std.Build.Step.Compile: deprecate out_filename --- lib/std/Build/Step/Compile.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 9a06ca87246bc344ab64166c7551a2c910854903..448e3b0dde840fae5d42c2b194fc404b5eb01cd0 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -25,6 +25,7 @@ root_module: *Module, name: []const u8, linker_script: ?LazyPath = null, version_script: ?LazyPath = null, +/// Deprecated. out_filename: []const u8, linkage: ?std.builtin.LinkMode = null, version: ?std.SemanticVersion, -- 2.54.0 From f3dd10d40fb9b6b94ccf7729c728c80ebd608ba7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 20:14:34 -0700 Subject: [PATCH 176/179] Maker: add --debug-maker-leaks flag and fix some leaks --- lib/compiler/Maker.zig | 35 ++++++++++++++++++++++++--------- lib/compiler/Maker/Step.zig | 14 ++++++++++--- lib/compiler/Maker/Step/Run.zig | 4 ++++ 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 4420b98eca6255901fb4c060592a01dab3df0958..cdb1fce66445b4690ef2add890b7b9321b9ec748 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -50,6 +50,7 @@ memory_blocked_steps: std.ArrayList(Configuration.Step.Index), /// Allocated into `gpa`. step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), pkg_config: PkgConfig, +debug_maker_leaks: bool, error_style: ErrorStyle, multiline_errors: MultilineErrors, @@ -73,6 +74,7 @@ pub fn main(init: process.Init.Minimal) !void { var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); defer arena_instance.deinit(); const arena = arena_instance.allocator(); + defer log.info("used {Bi} of arena", .{arena_instance.queryCapacity()}); const args = try init.args.toSlice(arena); @@ -150,7 +152,8 @@ pub fn main(init: process.Init.Minimal) !void { var fuzz: ?Fuzz.Mode = null; var debounce_interval_ms: u16 = 50; var webui_listen: ?Io.net.IpAddress = null; - var debug_pkg_config: bool = false; + var debug_pkg_config = false; + var debug_maker_leaks = false; var run_args: ?[]const []const u8 = null; if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { @@ -297,6 +300,8 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse fatal("unrecognized optimization mode: {s}", .{rest}); + } else if (mem.eql(u8, arg, "--debug-maker-leaks")) { + debug_maker_leaks = true; } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); @@ -538,6 +543,7 @@ pub fn main(init: process.Init.Minimal) !void { .memory_blocked_steps = .empty, .step_stack = .empty, .pkg_config = .{ .debug = debug_pkg_config }, + .debug_maker_leaks = debug_maker_leaks, .error_style = error_style, .multiline_errors = multiline_errors, @@ -603,10 +609,10 @@ pub fn main(init: process.Init.Minimal) !void { web_server.finishBuild(.{ .fuzz = fuzz != null }); } - if (maker.web_server) |*ws| { + if (maker.web_server) |*web_server| { const c = &scanned_config.configuration; assert(!watch); // fatal error after CLI parsing - while (true) switch (try ws.wait()) { + while (true) switch (try web_server.wait()) { .rebuild => { for (maker.step_stack.keys()) |step_index| { const step = maker.stepByIndex(step_index); @@ -620,6 +626,8 @@ pub fn main(init: process.Init.Minimal) !void { }; } + if (!maker.watch) return; + // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. if (!Watch.have_impl) unreachable; @@ -996,21 +1004,29 @@ fn makeStepNames( if (maker.watch or maker.web_server != null) return; - // Perhaps in the future there could be an Advanced Options flag such as - // --debug-build-runner-leaks which would make this code return instead of - // calling exit. - const code: u8 = code: { if (failure_count == 0) break :code 0; // success if (maker.error_style.verboseContext()) break :code 1; // failure; print build command break :code 2; // failure; do not print build command }; - if (code == 0) removePoisonedConfiguration(io, maker.scanned_config); + if (code == 0) { + removePoisonedConfiguration(io, maker.scanned_config); + if (builtin.mode == .Debug and maker.debug_maker_leaks) return deinit(maker); + } cleanup_task.await(io); // There is a defer above but an exit below. _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(code); } +fn deinit(maker: *Maker) void { + const gpa = maker.gpa; + for (maker.steps) |*step| { + step.clearFailedCommand(gpa); + step.clearErrorBundle(gpa); + step.inputs.deinit(gpa); + } +} + fn stepReady( maker: *Maker, group: *Io.Group, @@ -1457,6 +1473,7 @@ fn constructGraphAndCheckForDependencyLoop( ) error{ DependencyLoopDetected, OutOfMemory }!void { const c = &maker.scanned_config.configuration; const gpa = maker.gpa; + const arena = maker.graph.arena; const make_step = maker.stepByIndex(step_index); switch (make_step.state) { .precheck_started => { @@ -1480,7 +1497,7 @@ fn constructGraphAndCheckForDependencyLoop( for (deps) |dep| { const dep_step = maker.stepByIndex(dep); try step_stack.put(gpa, dep, {}); - try dep_step.dependants.append(gpa, step_index); + try dep_step.dependants.append(arena, step_index); constructGraphAndCheckForDependencyLoop(maker, dep, step_stack, rand) catch |err| switch (err) { error.DependencyLoopDetected => { log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 2367329a77b6cdab2858bc07c0d32e7b1c68fbf9..69d3676c32f4567c40cd97be29de8196f65f7756 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -190,6 +190,11 @@ pub const Inputs = struct { for (inputs.table.values()) |*files| files.deinit(gpa); inputs.table.clearRetainingCapacity(); } + + pub fn deinit(inputs: *Inputs, gpa: Allocator) void { + clear(inputs, gpa); + inputs.table.deinit(gpa); + } }; pub const TestResults = struct { @@ -313,9 +318,7 @@ pub fn reset(step: *Step, maker: *Maker) void { step.result_peak_rss = 0; step.test_results = .{}; clearWatchInputs(step, maker); - - step.result_error_bundle.deinit(gpa); - step.result_error_bundle = std.zig.ErrorBundle.empty; + clearErrorBundle(step, gpa); } pub const CaptureChildProcessError = error{ @@ -360,6 +363,11 @@ pub fn captureChildProcess(s: *Step, maker: *Maker, options: CaptureChildProcess return result; } +pub fn clearErrorBundle(s: *Step, gpa: Allocator) void { + s.result_error_bundle.deinit(gpa); + s.result_error_bundle = .empty; +} + pub fn clearFailedCommand(s: *Step, gpa: Allocator) void { if (s.result_failed_command) |cmd| { gpa.free(cmd); diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index dacafac739dffbddf9b21a154d61f2e4efd40cd7..fa666f2d7c46cb83409dbaa5304f8a4268c59086 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -49,7 +49,10 @@ pub fn make( const arena = arena_allocator.allocator(); var argv_list: std.ArrayList([]const u8) = .empty; + defer argv_list.deinit(gpa); + var output_placeholders: std.ArrayList(IndexedOutput) = .empty; + defer output_placeholders.deinit(gpa); var man = graph.cache.obtain(); defer man.deinit(); @@ -1523,6 +1526,7 @@ pub fn rerunInFuzzMode( const arena = arena_allocator.allocator(); var argv_list: std.ArrayList([]const u8) = .empty; + defer argv_list.deinit(gpa); for (conf_run.args.slice) |arg_index| { const arg = arg_index.get(conf); -- 2.54.0 From 5b022623cf7059b76d3bbc91c71a93ab96adbdeb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 20:17:41 -0700 Subject: [PATCH 177/179] Maker.Step.Compile: fix memory leak in checkCompileErrors --- lib/compiler/Maker/Step/Compile.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 75814a1e7bfdae19a12d6279978a6e6523a046ff..0b66dc282c801cd3cabff5c0a72e69cb36aab0c2 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -991,8 +991,9 @@ fn checkCompileErrors(arena: Allocator, maker: *Maker, step_index: Configuration const conf_comp = conf_step.extended.get(conf.extra).compile; // Clear this field so that it does not get printed by the build runner. - const actual_eb = step.result_error_bundle; + var actual_eb = step.result_error_bundle; step.result_error_bundle = .empty; + defer actual_eb.deinit(maker.gpa); const actual_errors = ae: { var aw: std.Io.Writer.Allocating = .init(arena); -- 2.54.0 From 1d750d7067e6e24af00aadc19379cd913652d80e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 20:34:44 -0700 Subject: [PATCH 178/179] Maker.Step.Run: fix leak in evalGeneric --- lib/compiler/Maker/Step/Run.zig | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index fa666f2d7c46cb83409dbaa5304f8a4268c59086..e59fec11efd18b5fd714a27b67e90ba4762e5bf3 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -1374,6 +1374,7 @@ fn sendRunFuzzTestMessage( } } +/// Uses `arena` to allocate the result. fn evalGeneric( arena: Allocator, run_index: Configuration.Step.Index, @@ -1382,7 +1383,6 @@ fn evalGeneric( ) !EvalGenericResult { const graph = maker.graph; const io = graph.io; - const gpa = maker.gpa; const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); const conf_run = conf_step.extended.get(conf.extra).run; @@ -1436,8 +1436,7 @@ fn evalGeneric( if (child.stderr) |stderr| { var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; var multi_reader: Io.File.MultiReader = undefined; - multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); - defer multi_reader.deinit(); + multi_reader.init(arena, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); const stdout_reader = multi_reader.reader(0); const stderr_reader = multi_reader.reader(1); @@ -1457,9 +1456,7 @@ fn evalGeneric( try multi_reader.checkAnyError(); - // TODO: this string can leak since alloc below can return error. stdout_bytes = try multi_reader.toOwnedSlice(0); - // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. stderr_bytes = try multi_reader.toOwnedSlice(1); } else { var stdout_reader = stdout.readerStreaming(io, &.{}); @@ -2201,7 +2198,7 @@ fn spawnChildAndCollect( try setColorEnvironmentVariables(&conf_run, environ_map, terminal_mode); const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalGeneric(arena, run_index, maker, spawn_options) catch |err| switch (err) { + const result = evalGeneric(graph.arena, run_index, maker, spawn_options) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| e, }; -- 2.54.0 From c9619d7086a4aa61b1bd69faff4ce58a7f9c811c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 25 May 2026 20:44:51 -0700 Subject: [PATCH 179/179] Maker: print amount of arena memory used only when --debug-maker-leaks --- lib/compiler/Maker.zig | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index cdb1fce66445b4690ef2add890b7b9321b9ec748..07ca3d436352ff001131f7dd79074c5d797ce934 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -50,7 +50,6 @@ memory_blocked_steps: std.ArrayList(Configuration.Step.Index), /// Allocated into `gpa`. step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), pkg_config: PkgConfig, -debug_maker_leaks: bool, error_style: ErrorStyle, multiline_errors: MultilineErrors, @@ -73,8 +72,8 @@ pub fn main(init: process.Init.Minimal) !void { // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); defer arena_instance.deinit(); + defer if (debugMakerLeaks()) log.debug("used {Bi} of arena", .{arena_instance.queryCapacity()}); const arena = arena_instance.allocator(); - defer log.info("used {Bi} of arena", .{arena_instance.queryCapacity()}); const args = try init.args.toSlice(arena); @@ -153,7 +152,6 @@ pub fn main(init: process.Init.Minimal) !void { var debounce_interval_ms: u16 = 50; var webui_listen: ?Io.net.IpAddress = null; var debug_pkg_config = false; - var debug_maker_leaks = false; var run_args: ?[]const []const u8 = null; if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { @@ -300,7 +298,7 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse fatal("unrecognized optimization mode: {s}", .{rest}); - } else if (mem.eql(u8, arg, "--debug-maker-leaks")) { + } else if (is_debug_mode and mem.eql(u8, arg, "--debug-maker-leaks")) { debug_maker_leaks = true; } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. @@ -543,7 +541,6 @@ pub fn main(init: process.Init.Minimal) !void { .memory_blocked_steps = .empty, .step_stack = .empty, .pkg_config = .{ .debug = debug_pkg_config }, - .debug_maker_leaks = debug_maker_leaks, .error_style = error_style, .multiline_errors = multiline_errors, @@ -1011,7 +1008,7 @@ fn makeStepNames( }; if (code == 0) { removePoisonedConfiguration(io, maker.scanned_config); - if (builtin.mode == .Debug and maker.debug_maker_leaks) return deinit(maker); + if (debugMakerLeaks()) return deinit(maker); } cleanup_task.await(io); // There is a defer above but an exit below. _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; @@ -2045,3 +2042,10 @@ fn removePoisonedConfiguration(io: Io, scanned_config: *const ScannedConfig) voi log.warn("failed deleting poisoned configuration file {s}: {t}", .{ scanned_config.path, err }); } } + +const is_debug_mode = builtin.mode == .Debug; +var debug_maker_leaks: bool = false; +inline fn debugMakerLeaks() bool { + if (!is_debug_mode) return false; + return debug_maker_leaks; +} -- 2.54.0