From 2e88ac8842a6a5bbd9fe654292ab416cadfaf8bf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Feb 2026 16:03:37 -0800 Subject: [PATCH] zig build: configure runner basics implemented --- lib/compiler/configure_runner.zig | 256 +- lib/compiler/{build_runner.zig => maker.zig} | 410 ++-- lib/{std/Build => compiler/maker}/Fuzz.zig | 2 +- lib/compiler/maker/Graph.zig | 92 + lib/compiler/maker/Package.zig | 12 + lib/compiler/maker/Step.zig | 850 +++++++ lib/compiler/maker/Step/Compile.zig | 1074 +++++++++ lib/compiler/maker/Step/InstallArtifact.zig | 96 + lib/compiler/maker/Step/Run.zig | 2127 +++++++++++++++++ lib/compiler/maker/Step/WriteFile.zig | 206 ++ lib/{std/Build => compiler/maker}/Watch.zig | 0 .../maker}/Watch/FsEvents.zig | 0 .../Build => compiler/maker}/WebServer.zig | 0 lib/std/Build.zig | 198 +- lib/std/Build/Step.zig | 913 +------ lib/std/Build/Step/CheckFile.zig | 4 +- lib/std/Build/Step/Compile.zig | 1078 +-------- lib/std/Build/Step/ConfigHeader.zig | 4 +- lib/std/Build/Step/Fail.zig | 4 +- lib/std/Build/Step/Fmt.zig | 4 +- lib/std/Build/Step/InstallArtifact.zig | 101 +- lib/std/Build/Step/InstallDir.zig | 4 +- lib/std/Build/Step/InstallFile.zig | 4 +- lib/std/Build/Step/ObjCopy.zig | 4 +- lib/std/Build/Step/Options.zig | 4 +- lib/std/Build/Step/Run.zig | 2116 +--------------- lib/std/Build/Step/TranslateC.zig | 4 +- lib/std/Build/Step/UpdateSourceFiles.zig | 4 +- lib/std/Build/Step/WriteFile.zig | 211 +- lib/std/zig.zig | 2 - lib/std/zig/Configuration.zig | 240 +- src/main.zig | 157 +- 32 files changed, 5192 insertions(+), 4989 deletions(-) rename lib/compiler/{build_runner.zig => maker.zig} (83%) rename lib/{std/Build => compiler/maker}/Fuzz.zig (99%) create mode 100644 lib/compiler/maker/Graph.zig create mode 100644 lib/compiler/maker/Package.zig create mode 100644 lib/compiler/maker/Step.zig create mode 100644 lib/compiler/maker/Step/Compile.zig create mode 100644 lib/compiler/maker/Step/InstallArtifact.zig create mode 100644 lib/compiler/maker/Step/Run.zig create mode 100644 lib/compiler/maker/Step/WriteFile.zig rename lib/{std/Build => compiler/maker}/Watch.zig (100%) rename lib/{std/Build => compiler/maker}/Watch/FsEvents.zig (100%) rename lib/{std/Build => compiler/maker}/WebServer.zig (100%) diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig index b1eafdb9ae7282fabf4102ee3eb3ec8c214383b0..78d3dff4c399392b1519d2d3d78368b5b4e2ffcd 100644 --- a/lib/compiler/configure_runner.zig +++ b/lib/compiler/configure_runner.zig @@ -8,12 +8,11 @@ const mem = std.mem; const process = std.process; const File = std.Io.File; const Step = std.Build.Step; -const Watch = std.Build.Watch; -const WebServer = std.Build.WebServer; const Allocator = std.mem.Allocator; const fatal = std.process.fatal; const Writer = std.Io.Writer; const Color = std.zig.Color; +const Configuration = std.Build.Configuration; pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); @@ -26,7 +25,11 @@ pub const std_options: std.Options = .{ pub fn main(init: process.Init.Minimal) !void { // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not // always the case. So, we do need a true gpa for some things. - var debug_gpa_state: std.heap.DebugAllocator(.{}) = .init; + var debug_gpa_state: std.heap.DebugAllocator(.{ + // We'd rather have `zig build` run faster than catch harmless leaks in + // the user's build.zig script. + .stack_trace_frames = 0, + }) = .init; defer _ = debug_gpa_state.deinit(); const gpa = debug_gpa_state.allocator(); @@ -47,11 +50,11 @@ pub fn main(init: process.Init.Minimal) !void { // skip my own exe name var arg_idx: usize = 1; - const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); - const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); - const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); - const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); - const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); + const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); + const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); + const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); + const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); const cwd: Io.Dir = .cwd(); @@ -66,8 +69,8 @@ pub fn main(init: process.Init.Minimal) !void { }; const local_cache_directory: std.Build.Cache.Directory = .{ - .path = cache_root, - .handle = try cwd.createDirPathOpen(io, cache_root, .{}), + .path = local_cache_root, + .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), }; const global_cache_directory: std.Build.Cache.Directory = .{ @@ -137,8 +140,6 @@ pub fn main(init: process.Init.Minimal) !void { if (try builder.addUserInputFlag(option_contents)) fatal(" access the help menu with 'zig build -h'", .{}); } - } else if (mem.eql(u8, arg, "--verbose")) { - builder.verbose = true; } else if (mem.startsWith(u8, arg, "-fsys=")) { const name = arg["-fsys=".len..]; graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); @@ -146,19 +147,14 @@ pub fn main(init: process.Init.Minimal) !void { const name = arg["-fno-sys=".len..]; graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); } else if (mem.eql(u8, arg, "--release")) { - builder.release_mode = .any; + graph.release_mode = .any; } else if (mem.startsWith(u8, arg, "--release=")) { const text = arg["--release=".len..]; - builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { + graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ arg, text, }); }; - } else if (mem.eql(u8, arg, "--search-prefix")) { - const search_prefix = nextArgOrFatal(args, &arg_idx); - builder.addSearchPrefix(search_prefix); - } else if (mem.eql(u8, arg, "--libc")) { - builder.libc_file = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--color")) { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); @@ -196,8 +192,6 @@ pub fn main(init: process.Init.Minimal) !void { style, @errorName(err), }); }; - } else if (mem.eql(u8, arg, "--debug-pkg-config")) { - builder.debug_pkg_config = true; } else if (mem.eql(u8, arg, "--debug-rt")) { graph.debug_compiler_runtime_libs = true; } else if (mem.eql(u8, arg, "--debug-compile-errors")) { @@ -209,71 +203,11 @@ pub fn main(init: process.Init.Minimal) !void { // but it is handled by the parent process. The build runner // only sees this flag. graph.system_package_mode = true; - } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { - // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--verbose-link")) { - builder.verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - builder.verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - builder.verbose_llvm_ir = "-"; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { - builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { - builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; - } else if (mem.eql(u8, arg, "--verbose-cimport")) { - builder.verbose_cimport = true; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - builder.verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - builder.verbose_llvm_cpu_features = true; - } else if (mem.eql(u8, arg, "-fincremental")) { - graph.incremental = true; - } else if (mem.eql(u8, arg, "-fno-incremental")) { - graph.incremental = false; - } else if (mem.eql(u8, arg, "-fwine")) { - builder.enable_wine = true; - } else if (mem.eql(u8, arg, "-fno-wine")) { - builder.enable_wine = false; - } else if (mem.eql(u8, arg, "-fqemu")) { - builder.enable_qemu = true; - } else if (mem.eql(u8, arg, "-fno-qemu")) { - builder.enable_qemu = false; - } else if (mem.eql(u8, arg, "-fwasmtime")) { - builder.enable_wasmtime = true; - } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - builder.enable_wasmtime = false; - } else if (mem.eql(u8, arg, "-frosetta")) { - builder.enable_rosetta = true; - } else if (mem.eql(u8, arg, "-fno-rosetta")) { - builder.enable_rosetta = false; - } else if (mem.eql(u8, arg, "-fdarling")) { - builder.enable_darling = true; - } else if (mem.eql(u8, arg, "-fno-darling")) { - builder.enable_darling = false; - } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { - graph.allow_so_scripts = true; - } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { - graph.allow_so_scripts = false; - } else if (mem.eql(u8, arg, "-freference-trace")) { - builder.reference_trace = 256; - } else if (mem.startsWith(u8, arg, "-freference-trace=")) { - const num = arg["-freference-trace=".len..]; - builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); - process.exit(1); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - builder.reference_trace = null; } else if (mem.cutPrefix(u8, arg, "-j")) |text| { const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); if (n < 1) fatal("number of jobs must be at least 1", .{}); threaded.setAsyncLimit(.limited(n)); - } else if (mem.eql(u8, arg, "--")) { - builder.args = argsRest(args, arg_idx); - break; } else { fatalWithHint("unrecognized argument: '{s}'", .{arg}); } @@ -289,6 +223,150 @@ pub fn main(init: process.Init.Minimal) !void { }; try builder.runBuild(root); + + var wc: Configuration.Wip = .init(gpa); + defer wc.deinit(); + + var stdout_buffer: [1024]u8 = undefined; + var file_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); + serialize(builder, &wc, &file_writer.interface) catch |err| switch (err) { + error.WriteFailed => fatal("failed to write configuration output: {t}", .{file_writer.err.?}), + error.OutOfMemory => |e| return e, + }; + + // This executable is short-lived and run in Debug mode, so we'd rather + // have `zig build` run faster than catch resource leaks in the user's + // build.zig script (or, frankly, this configure runner), therefore we call + // exit directly here rather than cleanExit. + process.exit(0); +} + +fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { + const graph = b.graph; + const arena = graph.arena; + const gpa = wc.gpa; + + // Starting from all top-level steps in `b`, traverse the entire step graph + // and add all step dependencies implied by module graphs. + const top_level_steps = b.top_level_steps.values(); + // Index corresponds to `Configuration.steps` index. + var step_map: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; + try step_map.ensureUnusedCapacity(arena, top_level_steps.len); + for (top_level_steps) |tls| { + step_map.putAssumeCapacityNoClobber(&tls.step, {}); + } + { + while (wc.steps.items.len < step_map.count()) { + const step = step_map.keys()[wc.steps.items.len]; + + // Set up any implied dependencies for this step. It's important that we do this first, so + // that the loop below discovers steps implied by the module graph. + try createModuleDependenciesForStep(step); + + try step_map.ensureUnusedCapacity(arena, step.dependencies.items.len); + for (step.dependencies.items) |other_step| { + step_map.putAssumeCapacity(other_step, {}); + } + + // Add and then de-duplicate dependencies. + const deps = d: { + const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len); + for (try wc.prepareDeps(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| + dep.* = @intCast(step_map.getIndex(dep_step).?); + break :d try wc.dedupeDeps(deps); + }; + + try wc.steps.ensureTotalCapacity(gpa, step_map.entries.capacity); + wc.steps.appendAssumeCapacity(.{ + .name = try wc.addString(step.name), + .flags = .{ .tag = step.tag }, + .deps = deps, + .extra_index = switch (step.tag) { + .top_level => e: { + const top_level: *Step.TopLevel = @fieldParentPtr("step", step); + break :e try wc.addExtra(@as(Configuration.Step.TopLevel, .{ + .description = try wc.addString(top_level.description), + })); + }, + .compile => @panic("TODO"), + .install_artifact => @panic("TODO"), + .install_file => @panic("TODO"), + .install_dir => @panic("TODO"), + .remove_dir => @panic("TODO"), + .fail => @panic("TODO"), + .fmt => @panic("TODO"), + .translate_c => @panic("TODO"), + .write_file => @panic("TODO"), + .update_source_files => @panic("TODO"), + .run => @panic("TODO"), + .check_file => @panic("TODO"), + .check_object => @panic("TODO"), + .config_header => @panic("TODO"), + .objcopy => @panic("TODO"), + .options => @panic("TODO"), + }, + }); + } + } + + try wc.unlazy_deps.ensureUnusedCapacity(gpa, graph.needed_lazy_dependencies.keys().len); + for (graph.needed_lazy_dependencies.keys()) |k| { + wc.unlazy_deps.appendAssumeCapacity(try wc.addString(k)); + } + + try wc.write(writer, .{ + .default_step = @intCast(step_map.getIndex(b.default_step).?), + }); +} + +/// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which +/// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. +fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { + const root_module = if (step.cast(Step.Compile)) |cs| root: { + break :root cs.root_module; + } else return; // not a compile step so no module dependencies + + // Starting from `root_module`, discover all modules in this graph. + const modules = root_module.getGraph().modules; + + // For each of those modules, set up the implied step dependencies. + for (modules) |mod| { + if (mod.root_source_file) |lp| lp.addStepDependencies(step); + for (mod.include_dirs.items) |include_dir| switch (include_dir) { + .path, + .path_system, + .path_after, + .framework_path, + .framework_path_system, + .embed_path, + => |lp| lp.addStepDependencies(step), + + .other_step => |other| { + other.getEmittedIncludeTree().addStepDependencies(step); + step.dependOn(&other.step); + }, + + .config_header_step => |other| step.dependOn(&other.step), + }; + for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); + for (mod.rpaths.items) |rpath| switch (rpath) { + .lazy_path => |lp| lp.addStepDependencies(step), + .special => {}, + }; + for (mod.link_objects.items) |link_object| switch (link_object) { + .static_path, + .assembly_file, + => |lp| lp.addStepDependencies(step), + .other_step => |other| step.dependOn(&other.step), + .system_lib => {}, + .c_source_file => |source| source.file.addStepDependencies(step), + .c_source_files => |source_files| source_files.root.addStepDependencies(step), + .win32_resource_file => |rc_source| { + rc_source.file.addStepDependencies(step); + for (rc_source.include_paths) |lp| lp.addStepDependencies(step); + }, + }; + } } fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { @@ -299,14 +377,17 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { return nextArg(args, idx) orelse { - std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); - process.exit(1); + fatal("expected argument after {q}\n access the help menu with \"zig build -h\"", .{ + args[idx.* - 1], + }); }; } -fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { - if (idx >= args.len) return null; - return args[idx..]; +fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { + const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); + if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); + const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); + return arg; } const ErrorStyle = enum { @@ -331,6 +412,5 @@ const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { - std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); - process.exit(1); + fatal(f ++ "\n access the help menu with \"zig build -h\"", args); } diff --git a/lib/compiler/build_runner.zig b/lib/compiler/maker.zig similarity index 83% rename from lib/compiler/build_runner.zig rename to lib/compiler/maker.zig index 439e46bb6e82bf93bf53652cc506c129781da740..44fe7170ab3b6043303880acb05b129c168d63d1 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/maker.zig @@ -1,4 +1,3 @@ -const runner = @This(); const builtin = @import("builtin"); const std = @import("std"); @@ -8,15 +7,17 @@ const fmt = std.fmt; const mem = std.mem; const process = std.process; const File = std.Io.File; -const Step = std.Build.Step; -const Watch = std.Build.Watch; -const WebServer = std.Build.WebServer; const Allocator = std.mem.Allocator; const fatal = std.process.fatal; const Writer = std.Io.Writer; +const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; -pub const root = @import("@build"); -pub const dependencies = @import("@dependencies"); +const Fuzz = @import("maker/Fuzz.zig"); +const Graph = @import("maker/Graph.zig"); +const Step = @import("maker/Step.zig"); +const Watch = @import("maker/Watch.zig"); +const WebServer = @import("maker/WebServer.zig"); pub const std_options: std.Options = .{ .side_channels_mitigations = .none, @@ -47,35 +48,36 @@ pub fn main(init: process.Init.Minimal) !void { // skip my own exe name var arg_idx: usize = 1; - const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); - const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); - const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); - const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); - const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + const zig_exe = cutArgPrefixOrFatal(args, &arg_idx, "--zig="); + const zig_lib_dir = cutArgPrefixOrFatal(args, &arg_idx, "--lib="); + const build_root = cutArgPrefixOrFatal(args, &arg_idx, "--build-root="); + const local_cache_root = cutArgPrefixOrFatal(args, &arg_idx, "--local-cache="); + const global_cache_root = cutArgPrefixOrFatal(args, &arg_idx, "--global-cache="); + const configure_path = cutArgPrefixOrFatal(args, &arg_idx, "--configure="); const cwd: Io.Dir = .cwd(); - const zig_lib_directory: std.Build.Cache.Directory = .{ + const zig_lib_directory: Cache.Directory = .{ .path = zig_lib_dir, .handle = try cwd.openDir(io, zig_lib_dir, .{}), }; - const build_root_directory: std.Build.Cache.Directory = .{ + const build_root_directory: Cache.Directory = .{ .path = build_root, .handle = try cwd.openDir(io, build_root, .{}), }; - const local_cache_directory: std.Build.Cache.Directory = .{ - .path = cache_root, - .handle = try cwd.createDirPathOpen(io, cache_root, .{}), + const local_cache_directory: Cache.Directory = .{ + .path = local_cache_root, + .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), }; - const global_cache_directory: std.Build.Cache.Directory = .{ + const global_cache_directory: Cache.Directory = .{ .path = global_cache_root, .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), }; - var graph: std.Build.Graph = .{ + var graph: Graph = .{ .io = io, .arena = arena, .cache = .{ @@ -101,18 +103,11 @@ pub fn main(init: process.Init.Minimal) !void { graph.cache.addPrefix(global_cache_directory); graph.cache.hash.addBytes(builtin.zig_version_string); - const builder = try std.Build.create( - &graph, - build_root_directory, - local_cache_directory, - dependencies.root_deps, - ); - var targets = std.array_list.Managed([]const u8).init(arena); var debug_log_scopes = std.array_list.Managed([]const u8).init(arena); var install_prefix: ?[]const u8 = null; - var dir_list = std.Build.DirList{}; + var dir_list: std.Build.DirList = .{}; var error_style: ErrorStyle = .verbose; var multiline_errors: MultilineErrors = .indent; var summary: ?Summary = null; @@ -122,11 +117,28 @@ pub fn main(init: process.Init.Minimal) !void { var color: Color = .auto; var help_menu = false; var steps_menu = false; - var output_tmp_nonce: ?[16]u8 = null; var watch = false; - var fuzz: ?std.Build.Fuzz.Mode = null; + var fuzz: ?Fuzz.Mode = null; var debounce_interval_ms: u16 = 50; var webui_listen: ?Io.net.IpAddress = null; + var verbose = false; + var sysroot: ?[]const u8 = null; + var search_prefixes: std.ArrayList([]const u8) = .empty; + var libc_file: ?[]const u8 = null; + var debug_pkg_config: bool = false; + // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, + // this will be the directory $glibc-build-dir/install/glibcs + // Given the example of the aarch64 target, this is the directory + // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. + // Also works for dynamic musl. + var libc_runtimes_dir: ?[]const u8 = null; + var enable_wine = false; + var enable_qemu = false; + var enable_wasmtime = false; + var enable_darling = false; + var enable_rosetta = false; + var reference_trace: ?u32 = null; + var run_args: ?[]const []const u8 = null; if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { if (std.meta.stringToEnum(ErrorStyle, str)) |style| { @@ -140,26 +152,23 @@ pub fn main(init: process.Init.Minimal) !void { } } + var configuration: Configuration = undefined; + { + var file = cwd.openFile(io, configure_path, .{}) catch |err| + fatal("failed to open configuration file {f}: {t}", .{ configure_path, err }); + defer file.close(io); + configuration = Configuration.load(arena, io, file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ configure_path, err }); + } + graph.configuration = &configuration; + graph.scanConfiguration(); + + std.log.err("TODO handle user -D options", .{}); + while (nextArg(args, &arg_idx)) |arg| { - if (mem.startsWith(u8, arg, "-Z")) { - if (arg.len != 18) fatalWithHint("bad argument: '{s}'", .{arg}); - output_tmp_nonce = arg[2..18].*; - } else if (mem.startsWith(u8, arg, "-D")) { - const option_contents = arg[2..]; - if (option_contents.len == 0) - fatalWithHint("expected option name after '-D'", .{}); - if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { - const option_name = option_contents[0..name_end]; - const option_value = option_contents[name_end + 1 ..]; - if (try builder.addUserInputOption(option_name, option_value)) - fatal(" access the help menu with 'zig build -h'", .{}); - } else { - if (try builder.addUserInputFlag(option_contents)) - fatal(" access the help menu with 'zig build -h'", .{}); - } - } else if (mem.startsWith(u8, arg, "-")) { + if (mem.startsWith(u8, arg, "-")) { if (mem.eql(u8, arg, "--verbose")) { - builder.verbose = true; + verbose = true; } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { help_menu = true; } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { @@ -172,15 +181,6 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.startsWith(u8, arg, "-fno-sys=")) { const name = arg["-fno-sys=".len..]; graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); - } else if (mem.eql(u8, arg, "--release")) { - builder.release_mode = .any; - } else if (mem.startsWith(u8, arg, "--release=")) { - const text = arg["--release=".len..]; - builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { - fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ - arg, text, - }); - }; } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { dir_list.lib_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { @@ -188,7 +188,7 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--prefix-include-dir")) { dir_list.include_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--sysroot")) { - builder.sysroot = nextArgOrFatal(args, &arg_idx); + sysroot = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--maxrss")) { const max_rss_text = nextArgOrFatal(args, &arg_idx); max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { @@ -235,10 +235,9 @@ pub fn main(init: process.Init.Minimal) !void { ); test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); } else if (mem.eql(u8, arg, "--search-prefix")) { - const search_prefix = nextArgOrFatal(args, &arg_idx); - builder.addSearchPrefix(search_prefix); + try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); } else if (mem.eql(u8, arg, "--libc")) { - builder.libc_file = nextArgOrFatal(args, &arg_idx); + libc_file = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--color")) { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); @@ -275,15 +274,6 @@ pub fn main(init: process.Init.Minimal) !void { next_arg, @errorName(err), }); }; - } else if (mem.eql(u8, arg, "--build-id")) { - builder.build_id = .fast; - } else if (mem.startsWith(u8, arg, "--build-id=")) { - const style = arg["--build-id=".len..]; - builder.build_id = std.zig.BuildId.parse(style) catch |err| { - fatal("unable to parse --build-id style '{s}': {s}", .{ - style, @errorName(err), - }); - }; } else if (mem.eql(u8, arg, "--debounce")) { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected u16 after '{s}'", .{arg}); @@ -304,17 +294,12 @@ pub fn main(init: process.Init.Minimal) !void { const next_arg = nextArgOrFatal(args, &arg_idx); try debug_log_scopes.append(next_arg); } else if (mem.eql(u8, arg, "--debug-pkg-config")) { - builder.debug_pkg_config = true; + debug_pkg_config = true; } else if (mem.eql(u8, arg, "--debug-rt")) { graph.debug_compiler_runtime_libs = .Debug; } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { - graph.debug_compiler_runtime_libs = - std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse - fatal("unrecognized optimization mode: '{s}'", .{rest}); - } else if (mem.eql(u8, arg, "--debug-compile-errors")) { - builder.debug_compile_errors = true; - } else if (mem.eql(u8, arg, "--debug-incremental")) { - builder.debug_incremental = true; + graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse + fatal("unrecognized optimization mode: {s}", .{rest}); } else if (mem.eql(u8, arg, "--system")) { // The usage text shows another argument after this parameter // but it is handled by the parent process. The build runner @@ -322,21 +307,7 @@ pub fn main(init: process.Init.Minimal) !void { graph.system_package_mode = true; } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--verbose-link")) { - builder.verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - builder.verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - builder.verbose_llvm_ir = "-"; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { - builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { - builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - builder.verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - builder.verbose_llvm_cpu_features = true; + libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--watch")) { watch = true; } else if (mem.eql(u8, arg, "--time-report")) { @@ -384,39 +355,39 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "-fno-incremental")) { graph.incremental = false; } else if (mem.eql(u8, arg, "-fwine")) { - builder.enable_wine = true; + enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { - builder.enable_wine = false; + enable_wine = false; } else if (mem.eql(u8, arg, "-fqemu")) { - builder.enable_qemu = true; + enable_qemu = true; } else if (mem.eql(u8, arg, "-fno-qemu")) { - builder.enable_qemu = false; + enable_qemu = false; } else if (mem.eql(u8, arg, "-fwasmtime")) { - builder.enable_wasmtime = true; + enable_wasmtime = true; } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - builder.enable_wasmtime = false; + enable_wasmtime = false; } else if (mem.eql(u8, arg, "-frosetta")) { - builder.enable_rosetta = true; + enable_rosetta = true; } else if (mem.eql(u8, arg, "-fno-rosetta")) { - builder.enable_rosetta = false; + enable_rosetta = false; } else if (mem.eql(u8, arg, "-fdarling")) { - builder.enable_darling = true; + enable_darling = true; } else if (mem.eql(u8, arg, "-fno-darling")) { - builder.enable_darling = false; + enable_darling = false; } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { graph.allow_so_scripts = true; } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { graph.allow_so_scripts = false; } else if (mem.eql(u8, arg, "-freference-trace")) { - builder.reference_trace = 256; + reference_trace = 256; } else if (mem.startsWith(u8, arg, "-freference-trace=")) { const num = arg["-freference-trace=".len..]; - builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); process.exit(1); }; } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - builder.reference_trace = null; + reference_trace = null; } else if (mem.cutPrefix(u8, arg, "-j")) |text| { const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); @@ -424,7 +395,7 @@ pub fn main(init: process.Init.Minimal) !void { threaded.setAsyncLimit(.limited(n)); graph.max_jobs = n; } else if (mem.eql(u8, arg, "--")) { - builder.args = argsRest(args, arg_idx); + run_args = argsRest(args, arg_idx); break; } else { fatalWithHint("unrecognized argument: '{s}'", .{arg}); @@ -453,51 +424,24 @@ pub fn main(init: process.Init.Minimal) !void { }); defer main_progress_node.end(); - builder.debug_log_scopes = debug_log_scopes.items; - builder.resolveInstallPrefix(install_prefix, dir_list); - { - var prog_node = main_progress_node.start("Configure", 0); - defer prog_node.end(); - try builder.runBuild(root); - createModuleDependencies(builder) catch @panic("OOM"); - } + graph.resolveInstallPrefix(install_prefix, dir_list); - if (graph.needed_lazy_dependencies.entries.len != 0) { - var buffer: std.ArrayList(u8) = .empty; - for (graph.needed_lazy_dependencies.keys()) |k| { - try buffer.appendSlice(arena, k); - try buffer.append(arena, '\n'); - } - const s = std.fs.path.sep_str; - const tmp_sub_path = "tmp" ++ s ++ (output_tmp_nonce orelse fatal("missing -Z arg", .{})); - local_cache_directory.handle.writeFile(io, .{ - .sub_path = tmp_sub_path, - .data = buffer.items, - .flags = .{ .exclusive = true }, - }) catch |err| { - fatal("unable to write configuration results to '{f}{s}': {s}", .{ - local_cache_directory, tmp_sub_path, @errorName(err), - }); - }; - process.exit(3); // Indicate configure phase failed with meaningful stdout. - } - - if (builder.validateUserInputDidItFail()) { + if (graph.validateUserInputDidItFail()) { fatal(" access the help menu with 'zig build -h'", .{}); } - validateSystemLibraryOptions(builder); + validateSystemLibraryOptions(&graph); if (help_menu) { var w = initStdoutWriter(io); - printUsage(builder, w) catch return stdout_writer_allocation.err.?; + printUsage(&graph, w) catch return stdout_writer_allocation.err.?; w.flush() catch return stdout_writer_allocation.err.?; return; } if (steps_menu) { var w = initStdoutWriter(io); - printSteps(builder, w) catch return stdout_writer_allocation.err.?; + printSteps(&graph, w) catch return stdout_writer_allocation.err.?; w.flush() catch return stdout_writer_allocation.err.?; return; } @@ -530,7 +474,7 @@ pub fn main(init: process.Init.Minimal) !void { run.max_rss_is_default = true; } - prepare(arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { + prepare(arena, &graph, targets.items, &run) catch |err| switch (err) { error.DependencyLoopDetected, error.InsufficientMemory => { // Perhaps in the future there could be an Advanced Options flag // such as --debug-build-runner-leaks which would make this code @@ -573,13 +517,7 @@ pub fn main(init: process.Init.Minimal) !void { }) { if (run.web_server) |*ws| ws.startBuild(); - try runStepNames( - builder, - targets.items, - main_progress_node, - &run, - fuzz, - ); + try runStepNames(graph, targets.items, main_progress_node, &run, fuzz); if (run.web_server) |*web_server| { if (fuzz) |mode| if (mode != .forever) fatal( @@ -678,23 +616,19 @@ const Run = struct { summary: Summary, }; -fn prepare( - arena: Allocator, - b: *std.Build, - step_names: []const []const u8, - run: *Run, - seed: u32, -) !void { +fn prepare(graph: *Graph, step_names: []const []const u8, run: *Run) !void { + const arena = graph.arena; + const seed: u32 = graph.random_seed; const gpa = run.gpa; const step_stack = &run.step_stack; if (step_names.len == 0) { - try step_stack.put(gpa, b.default_step, {}); + try step_stack.put(gpa, graph.configuration.default_step, {}); } else { try step_stack.ensureUnusedCapacity(gpa, step_names.len); for (0..step_names.len) |i| { const step_name = step_names[step_names.len - i - 1]; - const s = b.top_level_steps.get(step_name) orelse { + const s = graph.top_level_steps.get(step_name) orelse { std.log.info("access the help menu with \"zig build -h\"", .{}); fatal("no step named '{s}'", .{step_name}); }; @@ -709,7 +643,7 @@ fn prepare( rand.shuffle(*Step, starting_steps); for (starting_steps) |s| { - try constructGraphAndCheckForDependencyLoop(gpa, b, s, &run.step_stack, rand); + try constructGraphAndCheckForDependencyLoop(gpa, s, &run.step_stack, rand); } { @@ -745,14 +679,13 @@ fn prepare( } fn runStepNames( - b: *std.Build, + graph: *Graph, step_names: []const []const u8, parent_prog_node: std.Progress.Node, run: *Run, - fuzz: ?std.Build.Fuzz.Mode, + fuzz: ?Fuzz.Mode, ) !void { const gpa = run.gpa; - const graph = b.graph; const io = graph.io; const step_stack = &run.step_stack; @@ -775,7 +708,7 @@ fn runStepNames( var group: Io.Group = .init; defer group.cancel(io); // Start working on all of the initial steps... - for (initial_set.items) |s| try stepReady(&group, b, s, step_prog, run); + for (initial_set.items) |s| try stepReady(&group, s, step_prog, run); // ...and `makeStep` will trigger every other step when their last dependency finishes. try group.await(io); } @@ -848,7 +781,7 @@ fn runStepNames( } assert(mode == .limit); - var f = std.Build.Fuzz.init( + var f = Fuzz.init( gpa, io, step_stack.keys(), @@ -938,13 +871,13 @@ fn runStepNames( var print_node: PrintNode = .{ .parent = null }; if (step_names.len == 0) { print_node.last = true; - printTreeStep(b, b.default_step, run, t, &print_node, &step_stack_copy) catch {}; + printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {}; } else { - const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: { + const last_index = if (run.summary == .all) graph.top_level_steps.count() else blk: { var i: usize = step_names.len; while (i > 0) { i -= 1; - const step = b.top_level_steps.get(step_names[i]).?.step; + const step = graph.top_level_steps.get(step_names[i]).?.step; const found = switch (run.summary) { .all, .line, .none => unreachable, .failures => step.state != .success, @@ -952,12 +885,12 @@ fn runStepNames( }; if (found) break :blk i; } - break :blk b.top_level_steps.count(); + break :blk graph.top_level_steps.count(); }; for (step_names, 0..) |step_name, i| { - const tls = b.top_level_steps.get(step_name).?; + const tls = graph.top_level_steps.get(step_name).?; print_node.last = i + 1 == last_index; - printTreeStep(b, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; + printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; } } w.writeByte('\n') catch {}; @@ -1173,7 +1106,7 @@ fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void { } fn printTreeStep( - b: *std.Build, + graph: *Graph, s: *Step, run: *const Run, stderr: Io.Terminal, @@ -1231,7 +1164,7 @@ fn printTreeStep( .parent = parent_node, .last = i == last_index, }; - try printTreeStep(b, dep, run, stderr, &print_node, step_stack); + try printTreeStep(graph, dep, run, stderr, &print_node, step_stack); } } else { if (s.dependencies.items.len == 0) { @@ -1258,7 +1191,6 @@ fn printTreeStep( /// random order fn constructGraphAndCheckForDependencyLoop( gpa: Allocator, - b: *std.Build, s: *Step, step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), rand: std.Random, @@ -1282,8 +1214,8 @@ fn constructGraphAndCheckForDependencyLoop( for (deps) |dep| { try step_stack.put(gpa, dep, {}); - try dep.dependants.append(b.allocator, s); - constructGraphAndCheckForDependencyLoop(gpa, b, dep, step_stack, rand) catch |err| { + try dep.dependants.append(gpa, s); + constructGraphAndCheckForDependencyLoop(gpa, dep, step_stack, rand) catch |err| { if (err == error.DependencyLoopDetected) { std.debug.print(" {s}\n", .{s.name}); } @@ -1311,13 +1243,12 @@ fn constructGraphAndCheckForDependencyLoop( /// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked /// steps after "make" completes for `s`. fn makeStep( + graph: *Graph, group: *Io.Group, - b: *std.Build, s: *Step, root_prog_node: std.Progress.Node, run: *Run, ) Io.Cancelable!void { - const graph = b.graph; const io = graph.io; const gpa = run.gpa; @@ -1404,26 +1335,26 @@ fn makeStep( } } for (dispatch_set.items) |candidate| { - group.async(io, makeStep, .{ group, b, candidate, root_prog_node, run }); + group.async(io, makeStep, .{ graph, group, candidate, root_prog_node, run }); } } for (s.dependants.items) |dependant| { // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { - try stepReady(group, b, dependant, root_prog_node, run); + try stepReady(graph, group, dependant, root_prog_node, run); } } } fn stepReady( + graph: *Graph, group: *Io.Group, - b: *std.Build, s: *Step, root_prog_node: std.Progress.Node, run: *Run, ) !void { - const io = b.graph.io; + const io = graph.io; if (s.max_rss != 0) { try run.max_rss_mutex.lock(io); defer run.max_rss_mutex.unlock(io); @@ -1434,7 +1365,7 @@ fn stepReady( } run.available_rss -= s.max_rss; } - group.async(io, makeStep, .{ group, b, s, root_prog_node, run }); + group.async(io, makeStep, .{ graph, group, s, root_prog_node, run }); } pub fn printErrorMessages( @@ -1523,10 +1454,10 @@ pub fn printErrorMessages( try writer.writeByte('\n'); } -fn printSteps(builder: *std.Build, w: *Writer) !void { - const arena = builder.graph.arena; - for (builder.top_level_steps.values()) |top_level_step| { - const name = if (&top_level_step.step == builder.default_step) +fn printSteps(graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + for (graph.top_level_steps.values()) |top_level_step| { + const name = if (&top_level_step.step == graph.default_step) try fmt.allocPrint(arena, "{s} (default)", .{top_level_step.step.name}) else top_level_step.step.name; @@ -1534,26 +1465,26 @@ fn printSteps(builder: *std.Build, w: *Writer) !void { } } -fn printUsage(b: *std.Build, w: *Writer) !void { - const arena = b.graph.arena; +fn printUsage(graph: *Graph, w: *Writer) !void { + const arena = graph.arena; try w.print( \\Usage: {s} build [steps] [options] \\ \\Steps: \\ - , .{b.graph.zig_exe}); - try printSteps(b, w); + , .{graph.zig_exe}); + try printSteps(graph, w); try w.writeAll( \\ \\Project-Specific Options: \\ ); - if (b.available_options_list.items.len == 0) { + if (graph.available_options_list.items.len == 0) { try w.print(" (none)\n", .{}); } else { - for (b.available_options_list.items) |option| { + for (graph.available_options_list.items) |option| { const name = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ option.name, option.type_id }); try w.print("{s:<30} {s}\n", .{ name, option.description }); if (option.enum_options) |enum_options| { @@ -1597,10 +1528,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void { \\ Available System Integrations: Enabled: \\ ); - if (b.graph.system_library_options.entries.len == 0) { + if (graph.system_library_options.entries.len == 0) { try w.writeAll(" (none) -\n"); } else { - for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { + for (graph.system_library_options.keys(), graph.system_library_options.values()) |k, v| { const status = switch (v) { .declared_enabled => "yes", .declared_disabled => "no", @@ -1702,10 +1633,17 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { } fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); - process.exit(1); - }; + return nextArg(args, idx) orelse + fatal("expected argument after {q}\n access the help menu with \"zig build -h\"", .{args[idx.* - 1]}); +} + +fn cutArgPrefixOrFatal(args: []const [:0]const u8, idx: *usize, prefix: []const u8) []const u8 { + if (nextArg(args, idx)) |next_arg| { + if (mem.cutPrefix(u8, next_arg, prefix)) |arg| { + return arg; + } + } + fatal("expected argument after {q} to start with {q}", .{ args[idx.* - 1], prefix }); } fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { @@ -1740,9 +1678,9 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { process.exit(1); } -fn validateSystemLibraryOptions(b: *std.Build) void { +fn validateSystemLibraryOptions(graph: *Graph) void { var bad = false; - for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { + for (graph.system_library_options.keys(), graph.system_library_options.values()) |k, v| { switch (v) { .user_disabled, .user_enabled => { // The user tried to enable or disable a system library integration, but @@ -1759,84 +1697,6 @@ fn validateSystemLibraryOptions(b: *std.Build) void { } } -/// Starting from all top-level steps in `b`, traverses the entire step graph -/// and adds all step dependencies implied by module graphs. -fn createModuleDependencies(b: *std.Build) Allocator.Error!void { - const arena = b.graph.arena; - - var all_steps: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; - var next_step_idx: usize = 0; - - try all_steps.ensureUnusedCapacity(arena, b.top_level_steps.count()); - for (b.top_level_steps.values()) |tls| { - all_steps.putAssumeCapacityNoClobber(&tls.step, {}); - } - - while (next_step_idx < all_steps.count()) { - const step = all_steps.keys()[next_step_idx]; - next_step_idx += 1; - - // Set up any implied dependencies for this step. It's important that we do this first, so - // that the loop below discovers steps implied by the module graph. - try createModuleDependenciesForStep(step); - - try all_steps.ensureUnusedCapacity(arena, step.dependencies.items.len); - for (step.dependencies.items) |other_step| { - all_steps.putAssumeCapacity(other_step, {}); - } - } -} - -/// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which -/// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. -fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { - const root_module = if (step.cast(Step.Compile)) |cs| root: { - break :root cs.root_module; - } else return; // not a compile step so no module dependencies - - // Starting from `root_module`, discover all modules in this graph. - const modules = root_module.getGraph().modules; - - // For each of those modules, set up the implied step dependencies. - for (modules) |mod| { - if (mod.root_source_file) |lp| lp.addStepDependencies(step); - for (mod.include_dirs.items) |include_dir| switch (include_dir) { - .path, - .path_system, - .path_after, - .framework_path, - .framework_path_system, - .embed_path, - => |lp| lp.addStepDependencies(step), - - .other_step => |other| { - other.getEmittedIncludeTree().addStepDependencies(step); - step.dependOn(&other.step); - }, - - .config_header_step => |other| step.dependOn(&other.step), - }; - for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); - for (mod.rpaths.items) |rpath| switch (rpath) { - .lazy_path => |lp| lp.addStepDependencies(step), - .special => {}, - }; - for (mod.link_objects.items) |link_object| switch (link_object) { - .static_path, - .assembly_file, - => |lp| lp.addStepDependencies(step), - .other_step => |other| step.dependOn(&other.step), - .system_lib => {}, - .c_source_file => |source| source.file.addStepDependencies(step), - .c_source_files => |source_files| source_files.root.addStepDependencies(step), - .win32_resource_file => |rc_source| { - rc_source.file.addStepDependencies(step); - for (rc_source.include_paths) |lp| lp.addStepDependencies(step); - }, - }; - } -} - var stdio_buffer_allocation: [256]u8 = undefined; var stdout_writer_allocation: Io.File.Writer = undefined; @@ -1847,7 +1707,7 @@ fn initStdoutWriter(io: Io) *Writer { fn cleanTmpFiles(io: Io, steps: []const *Step) void { for (steps) |step| { - const wf = step.cast(std.Build.Step.WriteFile) orelse continue; + const wf = step.cast(Step.WriteFile) orelse continue; if (wf.mode != .tmp) continue; const path = wf.generated_directory.path orelse continue; Io.Dir.cwd().deleteTree(io, path) catch |err| { diff --git a/lib/std/Build/Fuzz.zig b/lib/compiler/maker/Fuzz.zig similarity index 99% rename from lib/std/Build/Fuzz.zig rename to lib/compiler/maker/Fuzz.zig index 9d3551a565a76ebfcae3f21e3e753cb553ef3cf2..a9b01e8111c0bf6034d740461d654f455e557813 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/compiler/maker/Fuzz.zig @@ -1,4 +1,4 @@ -const std = @import("../std.zig"); +const std = @import("Std"); const Io = std.Io; const Build = std.Build; const Cache = Build.Cache; diff --git a/lib/compiler/maker/Graph.zig b/lib/compiler/maker/Graph.zig new file mode 100644 index 0000000000000000000000000000000000000000..ed5e297b59339ce09271cf7a4ec49b4df4046a97 --- /dev/null +++ b/lib/compiler/maker/Graph.zig @@ -0,0 +1,92 @@ +//! Shared maker state among all steps. +const Graph = @This(); + +const std = @import("std"); +const Io = std.Io; +const Allocator = std.mem.Allocator; +const Configuration = std.Build.Configuration; + +const Step = @import("Step.zig"); +const Package = @import("Package.zig"); + +io: Io, +/// Process lifetime. +arena: Allocator, +system_library_options: std.StringArrayHashMapUnmanaged(std.Build.SystemLibraryMode), +system_package_mode: bool, +debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, +cache: std.Build.Cache, +zig_exe: [:0]const u8, +environ_map: std.process.Environ.Map, +global_cache_root: std.Build.Cache.Directory, +zig_lib_directory: std.Build.Cache.Directory, +incremental: ?bool, +random_seed: u32, +allow_so_scripts: ?bool, +time_report: bool, +/// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also +/// respects the '--color' flag. +stderr_mode: ?Io.Terminal.Mode, + +configuration: *const Configuration, +top_level_steps: std.AutoArrayHashMapUnmanaged(Configuration.String, Configuration.Step.Index), + +pub const DirList = struct { + lib_dir: ?[]const u8 = null, + exe_dir: ?[]const u8 = null, + include_dir: ?[]const u8 = null, +}; + +/// This function is intended to be called by lib/build_runner.zig, not a build.zig file. +pub fn resolveInstallPrefix(graph: *Graph, p: *Package, install_prefix: ?[]const u8, dir_list: DirList) !void { + if (p.dest_dir) |dest_dir| { + p.install_prefix = install_prefix orelse "/usr"; + p.install_path = b.pathJoin(&.{ dest_dir, p.install_prefix }); + } else { + p.install_prefix = install_prefix orelse + (p.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); + b.install_path = b.install_prefix; + } + + var lib_list = [_][]const u8{ b.install_path, "lib" }; + var exe_list = [_][]const u8{ b.install_path, "bin" }; + var h_list = [_][]const u8{ b.install_path, "include" }; + + if (dir_list.lib_dir) |dir| { + if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; + lib_list[1] = dir; + } + + if (dir_list.exe_dir) |dir| { + if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; + exe_list[1] = dir; + } + + if (dir_list.include_dir) |dir| { + if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; + h_list[1] = dir; + } + + b.lib_dir = b.pathJoin(&lib_list); + b.exe_dir = b.pathJoin(&exe_list); + b.h_dir = b.pathJoin(&h_list); +} + +fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { + // Create an installation directory local to this package. This will be used when + // dependant packages require a standard prefix, such as include directories for C headers. + var hash = b.graph.cache.hash; + // Random bytes to make unique. Refresh this with new random bytes when + // implementation is modified in a non-backwards-compatible way. + hash.add(@as(u32, 0xd8cb0056)); + hash.addBytes(b.dep_prefix); + + var wyhash = std.hash.Wyhash.init(0); + hashUserInputOptionsMap(b.allocator, b.user_input_options, &wyhash); + hash.add(wyhash.final()); + + const digest = hash.final(); + const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); + b.resolveInstallPrefix(install_prefix, .{}); +} + diff --git a/lib/compiler/maker/Package.zig b/lib/compiler/maker/Package.zig new file mode 100644 index 0000000000000000000000000000000000000000..e3ad442f69d6fa215b7dde6394211bc9cda283bf --- /dev/null +++ b/lib/compiler/maker/Package.zig @@ -0,0 +1,12 @@ +const Package = @This(); + +const std = @import("std"); + +install_prefix: []const u8, +install_path: []const u8, +dest_dir: ?[]const u8, +lib_dir: []const u8, +exe_dir: []const u8, +h_dir: []const u8, +/// Path to the directory containing build.zig. +build_root: std.Build.Cache.Path, diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/maker/Step.zig new file mode 100644 index 0000000000000000000000000000000000000000..295993ec245d79d5edde3444f8ed7aec7e2c6b5b --- /dev/null +++ b/lib/compiler/maker/Step.zig @@ -0,0 +1,850 @@ +const Step = @This(); + +const std = @import("std"); +const Io = std.Io; +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const assert = std.debug.assert; + +const WebServer = @import("WebServer.zig"); + +pub const Compile = @import("Step/Compile.zig"); +pub const Run = @import("Step/Run.zig"); + +state: State, +makeFn: MakeFn, +dependants: std.ArrayList(*Step), +/// Collects the set of files that retrigger this step to run. +/// +/// This is used by the build system's implementation of `--watch` but it can +/// also be potentially useful for IDEs to know what effects editing a +/// particular file has. +/// +/// Populated within `make`. Implementation may choose to clear and repopulate, +/// retain previous value, or update. +inputs: Inputs = .init, +pending_deps: u32, + +result_error_msgs: std.ArrayList([]const u8), +result_error_bundle: std.zig.ErrorBundle, +result_stderr: []const u8, +result_cached: bool, +result_duration_ns: ?u64, +/// 0 means unavailable or not reported. +result_peak_rss: usize, +/// If the step is failed and this field is populated, this is the command which failed. +/// This field may be populated even if the step succeeded. +result_failed_command: ?[]const u8, +test_results: TestResults, + + +pub const State = enum { + precheck_unstarted, + precheck_started, + /// This is also used to indicate "dirty" steps that have been modified + /// after a previous build completed, in which case, the step may or may + /// not have been completed before. Either way, one or more of its direct + /// file system inputs have been modified, meaning that the step needs to + /// be re-evaluated. + precheck_done, + dependency_failure, + success, + failure, + /// This state indicates that the step did not complete, however, it also did not fail, + /// and it is safe to continue executing its dependencies. + skipped, + /// This step was skipped because it specified a max_rss that exceeded the runner's maximum. + /// It is not safe to run its dependencies. + skipped_oom, +}; + +pub const Inputs = struct { + table: Table, + + pub const init: Inputs = .{ + .table = .{}, + }; + + pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, Files, Cache.Path.TableAdapter, false); + /// The special file name "." means any changes inside the directory. + pub const Files = std.ArrayList([]const u8); + + pub fn populated(inputs: *Inputs) bool { + return inputs.table.count() != 0; + } + + pub fn clear(inputs: *Inputs, gpa: Allocator) void { + for (inputs.table.values()) |*files| files.deinit(gpa); + inputs.table.clearRetainingCapacity(); + } +}; + +pub const TestResults = struct { + /// The total number of tests in the step. Every test has a "status" from the following: + /// * passed + /// * skipped + /// * failed cleanly + /// * crashed + /// * timed out + test_count: u32 = 0, + + /// The number of tests which were skipped (`error.SkipZigTest`). + skip_count: u32 = 0, + /// The number of tests which failed cleanly. + fail_count: u32 = 0, + /// The number of tests which terminated unexpectedly, i.e. crashed. + crash_count: u32 = 0, + /// The number of tests which timed out. + timeout_count: u32 = 0, + + /// The number of detected memory leaks. The associated test may still have passed; indeed, *all* + /// individual tests may have passed. However, the step as a whole fails if any test has leaks. + leak_count: u32 = 0, + /// The number of detected error logs. The associated test may still have passed; indeed, *all* + /// individual tests may have passed. However, the step as a whole fails if any test logs errors. + log_err_count: u32 = 0, + + pub fn isSuccess(tr: TestResults) bool { + // all steps are success or skip + return tr.fail_count == 0 and + tr.crash_count == 0 and + tr.timeout_count == 0 and + // no (otherwise successful) step leaked memory or logged errors + tr.leak_count == 0 and + tr.log_err_count == 0; + } + + /// Computes the number of tests which passed from the other values. + pub fn passCount(tr: TestResults) u32 { + return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count; + } +}; + +pub const MakeOptions = struct { + progress_node: std.Progress.Node, + watch: bool, + web_server: ?*WebServer, + /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. + unit_test_timeout_ns: ?u64, + /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. + gpa: Allocator, +}; + +pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; + +/// If the Step's `make` function reports `error.MakeFailed`, it indicates they +/// have already reported the error. Otherwise, we add a simple error report +/// here. +pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { + const arena = s.owner.allocator; + const graph = s.owner.graph; + const io = graph.io; + + var start_ts: ?Io.Timestamp = t: { + if (!graph.time_report) break :t null; + if (s.id == .compile) break :t null; + if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; + break :t Io.Clock.awake.now(io); + }; + const make_result = s.makeFn(s, options); + if (start_ts) |*ts| { + const duration = ts.untilNow(io, .awake); + options.web_server.?.updateTimeReportGeneric(s, duration); + } + + make_result catch |err| switch (err) { + error.MakeFailed, error.MakeSkipped => |e| return e, + else => { + s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); + return error.MakeFailed; + }, + }; + + if (!s.test_results.isSuccess()) { + return error.MakeFailed; + } + + if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { + const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ + s.result_peak_rss, s.max_rss, + }) catch @panic("OOM"); + s.result_error_msgs.append(arena, msg) catch @panic("OOM"); + } +} + +fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { + _ = options; + + var all_cached = true; + + for (step.dependencies.items) |dep| { + all_cached = all_cached and dep.result_cached; + } + + step.result_cached = all_cached; +} + +/// Implementation detail of file watching. Prepares the step for being re-evaluated. +/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. +pub fn invalidateResult(step: *Step, gpa: Allocator) bool { + if (step.state == .precheck_done) return false; + assert(step.pending_deps == 0); + step.state = .precheck_done; + step.reset(gpa); + for (step.dependants.items) |dependant| { + _ = dependant.invalidateResult(gpa); + dependant.pending_deps += 1; + } + return true; +} + +/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. +pub fn reset(step: *Step, gpa: Allocator) void { + assert(step.state == .precheck_done); + + if (step.result_failed_command) |cmd| gpa.free(cmd); + + step.result_error_msgs.clearRetainingCapacity(); + step.result_stderr = ""; + step.result_cached = false; + step.result_duration_ns = null; + step.result_peak_rss = 0; + step.result_failed_command = null; + step.test_results = .{}; + step.clearWatchInputs(); + + step.result_error_bundle.deinit(gpa); + step.result_error_bundle = std.zig.ErrorBundle.empty; +} + +/// Populates `s.result_failed_command`. +pub fn captureChildProcess( + s: *Step, + gpa: Allocator, + progress_node: std.Progress.Node, + argv: []const []const u8, +) !std.process.RunResult { + const graph = s.owner.graph; + const arena = graph.arena; + const io = graph.io; + + // If an error occurs, it's happened in this command: + assert(s.result_failed_command == null); + s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); + + try handleChildProcUnsupported(s); + try handleVerbose(s.owner, .inherit, argv); + + const result = std.process.run(arena, io, .{ + .argv = argv, + .environ_map = &graph.environ_map, + .progress_node = progress_node, + }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); + + if (result.stderr.len > 0) { + try s.result_error_msgs.append(arena, result.stderr); + } + + return result; +} + +pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { + try step.addError(fmt, args); + return error.MakeFailed; +} + +pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { + const arena = step.owner.allocator; + const msg = try std.fmt.allocPrint(arena, fmt, args); + try step.result_error_msgs.append(arena, msg); +} + +pub const ZigProcess = struct { + child: std.process.Child, + multi_reader_buffer: Io.File.MultiReader.Buffer(2), + multi_reader: Io.File.MultiReader, + progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn, + + pub const StreamEnum = enum { stdout, stderr }; + + pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void { + zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null; + } + + pub fn deinit(zp: *ZigProcess, io: Io) void { + zp.child.kill(io); + zp.multi_reader.deinit(); + zp.* = undefined; + } +}; + +/// Assumes that argv contains `--listen=-` and that the process being spawned +/// is the zig compiler - the same version that compiled the build runner. +/// Populates `s.result_failed_command`. +pub fn evalZigProcess( + s: *Step, + argv: []const []const u8, + prog_node: std.Progress.Node, + watch: bool, + web_server: ?*WebServer, + gpa: Allocator, +) !?Cache.Path { + const b = s.owner; + const io = b.graph.io; + + // If an error occurs, it's happened in this command: + assert(s.result_failed_command == null); + s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); + + if (s.getZigProcess()) |zp| update: { + assert(watch); + if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index); + zp.progress_ipc_index = null; + var exited = false; + defer if (exited) { + s.cast(Compile).?.zig_process = null; + zp.deinit(io); + gpa.destroy(zp); + } else zp.saveState(prog_node); + const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { + error.BrokenPipe, error.EndOfStream => |reason| { + std.log.info("{s} restart required: {t}", .{ argv[0], reason }); + // Process restart required. + const term = zp.child.wait(io) catch |e| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); + }; + _ = term; + exited = true; + break :update; + }, + else => |e| return e, + }; + + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + } + + if (s.result_error_msgs.items.len > 0 and result == null) { + // Crash detected. + const term = zp.child.wait(io) catch |e| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + exited = true; + try handleChildProcessTerm(s, term); + return error.MakeFailed; + } + + return result; + } + assert(argv.len != 0); + + try handleChildProcUnsupported(s); + try handleVerbose(s.owner, .inherit, argv); + + const zp = try gpa.create(ZigProcess); + defer if (!watch) gpa.destroy(zp); + + zp.child = std.process.spawn(io, .{ + .argv = argv, + .environ_map = &b.graph.environ_map, + .stdin = .pipe, + .stdout = .pipe, + .stderr = .pipe, + .request_resource_usage_statistics = true, + .progress_node = prog_node, + }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); + + zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ + zp.child.stdout.?, zp.child.stderr.?, + }); + if (watch) s.cast(Compile).?.zig_process = zp; + defer if (!watch) zp.deinit(io); + + const result = result: { + defer if (watch) zp.saveState(prog_node); + break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); + }; + + if (!watch) { + // Send EOF to stdin. + zp.child.stdin.?.close(io); + zp.child.stdin = null; + + const term = zp.child.wait(io) catch |err| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + + // Special handling for Compile step that is expecting compile errors. + if (s.cast(Compile)) |compile| switch (term) { + .exited => { + // Note that the exit code may be 0 in this case due to the + // compiler server protocol. + if (compile.expect_errors != null) { + return error.NeedCompileErrorCheck; + } + }, + else => {}, + }; + + try handleChildProcessTerm(s, term); + } + + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + } + + return result; +} + +/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. +pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { + const b = s.owner; + const io = b.graph.io; + const src_path = src_lazy_path.getPath3(b, s); + try handleVerbose(b, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); + return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| + return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); +} + +/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. +pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { + const b = s.owner; + const io = b.graph.io; + try handleVerbose(b, .inherit, &.{ "install", "-d", dest_path }); + return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| + return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); +} + +fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebServer, gpa: Allocator) !?Path { + const b = s.owner; + const arena = b.allocator; + const io = b.graph.io; + + const start_ts = Io.Clock.awake.now(io); + + try sendMessage(io, zp.child.stdin.?, .update); + if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); + + var result: ?Path = null; + var eos_err: error{EndOfStream}!void = {}; + + const stdout = zp.multi_reader.fileReader(0); + + while (true) { + const Header = std.zig.Server.Message.Header; + const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) { + error.EndOfStream => break, + error.ReadFailed => return stdout.err.?, + }; + const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) { + error.EndOfStream => |e| { + // Better to report the crash with stderr below, but we set + // this in case the child exits successfully while violating + // this protocol. + eos_err = e; + break; + }, + error.ReadFailed => return stdout.err.?, + }; + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return s.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + } + }, + .error_bundle => { + s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); + // This message indicates the end of the update. + if (watch) break; + }, + .emit_digest => { + const EmitDigest = std.zig.Server.Message.EmitDigest; + const emit_digest: *align(1) const EmitDigest = @ptrCast(body); + s.result_cached = emit_digest.flags.cache_hit; + const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; + result = .{ + .root_dir = b.cache_root, + .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), + }; + }, + .file_system_inputs => { + s.clearWatchInputs(); + var it = std.mem.splitScalar(u8, body, 0); + while (it.next()) |prefixed_path| { + const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); + const sub_path = try arena.dupe(u8, prefixed_path[1..]); + const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; + switch (prefix_index) { + .cwd => { + const path: Cache.Path = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .zig_lib => zl: { + if (s.cast(Step.Compile)) |compile| { + if (compile.zig_lib_dir) |zig_lib_dir| { + const lp = try zig_lib_dir.join(arena, sub_path); + try addWatchInput(s, lp); + break :zl; + } + } + const path: Cache.Path = .{ + .root_dir = s.owner.graph.zig_lib_directory, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .local_cache => { + const path: Cache.Path = .{ + .root_dir = b.cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .global_cache => { + const path: Cache.Path = .{ + .root_dir = s.owner.graph.global_cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + } + } + }, + .time_report => if (web_server) |ws| { + const TimeReport = std.zig.Server.Message.TimeReport; + const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); + ws.updateTimeReportCompile(.{ + .compile = s.cast(Step.Compile).?, + .use_llvm = tr.flags.use_llvm, + .stats = tr.stats, + .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), + .llvm_pass_timings_len = tr.llvm_pass_timings_len, + .files_len = tr.files_len, + .decls_len = tr.decls_len, + .trailing = body[@sizeOf(TimeReport)..], + }); + }, + else => {}, // ignore other messages + } + } + + s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()); + + const stderr_contents = zp.multi_reader.reader(1).buffered(); + if (stderr_contents.len > 0) { + try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); + } + + try eos_err; + + return result; +} + +pub fn getZigProcess(s: *Step) ?*ZigProcess { + return switch (s.id) { + .compile => s.cast(Compile).?.zig_process, + else => null, + }; +} + +fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + var w = file.writer(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +pub fn handleVerbose( + b: *Build, + cwd: std.process.Child.Cwd, + argv: []const []const u8, +) error{OutOfMemory}!void { + return handleVerbose2(b, cwd, null, argv); +} + +pub fn handleVerbose2( + b: *Build, + cwd: std.process.Child.Cwd, + opt_env: ?*const std.process.Environ.Map, + argv: []const []const u8, +) error{OutOfMemory}!void { + if (b.verbose) { + const graph = b.graph; + // Intention of verbose is to print all sub-process command lines to + // stderr before spawning them. + const text = try allocPrintCmd(b.allocator, cwd, if (opt_env) |env| .{ + .child = env, + .parent = &graph.environ_map, + } else null, argv); + std.debug.print("{s}\n", .{text}); + } +} + +/// Asserts that the caller has already populated `s.result_failed_command`. +pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { + if (!std.process.can_spawn) { + return s.fail("unable to spawn process: host cannot spawn child processes", .{}); + } +} + +/// Asserts that the caller has already populated `s.result_failed_command`. +pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { + assert(s.result_failed_command != null); + return switch (term) { + .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), + .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), + .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), + .unknown => s.fail("process terminated unexpectedly", .{}), + }; +} + +/// Prefer `cacheHitAndWatch` unless you already added watch inputs +/// separately from using the cache system. +pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool { + s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); + return s.result_cached; +} + +/// Clears previous watch inputs, if any, and then populates watch inputs from +/// the full set of files picked up by the cache manifest. +/// +/// Must be accompanied with `writeManifestAndWatch`. +pub fn cacheHitAndWatch(s: *Step, man: *Cache.Manifest) !bool { + const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); + s.result_cached = is_hit; + // The above call to hit() populates the manifest with files, so in case of + // a hit, we need to populate watch inputs. + if (is_hit) try setWatchInputsFromManifest(s, man); + return is_hit; +} + +fn failWithCacheError( + s: *Step, + man: *const Cache.Manifest, + err: Cache.Manifest.HitError, +) error{ OutOfMemory, Canceled, MakeFailed } { + switch (err) { + error.CacheCheckFailed => switch (man.diagnostic) { + .none => unreachable, + .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ + man.diagnostic, e, + }), + .file_open, .file_stat, .file_read, .file_hash => |op| { + const pp = man.files.keys()[op.file_index].prefixed_path; + const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; + return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ + prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, + }); + }, + }, + error.OutOfMemory, error.Canceled => |e| return e, + error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), + } +} + +/// Prefer `writeManifestAndWatch` unless you already added watch inputs +/// separately from using the cache system. +pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { + if (s.test_results.isSuccess()) { + man.writeManifest() catch |err| { + try s.addError("unable to write cache manifest: {t}", .{err}); + }; + } +} + +/// Clears previous watch inputs, if any, and then populates watch inputs from +/// the full set of files picked up by the cache manifest. +/// +/// Must be accompanied with `cacheHitAndWatch`. +pub fn writeManifestAndWatch(s: *Step, man: *Cache.Manifest) !void { + try writeManifest(s, man); + try setWatchInputsFromManifest(s, man); +} + +fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void { + const arena = s.owner.allocator; + const prefixes = man.cache.prefixes(); + clearWatchInputs(s); + for (man.files.keys()) |file| { + // The file path data is freed when the cache manifest is cleaned up at the end of `make`. + const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); + try addWatchInputFromPath(s, .{ + .root_dir = prefixes[file.prefixed_path.prefix], + .sub_path = std.fs.path.dirname(sub_path) orelse "", + }, std.fs.path.basename(sub_path)); + } +} + +/// For steps that have a single input that never changes when re-running `make`. +pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { + if (!step.inputs.populated()) try step.addWatchInput(lazy_path); +} + +pub fn clearWatchInputs(step: *Step) void { + const gpa = step.owner.allocator; + step.inputs.clear(gpa); +} + +/// Places a *file* dependency on the path. +pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!void { + switch (lazy_file) { + .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), + .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), + .cwd_relative => |path_string| { + try addWatchInputFromPath(step, .{ + .root_dir = .{ + .path = null, + .handle = Io.Dir.cwd(), + }, + .sub_path = std.fs.path.dirname(path_string) orelse "", + }, std.fs.path.basename(path_string)); + }, + // Nothing to watch because this dependency edge is modeled instead via `dependants`. + .generated => {}, + } +} + +/// Any changes inside the directory will trigger invalidation. +/// +/// See also `addDirectoryWatchInputFromPath` which takes a `Cache.Path` instead. +/// +/// Paths derived from this directory should also be manually added via +/// `addDirectoryWatchInputFromPath` if and only if this function returns +/// `true`. +pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool { + switch (lazy_directory) { + .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), + .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), + .cwd_relative => |path_string| { + try addDirectoryWatchInputFromPath(step, .{ + .root_dir = .{ + .path = null, + .handle = Io.Dir.cwd(), + }, + .sub_path = path_string, + }); + }, + // Nothing to watch because this dependency edge is modeled instead via `dependants`. + .generated => return false, + } + return true; +} + +/// Any changes inside the directory will trigger invalidation. +/// +/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. +/// +/// This function should only be called when it has been verified that the +/// dependency on `path` is not already accounted for by a `Step` dependency. +/// In other words, before calling this function, first check that the +/// `Build.LazyPath` which this `path` is derived from is not `generated`. +pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void { + return addWatchInputFromPath(step, path, "."); +} + +fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { + return addWatchInputFromPath(step, .{ + .root_dir = builder.build_root, + .sub_path = std.fs.path.dirname(sub_path) orelse "", + }, std.fs.path.basename(sub_path)); +} + +fn addDirectoryWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { + return addDirectoryWatchInputFromPath(step, .{ + .root_dir = builder.build_root, + .sub_path = sub_path, + }); +} + +fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !void { + const gpa = step.owner.allocator; + const gop = try step.inputs.table.getOrPut(gpa, path); + if (!gop.found_existing) gop.value_ptr.* = .empty; + try gop.value_ptr.append(gpa, basename); +} + +pub fn allocPrintCmd( + gpa: Allocator, + cwd: std.process.Child.Cwd, + opt_env: ?struct { + child: *const std.process.Environ.Map, + parent: *const std.process.Environ.Map, + }, + argv: []const []const u8, +) Allocator.Error![]u8 { + const shell = struct { + fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { + for (string) |c| { + if (switch (c) { + else => true, + '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, + '=' => is_argv0, + }) break; + } else return writer.writeAll(string); + + try writer.writeByte('"'); + for (string) |c| { + if (switch (c) { + std.ascii.control_code.nul => break, + '!', '"', '$', '\\', '`' => true, + else => !std.ascii.isPrint(c), + }) try writer.writeByte('\\'); + switch (c) { + std.ascii.control_code.nul => unreachable, + std.ascii.control_code.bel => try writer.writeByte('a'), + std.ascii.control_code.bs => try writer.writeByte('b'), + std.ascii.control_code.ht => try writer.writeByte('t'), + std.ascii.control_code.lf => try writer.writeByte('n'), + std.ascii.control_code.vt => try writer.writeByte('v'), + std.ascii.control_code.ff => try writer.writeByte('f'), + std.ascii.control_code.cr => try writer.writeByte('r'), + std.ascii.control_code.esc => try writer.writeByte('E'), + ' '...'~' => try writer.writeByte(c), + else => try writer.print("{o:0>3}", .{c}), + } + } + try writer.writeByte('"'); + } + }; + + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + const writer = &aw.writer; + switch (cwd) { + .inherit => {}, + .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, + .dir => @panic("TODO"), + } + if (opt_env) |env| { + var it = env.child.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + if (env.parent.get(key)) |process_value| { + if (std.mem.eql(u8, value, process_value)) continue; + } + writer.print("{s}=", .{key}) catch return error.OutOfMemory; + shell.escape(writer, value, false) catch return error.OutOfMemory; + writer.writeByte(' ') catch return error.OutOfMemory; + } + } + shell.escape(writer, argv[0], true) catch return error.OutOfMemory; + for (argv[1..]) |arg| { + writer.writeByte(' ') catch return error.OutOfMemory; + shell.escape(writer, arg, false) catch return error.OutOfMemory; + } + return aw.toOwnedSlice(); +} + diff --git a/lib/compiler/maker/Step/Compile.zig b/lib/compiler/maker/Step/Compile.zig new file mode 100644 index 0000000000000000000000000000000000000000..e3727025e25d189080e18cce93e48b13e0a226d1 --- /dev/null +++ b/lib/compiler/maker/Step/Compile.zig @@ -0,0 +1,1074 @@ +/// Populated during the make phase when there is a long-lived compiler process. +/// Managed by the build runner, not user build script. +zig_process: ?*Step.ZigProcess, + +fn make(step: *Step, options: Step.MakeOptions) !void { + const b = step.owner; + const compile: *Compile = @fieldParentPtr("step", step); + + const zig_args = try getZigArgs(compile, false); + + const maybe_output_dir = step.evalZigProcess( + zig_args, + options.progress_node, + (b.graph.incremental == true) and (options.watch or options.web_server != null), + options.web_server, + options.gpa, + ) catch |err| switch (err) { + error.NeedCompileErrorCheck => { + assert(compile.expect_errors != null); + try checkCompileErrors(compile); + return; + }, + else => |e| return e, + }; + + // Update generated files + if (maybe_output_dir) |output_dir| { + if (compile.emit_directory) |lp| { + lp.path = b.fmt("{f}", .{output_dir}); + } + + // zig fmt: off + if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); + if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); + // hack for stage2_x86_64 + coff + if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); + if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); + if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); + if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); + if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); + if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); + if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); + // zig fmt: on + } + + if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and + compile.version != null and compile.generated_bin != null and + std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) + { + try doAtomicSymLinks( + step, + compile.getEmittedBin().getPath2(b, step), + compile.major_only_filename.?, + compile.name_only_filename.?, + ); + } +} + +fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { + const step = &compile.step; + const b = step.owner; + const arena = b.allocator; + + var zig_args = std.array_list.Managed([]const u8).init(arena); + defer zig_args.deinit(); + + try zig_args.append(b.graph.zig_exe); + + const cmd = switch (compile.kind) { + .lib => "build-lib", + .exe => "build-exe", + .obj => "build-obj", + .@"test" => "test", + .test_obj => "test-obj", + }; + try zig_args.append(cmd); + + if (b.reference_trace) |some| { + try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); + } + try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); + + try addFlag(&zig_args, "llvm", compile.use_llvm); + try addFlag(&zig_args, "lld", compile.use_lld); + try addFlag(&zig_args, "new-linker", compile.use_new_linker); + + if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { + try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); + } + + switch (compile.entry) { + .default => {}, + .disabled => try zig_args.append("-fno-entry"), + .enabled => try zig_args.append("-fentry"), + .symbol_name => |entry_name| { + try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); + }, + } + + { + var symbol_it = compile.force_undefined_symbols.keyIterator(); + while (symbol_it.next()) |symbol_name| { + try zig_args.append("--force_undefined"); + try zig_args.append(symbol_name.*); + } + } + + if (compile.stack_size) |stack_size| { + try zig_args.append("--stack"); + try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); + } + + if (fuzz) { + try zig_args.append("-ffuzz"); + } + + { + // Stores system libraries that have already been seen for at least one + // module, along with any arguments that need to be passed to the + // compiler for each module individually. + var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; + var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; + + var prev_has_cflags = false; + var prev_has_rcflags = false; + var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; + var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; + // Track the number of positional arguments so that a nice error can be + // emitted if there is nothing to link. + var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); + + // Fully recursive iteration including dynamic libraries to detect + // libc and libc++ linkage. + for (compile.getCompileDependencies(true)) |some_compile| { + for (some_compile.root_module.getGraph().modules) |mod| { + if (mod.link_libc == true) compile.is_linking_libc = true; + if (mod.link_libcpp == true) compile.is_linking_libcpp = true; + } + } + + var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); + + // For this loop, don't chase dynamic libraries because their link + // objects are already linked. + for (compile.getCompileDependencies(false)) |dep_compile| { + for (dep_compile.root_module.getGraph().modules) |mod| { + // While walking transitive dependencies, if a given link object is + // already included in a library, it should not redundantly be + // placed on the linker line of the dependee. + const my_responsibility = dep_compile == compile; + const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); + + // Inherit dependencies on darwin frameworks. + if (!already_linked) { + for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { + try frameworks.put(arena, name, info); + } + } + + // Inherit dependencies on system libraries and static libraries. + for (mod.link_objects.items) |link_object| { + switch (link_object) { + .static_path => |static_path| { + if (my_responsibility) { + try zig_args.append(static_path.getPath2(mod.owner, step)); + total_linker_objects += 1; + } + }, + .system_lib => |system_lib| { + const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); + if (system_lib_gop.found_existing) { + try zig_args.appendSlice(system_lib_gop.value_ptr.*); + continue; + } else { + system_lib_gop.value_ptr.* = &.{}; + } + + if (already_linked) + continue; + + if ((system_lib.search_strategy != prev_search_strategy or + system_lib.preferred_link_mode != prev_preferred_link_mode) and + compile.linkage != .static) + { + switch (system_lib.search_strategy) { + .no_fallback => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_dylibs_only"), + .static => try zig_args.append("-search_static_only"), + }, + .paths_first => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_paths_first"), + .static => try zig_args.append("-search_paths_first_static"), + }, + .mode_first => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_dylibs_first"), + .static => try zig_args.append("-search_static_first"), + }, + } + prev_search_strategy = system_lib.search_strategy; + prev_preferred_link_mode = system_lib.preferred_link_mode; + } + + const prefix: []const u8 = prefix: { + if (system_lib.needed) break :prefix "-needed-l"; + if (system_lib.weak) break :prefix "-weak-l"; + break :prefix "-l"; + }; + switch (system_lib.use_pkg_config) { + .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), + .yes, .force => { + if (compile.runPkgConfig(system_lib.name)) |result| { + try zig_args.appendSlice(result.cflags); + try zig_args.appendSlice(result.libs); + try seen_system_libs.put(arena, system_lib.name, result.cflags); + } else |err| switch (err) { + error.PkgConfigInvalidOutput, + error.PkgConfigCrashed, + error.PkgConfigFailed, + error.PkgConfigNotInstalled, + error.PackageNotFound, + => switch (system_lib.use_pkg_config) { + .yes => { + // pkg-config failed, so fall back to linking the library + // by name directly. + try zig_args.append(b.fmt("{s}{s}", .{ + prefix, + system_lib.name, + })); + }, + .force => { + panic("pkg-config failed for library {s}", .{system_lib.name}); + }, + .no => unreachable, + }, + + else => |e| return e, + } + }, + } + }, + .other_step => |other| { + switch (other.kind) { + .exe => return step.fail("cannot link with an executable build artifact", .{}), + .@"test" => return step.fail("cannot link with a test", .{}), + .obj, .test_obj => { + const included_in_lib_or_obj = !my_responsibility and + (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); + if (!already_linked and !included_in_lib_or_obj) { + try zig_args.append(other.getEmittedBin().getPath2(b, step)); + total_linker_objects += 1; + } + }, + .lib => l: { + const other_produces_implib = other.producesImplib(); + const other_is_static = other_produces_implib or other.isStaticLibrary(); + + if (compile.isStaticLibrary() and other_is_static) { + // Avoid putting a static library inside a static library. + break :l; + } + + // For DLLs, we must link against the implib. + // For everything else, we directly link + // against the library file. + const full_path_lib = if (other_produces_implib) + try other.getGeneratedFilePath("generated_implib", &compile.step) + else + try other.getGeneratedFilePath("generated_bin", &compile.step); + + try zig_args.append(full_path_lib); + total_linker_objects += 1; + + if (other.linkage == .dynamic and + compile.rootModuleTarget().os.tag != .windows) + { + if (fs.path.dirname(full_path_lib)) |dirname| { + try zig_args.append("-rpath"); + try zig_args.append(dirname); + } + } + }, + } + }, + .assembly_file => |asm_file| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags) { + try zig_args.append("-cflags"); + try zig_args.append("--"); + prev_has_cflags = false; + } + try zig_args.append(asm_file.getPath2(mod.owner, step)); + total_linker_objects += 1; + }, + + .c_source_file => |c_source_file| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags or c_source_file.flags.len != 0) { + try zig_args.append("-cflags"); + for (c_source_file.flags) |arg| { + try zig_args.append(arg); + } + try zig_args.append("--"); + } + prev_has_cflags = (c_source_file.flags.len != 0); + + if (c_source_file.language) |lang| { + try zig_args.append("-x"); + try zig_args.append(lang.internalIdentifier()); + } + + try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); + + if (c_source_file.language != null) { + try zig_args.append("-x"); + try zig_args.append("none"); + } + total_linker_objects += 1; + }, + + .c_source_files => |c_source_files| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags or c_source_files.flags.len != 0) { + try zig_args.append("-cflags"); + for (c_source_files.flags) |arg| { + try zig_args.append(arg); + } + try zig_args.append("--"); + } + prev_has_cflags = (c_source_files.flags.len != 0); + + if (c_source_files.language) |lang| { + try zig_args.append("-x"); + try zig_args.append(lang.internalIdentifier()); + } + + const root_path = c_source_files.root.getPath2(mod.owner, step); + for (c_source_files.files) |file| { + try zig_args.append(b.pathJoin(&.{ root_path, file })); + } + + if (c_source_files.language != null) { + try zig_args.append("-x"); + try zig_args.append("none"); + } + + total_linker_objects += c_source_files.files.len; + }, + + .win32_resource_file => |rc_source_file| l: { + if (!my_responsibility) break :l; + + if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { + if (prev_has_rcflags) { + try zig_args.append("-rcflags"); + try zig_args.append("--"); + prev_has_rcflags = false; + } + } else { + try zig_args.append("-rcflags"); + for (rc_source_file.flags) |arg| { + try zig_args.append(arg); + } + for (rc_source_file.include_paths) |include_path| { + try zig_args.append("/I"); + try zig_args.append(include_path.getPath2(mod.owner, step)); + } + try zig_args.append("--"); + prev_has_rcflags = true; + } + try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); + total_linker_objects += 1; + }, + } + } + + // We need to emit the --mod argument here so that the above link objects + // have the correct parent module, but only if the module is part of + // this compilation. + if (!my_responsibility) continue; + if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { + const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; + try mod.appendZigProcessFlags(&zig_args, step); + + // --dep arguments + try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); + for (mod.import_table.keys(), mod.import_table.values()) |name, import| { + const import_index = cli_named_modules.modules.getIndex(import).?; + const import_cli_name = cli_named_modules.names.keys()[import_index]; + zig_args.appendAssumeCapacity("--dep"); + if (std.mem.eql(u8, import_cli_name, name)) { + zig_args.appendAssumeCapacity(import_cli_name); + } else { + zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); + } + } + + // When the CLI sees a -M argument, it determines whether it + // implies the existence of a Zig compilation unit based on + // whether there is a root source file. If there is no root + // source file, then this is not a zig compilation unit - it is + // perhaps a set of linker objects, or C source files instead. + // Linker objects are added to the CLI globally, while C source + // files must have a module parent. + if (mod.root_source_file) |lp| { + const src = lp.getPath2(mod.owner, step); + try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); + } else if (moduleNeedsCliArg(mod)) { + try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); + } + } + } + } + + if (total_linker_objects == 0) { + return step.fail("the linker needs one or more objects to link", .{}); + } + + for (frameworks.keys(), frameworks.values()) |name, info| { + if (info.needed) { + try zig_args.append("-needed_framework"); + } else if (info.weak) { + try zig_args.append("-weak_framework"); + } else { + try zig_args.append("-framework"); + } + try zig_args.append(name); + } + + if (compile.is_linking_libcpp) { + try zig_args.append("-lc++"); + } + + if (compile.is_linking_libc) { + try zig_args.append("-lc"); + } + } + + if (compile.win32_manifest) |manifest_file| { + try zig_args.append(manifest_file.getPath2(b, step)); + } + + if (compile.win32_module_definition) |module_file| { + try zig_args.append(module_file.getPath2(b, step)); + } + + if (compile.image_base) |image_base| { + try zig_args.append("--image-base"); + try zig_args.append(b.fmt("0x{x}", .{image_base})); + } + + for (compile.filters) |filter| { + try zig_args.append("--test-filter"); + try zig_args.append(filter); + } + + if (compile.test_runner) |test_runner| { + try zig_args.append("--test-runner"); + try zig_args.append(test_runner.path.getPath2(b, step)); + } + + for (b.debug_log_scopes) |log_scope| { + try zig_args.append("--debug-log"); + try zig_args.append(log_scope); + } + + if (b.debug_compile_errors) { + try zig_args.append("--debug-compile-errors"); + } + + if (b.debug_incremental) { + try zig_args.append("--debug-incremental"); + } + + if (b.verbose_air) try zig_args.append("--verbose-air"); + if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); + if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); + if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); + if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); + if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); + if (b.graph.time_report) try zig_args.append("--time-report"); + + if (compile.generated_asm != null) try zig_args.append("-femit-asm"); + if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); + if (compile.generated_docs != null) try zig_args.append("-femit-docs"); + if (compile.generated_implib != null) try zig_args.append("-femit-implib"); + if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); + if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); + if (compile.generated_h != null) try zig_args.append("-femit-h"); + + try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); + + switch (compile.compress_debug_sections) { + .none => {}, + .zlib => try zig_args.append("--compress-debug-sections=zlib"), + .zstd => try zig_args.append("--compress-debug-sections=zstd"), + } + + if (compile.link_eh_frame_hdr) { + try zig_args.append("--eh-frame-hdr"); + } + if (compile.link_emit_relocs) { + try zig_args.append("--emit-relocs"); + } + if (compile.link_function_sections) { + try zig_args.append("-ffunction-sections"); + } + if (compile.link_data_sections) { + try zig_args.append("-fdata-sections"); + } + if (compile.link_gc_sections) |x| { + try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); + } + if (!compile.linker_dynamicbase) { + try zig_args.append("--no-dynamicbase"); + } + if (compile.linker_allow_shlib_undefined) |x| { + try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); + } + if (compile.link_z_notext) { + try zig_args.append("-z"); + try zig_args.append("notext"); + } + if (!compile.link_z_relro) { + try zig_args.append("-z"); + try zig_args.append("norelro"); + } + if (compile.link_z_lazy) { + try zig_args.append("-z"); + try zig_args.append("lazy"); + } + if (compile.link_z_common_page_size) |size| { + try zig_args.append("-z"); + try zig_args.append(b.fmt("common-page-size={d}", .{size})); + } + if (compile.link_z_max_page_size) |size| { + try zig_args.append("-z"); + try zig_args.append(b.fmt("max-page-size={d}", .{size})); + } + if (compile.link_z_defs) { + try zig_args.append("-z"); + try zig_args.append("defs"); + } + + if (compile.libc_file) |libc_file| { + try zig_args.append("--libc"); + try zig_args.append(libc_file.getPath2(b, step)); + } else if (b.libc_file) |libc_file| { + try zig_args.append("--libc"); + try zig_args.append(libc_file); + } + + try zig_args.append("--cache-dir"); + try zig_args.append(b.cache_root.path orelse "."); + + try zig_args.append("--global-cache-dir"); + try zig_args.append(b.graph.global_cache_root.path orelse "."); + + if (b.graph.debug_compiler_runtime_libs) |mode| + try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); + + try zig_args.append("--name"); + try zig_args.append(compile.name); + + if (compile.linkage) |some| switch (some) { + .dynamic => try zig_args.append("-dynamic"), + .static => try zig_args.append("-static"), + }; + if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { + if (compile.version) |version| { + try zig_args.append("--version"); + try zig_args.append(b.fmt("{f}", .{version})); + } + + if (compile.rootModuleTarget().os.tag.isDarwin()) { + const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ + compile.rootModuleTarget().libPrefix(), + compile.name, + compile.rootModuleTarget().dynamicLibSuffix(), + }); + try zig_args.append("-install_name"); + try zig_args.append(install_name); + } + } + + if (compile.entitlements) |entitlements| { + try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); + } + if (compile.pagezero_size) |pagezero_size| { + const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); + try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); + } + if (compile.headerpad_size) |headerpad_size| { + const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); + try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); + } + if (compile.headerpad_max_install_names) { + try zig_args.append("-headerpad_max_install_names"); + } + if (compile.dead_strip_dylibs) { + try zig_args.append("-dead_strip_dylibs"); + } + if (compile.force_load_objc) { + try zig_args.append("-ObjC"); + } + if (compile.discard_local_symbols) { + try zig_args.append("--discard-all"); + } + + try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); + try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); + try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); + if (compile.rdynamic) { + try zig_args.append("-rdynamic"); + } + if (compile.import_memory) { + try zig_args.append("--import-memory"); + } + if (compile.export_memory) { + try zig_args.append("--export-memory"); + } + if (compile.import_symbols) { + try zig_args.append("--import-symbols"); + } + if (compile.import_table) { + try zig_args.append("--import-table"); + } + if (compile.export_table) { + try zig_args.append("--export-table"); + } + if (compile.initial_memory) |initial_memory| { + try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); + } + if (compile.max_memory) |max_memory| { + try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); + } + if (compile.shared_memory) { + try zig_args.append("--shared-memory"); + } + if (compile.global_base) |global_base| { + try zig_args.append(b.fmt("--global-base={d}", .{global_base})); + } + + if (compile.wasi_exec_model) |model| { + try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); + } + if (compile.linker_script) |linker_script| { + try zig_args.append("--script"); + try zig_args.append(linker_script.getPath2(b, step)); + } + + if (compile.version_script) |version_script| { + try zig_args.append("--version-script"); + try zig_args.append(version_script.getPath2(b, step)); + } + if (compile.linker_allow_undefined_version) |x| { + try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); + } + + if (compile.linker_enable_new_dtags) |enabled| { + try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); + } + + if (compile.kind == .@"test") { + if (compile.exec_cmd_args) |exec_cmd_args| { + for (exec_cmd_args) |cmd_arg| { + if (cmd_arg) |arg| { + try zig_args.append("--test-cmd"); + try zig_args.append(arg); + } else { + try zig_args.append("--test-cmd-bin"); + } + } + } + } + + if (b.sysroot) |sysroot| { + try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); + } + + // -I and -L arguments that appear after the last --mod argument apply to all modules. + const cwd: Io.Dir = .cwd(); + const io = b.graph.io; + + for (b.search_prefixes.items) |search_prefix| { + var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { + return step.fail("unable to open prefix directory '{s}': {s}", .{ + search_prefix, @errorName(err), + }); + }; + defer prefix_dir.close(io); + + // Avoid passing -L and -I flags for nonexistent directories. + // This prevents a warning, that should probably be upgraded to an error in Zig's + // CLI parsing code, when the linker sees an -L directory that does not exist. + + if (prefix_dir.access(io, "lib", .{})) |_| { + try zig_args.appendSlice(&.{ + "-L", b.pathJoin(&.{ search_prefix, "lib" }), + }); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ + search_prefix, @errorName(e), + }), + } + + if (prefix_dir.access(io, "include", .{})) |_| { + try zig_args.appendSlice(&.{ + "-I", b.pathJoin(&.{ search_prefix, "include" }), + }); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ + search_prefix, @errorName(e), + }), + } + } + + if (compile.rc_includes != .any) { + try zig_args.append("-rcincludes"); + try zig_args.append(@tagName(compile.rc_includes)); + } + + try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); + + if (compile.build_id orelse b.build_id) |build_id| { + try zig_args.append(switch (build_id) { + .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), + .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), + }); + } + + const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| + dir.getPath2(b, step) + else if (b.graph.zig_lib_directory.path) |_| + b.fmt("{f}", .{b.graph.zig_lib_directory}) + else + null; + + if (opt_zig_lib_dir) |zig_lib_dir| { + try zig_args.append("--zig-lib-dir"); + try zig_args.append(zig_lib_dir); + } + + try addFlag(&zig_args, "PIE", compile.pie); + + if (compile.lto) |lto| { + try zig_args.append(switch (lto) { + .full => "-flto=full", + .thin => "-flto=thin", + .none => "-fno-lto", + }); + } + + try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); + + if (compile.subsystem) |subsystem| { + try zig_args.append("--subsystem"); + try zig_args.append(@tagName(subsystem)); + } + + if (compile.mingw_unicode_entry_point) { + try zig_args.append("-municode"); + } + + if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ + "--error-limit", b.fmt("{d}", .{err_limit}), + }); + + try addFlag(&zig_args, "incremental", b.graph.incremental); + + try zig_args.append("--listen=-"); + + // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux + // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and + // pass that to zig, e.g. via 'zig build-lib @args.rsp' + // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html + var args_length: usize = 0; + for (zig_args.items) |arg| { + args_length += arg.len + 1; // +1 to account for null terminator + } + if (args_length >= 30 * 1024) { + try b.cache_root.handle.createDirPath(io, "args"); + + const args_to_escape = zig_args.items[2..]; + var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); + arg_blk: for (args_to_escape) |arg| { + for (arg, 0..) |c, arg_idx| { + if (c == '\\' or c == '"') { + // Slow path for arguments that need to be escaped. We'll need to allocate and copy + var escaped: std.ArrayList(u8) = .empty; + try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1); + try escaped.appendSlice(arena, arg[0..arg_idx]); + for (arg[arg_idx..]) |to_escape| { + if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\'); + try escaped.append(arena, to_escape); + } + escaped_args.appendAssumeCapacity(escaped.items); + continue :arg_blk; + } + } + escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument + } + + // Write the args to zig-cache/args/ to avoid conflicts with + // other zig build commands running in parallel. + const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); + const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); + + var args_hash: [Sha256.digest_length]u8 = undefined; + Sha256.hash(args, &args_hash, .{}); + var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; + _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); + + const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; + if (b.cache_root.handle.access(io, args_file, .{})) |_| { + // The args file is already present from a previous run. + } else |err| switch (err) { + error.FileNotFound => { + var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ + .replace = false, + .make_path = true, + }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ + b.cache_root, args_file, e, + }); + defer af.deinit(io); + + af.file.writeStreamingAll(io, args) catch |e| { + return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ + b.cache_root, args_file, e, + }); + }; + // Note we can't clean up this file, not even after build + // success, because that might interfere with another build + // process that needs the same file. + af.link(io) catch |e| switch (e) { + error.PathAlreadyExists => { + // The args file was created by another concurrent build process. + }, + else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ + b.cache_root, args_file, other_err, + }), + }; + }, + else => |other_err| return other_err, + } + + const resolved_args_file = try mem.concat(arena, u8, &.{ + "@", + try b.cache_root.join(arena, &.{args_file}), + }); + + zig_args.shrinkRetainingCapacity(2); + try zig_args.append(resolved_args_file); + } + + return try zig_args.toOwnedSlice(); +} + +pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { + c.step.result_error_msgs.clearRetainingCapacity(); + c.step.result_stderr = ""; + + c.step.result_error_bundle.deinit(gpa); + c.step.result_error_bundle = std.zig.ErrorBundle.empty; + + if (c.step.result_failed_command) |cmd| { + gpa.free(cmd); + c.step.result_failed_command = null; + } + + const zig_args = try getZigArgs(c, true); + const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); + return maybe_output_bin_path.?; +} + +pub fn doAtomicSymLinks( + step: *Step, + output_path: []const u8, + filename_major_only: []const u8, + filename_name_only: []const u8, +) !void { + const b = step.owner; + const io = b.graph.io; + const out_dir = fs.path.dirname(output_path) orelse "."; + const out_basename = fs.path.basename(output_path); + // sym link for libfoo.so.1 to libfoo.so.1.2.3 + const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); + const cwd: Io.Dir = .cwd(); + cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { + return step.fail("unable to symlink {s} -> {s}: {s}", .{ + major_only_path, out_basename, @errorName(err), + }); + }; + // sym link for libfoo.so to libfoo.so.1 + const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); + cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { + return step.fail("Unable to symlink {s} -> {s}: {s}", .{ + name_only_path, filename_major_only, @errorName(err), + }); + }; +} + +fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { + const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); + var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); + errdefer list.deinit(); + var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); + while (line_it.next()) |line| { + if (mem.trim(u8, line, " \t").len == 0) continue; + var tok_it = mem.tokenizeAny(u8, line, " \t"); + try list.append(PkgConfigPkg{ + .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, + .desc = tok_it.rest(), + }); + } + return list.toOwnedSlice(); +} + +fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { + if (b.pkg_config_pkg_list) |res| { + return res; + } + var code: u8 = undefined; + if (execPkgConfigList(b, &code)) |list| { + b.pkg_config_pkg_list = list; + return list; + } else |err| { + const result = switch (err) { + error.ProcessTerminated => error.PkgConfigCrashed, + error.ExecNotSupported => error.PkgConfigFailed, + error.ExitCodeFailure => error.PkgConfigFailed, + error.FileNotFound => error.PkgConfigNotInstalled, + error.InvalidName => error.PkgConfigNotInstalled, + error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, + else => return err, + }; + b.pkg_config_pkg_list = result; + return result; + } +} + +fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { + const cond = opt orelse return; + try args.ensureUnusedCapacity(1); + if (cond) { + args.appendAssumeCapacity("-f" ++ name); + } else { + args.appendAssumeCapacity("-fno-" ++ name); + } +} + +const PkgConfigResult = struct { + cflags: []const []const u8, + libs: []const []const u8, +}; + +/// Run pkg-config for the given library name and parse the output, returning the arguments +/// that should be passed to zig to link the given library. +fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult { + const wl_rpath_prefix = "-Wl,-rpath,"; + + const b = compile.step.owner; + const arena = b.allocator; + const pkg_name = match: { + // First we have to map the library name to pkg config name. Unfortunately, + // there are several examples where this is not straightforward: + // -lSDL2 -> pkg-config sdl2 + // -lgdk-3 -> pkg-config gdk-3.0 + // -latk-1.0 -> pkg-config atk + // -lpulse -> pkg-config libpulse + const pkgs = try getPkgConfigList(b); + + // Exact match means instant winner. + for (pkgs) |pkg| { + if (mem.eql(u8, pkg.name, lib_name)) { + break :match pkg.name; + } + } + + // Next we'll try ignoring case. + for (pkgs) |pkg| { + if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { + break :match pkg.name; + } + } + + // Prefixed "lib" or suffixed ".0". + for (pkgs) |pkg| { + if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { + const prefix = pkg.name[0..pos]; + const suffix = pkg.name[pos + lib_name.len ..]; + if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; + if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; + break :match pkg.name; + } + } + + // Trimming "-1.0". + if (mem.endsWith(u8, lib_name, "-1.0")) { + const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; + for (pkgs) |pkg| { + if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { + break :match pkg.name; + } + } + } + + return error.PackageNotFound; + }; + + var code: u8 = undefined; + const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = if (b.runAllowFail(&[_][]const u8{ + pkg_config_exe, + pkg_name, + "--cflags", + "--libs", + }, &code, .ignore)) |stdout| stdout else |err| switch (err) { + error.ProcessTerminated => return error.PkgConfigCrashed, + error.ExecNotSupported => return error.PkgConfigFailed, + error.ExitCodeFailure => return error.PkgConfigFailed, + error.FileNotFound => return error.PkgConfigNotInstalled, + else => return err, + }; + + var zig_cflags: std.ArrayList([]const u8) = .empty; + defer zig_cflags.deinit(arena); + var zig_libs: std.ArrayList([]const u8) = .empty; + defer zig_libs.deinit(arena); + + var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); + while (arg_it.next()) |arg| { + if (mem.eql(u8, arg, "-I")) { + const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_cflags.appendSlice(arena, &.{ "-I", dir }); + } else if (mem.startsWith(u8, arg, "-I")) { + try zig_cflags.append(arena, arg); + } else if (mem.eql(u8, arg, "-L")) { + const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_libs.appendSlice(arena, &.{ "-L", dir }); + } else if (mem.startsWith(u8, arg, "-L")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-l")) { + const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_libs.appendSlice(arena, &.{ "-l", lib }); + } else if (mem.startsWith(u8, arg, "-l")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-D")) { + const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_cflags.appendSlice(arena, &.{ "-D", macro }); + } else if (mem.startsWith(u8, arg, "-D")) { + try zig_cflags.append(arena, arg); + } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { + try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); + } else if (b.debug_pkg_config) { + return compile.step.fail("unknown pkg-config flag '{s}'", .{arg}); + } + } + + try zig_cflags.shrinkToLen(arena); + try zig_libs.shrinkToLen(arena); + + return .{ + .cflags = zig_cflags.toOwnedSliceAssert(), + .libs = zig_libs.toOwnedSliceAssert(), + }; +} + + diff --git a/lib/compiler/maker/Step/InstallArtifact.zig b/lib/compiler/maker/Step/InstallArtifact.zig new file mode 100644 index 0000000000000000000000000000000000000000..ba3846c1a574f9934f4196f03529dac3abcbe275 --- /dev/null +++ b/lib/compiler/maker/Step/InstallArtifact.zig @@ -0,0 +1,96 @@ + +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; + const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); + const b = step.owner; + const io = b.graph.io; + + var all_cached = true; + + if (install_artifact.dest_dir) |dest_dir| { + const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); + const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); + all_cached = all_cached and p == .fresh; + + if (install_artifact.dylib_symlinks) |dls| { + try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); + } + + install_artifact.artifact.installed_path = full_dest_path; + } + + if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { + const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.implib_dir) |implib_dir| { + const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.pdb_dir) |pdb_dir| { + const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.h_dir) |h_dir| { + if (install_artifact.emitted_h) |emitted_h| { + const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); + const p = try step.installFile(emitted_h, full_h_path); + all_cached = all_cached and p == .fresh; + } + + for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { + .file => |file| { + const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); + const p = try step.installFile(file.source, full_h_path); + all_cached = all_cached and p == .fresh; + }, + .directory => |dir| { + const src_dir_path = dir.source.getPath3(b, step); + const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); + + var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{f}': {s}", .{ + src_dir_path, @errorName(err), + }); + }; + defer src_dir.close(io); + + var it = try src_dir.walk(b.allocator); + next_entry: while (try it.next(io)) |entry| { + for (dir.options.exclude_extensions) |ext| { + if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; + } + if (dir.options.include_extensions) |incs| { + for (incs) |inc| { + if (std.mem.endsWith(u8, entry.path, inc)) break; + } else { + continue :next_entry; + } + } + + const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); + switch (entry.kind) { + .directory => { + try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); + const p = try step.installDir(full_dest_path); + all_cached = all_cached and p == .existed; + }, + .file => { + const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); + all_cached = all_cached and p == .fresh; + }, + else => continue, + } + } + }, + }; + } + + step.result_cached = all_cached; +} diff --git a/lib/compiler/maker/Step/Run.zig b/lib/compiler/maker/Step/Run.zig new file mode 100644 index 0000000000000000000000000000000000000000..79b4d57b7e6fa772191b434bc90d7432b3442af3 --- /dev/null +++ b/lib/compiler/maker/Step/Run.zig @@ -0,0 +1,2127 @@ +const Run = @This(); + +const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; +const Dir = std.Io.Dir; +const mem = std.mem; +const process = std.process; +const EnvMap = std.process.Environ.Map; +const assert = std.debug.assert; +const Cache = std.Build.Cache; +const Path = std.Build.Cache.Path; + +const Step = @import("../Step.zig"); + +/// If this is a Zig unit test binary, this tracks the names of the unit +/// tests that are also fuzz tests. Indexes cannot be used as they may +/// change between reruns. +fuzz_tests: std.ArrayList([]const u8), +cached_test_metadata: ?CachedTestMetadata = null, + + +fn make(step: *Step, options: Step.MakeOptions) !void { + const b = step.owner; + const io = b.graph.io; + const arena = b.allocator; + const run: *Run = @fieldParentPtr("step", step); + const has_side_effects = run.hasSideEffects(); + + var argv_list = std.array_list.Managed([]const u8).init(arena); + var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); + + var man = b.graph.cache.obtain(); + defer man.deinit(); + + if (run.environ_map) |environ_map| { + for (environ_map.keys(), environ_map.values()) |key, value| { + man.hash.addBytes(key); + man.hash.addBytes(value); + } + } + + man.hash.add(run.color); + man.hash.add(run.disable_zig_progress); + + for (run.argv.items) |arg| { + switch (arg) { + .bytes => |bytes| { + try argv_list.append(bytes); + man.hash.addBytes(bytes); + }, + .lazy_path => |file| { + const file_path = file.lazy_path.getPath3(b, step); + try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); + man.hash.addBytes(file.prefix); + _ = try man.addFilePath(file_path, null); + }, + .decorated_directory => |dd| { + const file_path = dd.lazy_path.getPath3(b, step); + const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); + try argv_list.append(resolved_arg); + man.hash.addBytes(resolved_arg); + }, + .file_content => |file_plp| { + const file_path = file_plp.lazy_path.getPath3(b, step); + + var result: std.Io.Writer.Allocating = .init(arena); + errdefer result.deinit(); + result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + + const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { + return step.fail( + "unable to open input file '{f}': {t}", + .{ file_path, err }, + ); + }; + defer file.close(io); + + var buf: [1024]u8 = undefined; + var file_reader = file.reader(io, &buf); + _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { + error.ReadFailed => return step.fail( + "failed to read from '{f}': {t}", + .{ file_path, file_reader.err.? }, + ), + error.WriteFailed => return error.OutOfMemory, + }; + + try argv_list.append(result.written()); + man.hash.addBytes(file_plp.prefix); + _ = try man.addFilePath(file_path, null); + }, + .artifact => |pa| { + const artifact = pa.artifact; + + if (artifact.rootModuleTarget().os.tag == .windows) { + // On Windows we don't have rpaths so we have to add .dll search paths to PATH + run.addPathForDynLibs(artifact); + } + const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; + + try argv_list.append(b.fmt("{s}{s}", .{ + pa.prefix, + run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), + })); + + _ = try man.addFile(file_path, null); + }, + .output_file, .output_directory => |output| { + man.hash.addBytes(output.prefix); + man.hash.addBytes(output.basename); + // Add a placeholder into the argument list because we need the + // manifest hash to be updated with all arguments before the + // object directory is computed. + try output_placeholders.append(.{ + .index = argv_list.items.len, + .tag = arg, + .output = output, + }); + _ = try argv_list.addOne(); + }, + } + } + + switch (run.stdin) { + .bytes => |bytes| { + man.hash.addBytes(bytes); + }, + .lazy_path => |lazy_path| { + const file_path = lazy_path.getPath2(b, step); + _ = try man.addFile(file_path, null); + }, + .none => {}, + } + + if (run.captured_stdout) |captured| { + man.hash.addBytes(captured.output.basename); + man.hash.add(captured.trim_whitespace); + } + + if (run.captured_stderr) |captured| { + man.hash.addBytes(captured.output.basename); + man.hash.add(captured.trim_whitespace); + } + + hashStdIo(&man.hash, run.stdio); + + for (run.file_inputs.items) |lazy_path| { + _ = try man.addFile(lazy_path.getPath2(b, step), null); + } + + if (run.cwd) |cwd| { + const cwd_path = cwd.getPath3(b, step); + _ = man.hash.addBytes(try cwd_path.toString(arena)); + } + + if (!has_side_effects and try step.cacheHitAndWatch(&man)) { + // cache hit, skip running command + const digest = man.final(); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); + + step.result_cached = true; + return; + } + + const dep_output_file = run.dep_output_file orelse { + // We already know the final output paths, use them directly. + const digest = if (has_side_effects) + man.hash.final() + else + man.final(); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); + + const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; + for (output_placeholders.items) |placeholder| { + const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); + const output_sub_dir_path = switch (placeholder.tag) { + .output_file => Dir.path.dirname(output_sub_path).?, + .output_directory => output_sub_path, + else => unreachable, + }; + b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, output_sub_dir_path, @errorName(err), + }); + }; + const arg_output_path = run.convertPathArg(.{ + .root_dir = .cwd(), + .sub_path = placeholder.output.generated_file.getPath(), + }); + argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) + arg_output_path + else + b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); + } + + try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); + if (!has_side_effects) try step.writeManifestAndWatch(&man); + return; + }; + + // We do not know the final output paths yet, use temp paths to run the command. + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + + for (output_placeholders.items) |placeholder| { + const output_components = .{ tmp_dir_path, placeholder.output.basename }; + const output_sub_path = b.pathJoin(&output_components); + const output_sub_dir_path = switch (placeholder.tag) { + .output_file => Dir.path.dirname(output_sub_path).?, + .output_directory => output_sub_path, + else => unreachable, + }; + b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, output_sub_dir_path, @errorName(err), + }); + }; + const raw_output_path: Cache.Path = .{ + .root_dir = b.cache_root, + .sub_path = b.pathJoin(&output_components), + }; + placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); + argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ + placeholder.output.prefix, + run.convertPathArg(raw_output_path), + }); + } + + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); + + const dep_file_dir = Dir.cwd(); + const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); + if (has_side_effects) + try man.addDepFile(dep_file_dir, dep_file_basename) + else + try man.addDepFilePost(dep_file_dir, dep_file_basename); + + const digest = if (has_side_effects) + man.hash.final() + else + man.final(); + + const any_output = output_placeholders.items.len > 0 or + run.captured_stdout != null or run.captured_stderr != null; + + // Rename into place + if (any_output) { + const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; + + b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { + Dir.RenameError.DirNotEmpty => { + b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { + return step.fail("unable to remove dir '{f}'{s}: {t}", .{ + b.cache_root, tmp_dir_path, del_err, + }); + }; + b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { + return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ + b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, + }); + }; + }, + else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ + b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, + }), + }; + } + + if (!has_side_effects) try step.writeManifestAndWatch(&man); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); +} + +/// Reads stdout of a Zig test process until a termination condition is reached: +/// * A write fails, indicating the child unexpectedly closed stdin +/// * A test (or a response from the test runner) times out +/// * The wait fails, indicating the child closed stdout and stderr +fn waitZigTest( + run: *Run, + child: *process.Child, + options: Step.MakeOptions, + multi_reader: *Io.File.MultiReader, + opt_metadata: *?TestMetadata, + results: *Step.TestResults, +) !union(enum) { + write_failed: anyerror, + no_poll: struct { + active_test_index: ?u32, + ns_elapsed: u64, + }, + timeout: struct { + active_test_index: ?u32, + ns_elapsed: u64, + }, +} { + const gpa = run.step.owner.allocator; + const arena = run.step.owner.allocator; + const io = run.step.owner.graph.io; + + var sub_prog_node: ?std.Progress.Node = null; + defer if (sub_prog_node) |n| n.end(); + + if (opt_metadata.*) |*md| { + // Previous unit test process died or was killed; we're continuing where it left off + requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; + } else { + // Running unit tests normally + run.fuzz_tests.clearRetainingCapacity(); + sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err }; + } + + var active_test_index: ?u32 = null; + + var last_update: Io.Clock.Timestamp = .now(io, .awake); + + // This timeout is used when we're waiting on the test runner itself rather than a user-specified + // test. For instance, if the test runner leaves this much time between us requesting a test to + // start and it acknowledging the test starting, we terminate the child and raise an error. This + // *should* never happen, but could in theory be caused by some very unlucky IB in a test. + const response_timeout: Io.Clock.Duration = t: { + if (fuzz_context != null) break :t null; // don't timeout fuzz tests + const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); + break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; + }; + const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ + .clock = .awake, + .raw = .fromNanoseconds(ns), + } else null; + + const stdout = multi_reader.reader(0); + const stderr = multi_reader.reader(1); + const Header = std.zig.Server.Message.Header; + + while (true) { + const timeout: Io.Timeout = t: { + const opt_duration = if (active_test_index == null) response_timeout else test_timeout; + const duration = opt_duration orelse break :t .none; + break :t .{ .deadline = last_update.addDuration(duration) }; + }; + + // This block is exited when `stdout` contains enough bytes for a `Header`. + header_ready: { + if (stdout.buffered().len >= @sizeOf(Header)) { + // We already have one, no need to poll! + break :header_ready; + } + + multi_reader.fill(64, timeout) catch |err| switch (err) { + error.Timeout => return .{ .timeout = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + error.EndOfStream => return .{ .no_poll = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + else => |e| return e, + }; + + continue; + } + // There is definitely a header available now -- read it. + const header = stdout.takeStruct(Header, .little) catch unreachable; + + while (stdout.buffered().len < header.bytes_len) { + multi_reader.fill(64, timeout) catch |err| switch (err) { + error.Timeout => return .{ .timeout = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + error.EndOfStream => return .{ .no_poll = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + else => |e| return e, + }; + } + + const body = stdout.take(header.bytes_len) catch unreachable; + var body_r: std.Io.Reader = .fixed(body); + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + }, + .test_metadata => { + // `metadata` would only be populated if we'd already seen a `test_metadata`, but we + // only request it once (and importantly, we don't re-request it if we kill and + // restart the test runner). + assert(opt_metadata.* == null); + + const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable; + results.test_count = tm_hdr.tests_len; + + const names = try arena.alloc(u32, results.test_count); + for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; + + const expected_panic_msgs = try arena.alloc(u32, results.test_count); + for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; + + const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; + + options.progress_node.setEstimatedTotalItems(names.len); + opt_metadata.* = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .ns_per_test = try arena.alloc(u64, results.test_count), + .names = names, + .expected_panic_msgs = expected_panic_msgs, + .next_index = 0, + .prog_node = options.progress_node, + }; + @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); + + active_test_index = null; + last_update = .now(io, .awake); + + requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err }; + }, + .test_started => { + active_test_index = opt_metadata.*.?.next_index - 1; + last_update = .now(io, .awake); + }, + .test_results => { + const md = &opt_metadata.*.?; + + const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable; + assert(tr_hdr.index == active_test_index); + + switch (tr_hdr.flags.status) { + .pass => {}, + .skip => results.skip_count +|= 1, + .fail => results.fail_count +|= 1, + } + const leak_count = tr_hdr.flags.leak_count; + const log_err_count = tr_hdr.flags.log_err_count; + results.leak_count +|= leak_count; + results.log_err_count +|= log_err_count; + + if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index)); + + if (tr_hdr.flags.status == .fail) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + if (stderr_bytes.len == 0) { + try run.step.addError("'{s}' failed without output", .{name}); + } else { + try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); + } + } else if (leak_count > 0) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); + } else if (log_err_count > 0) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); + } + + active_test_index = null; + + const now: Io.Clock.Timestamp = .now(io, .awake); + md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds); + last_update = now; + + requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; + }, + else => {}, // ignore other messages + } + } +} + +const FuzzTestRunner = struct { + run: *Run, + ctx: FuzzContext, + coverage_id: ?u64, + + instances: []Instance, + /// The indexes of this are layed out such that it is effectively an array + /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr. + batch: Io.Batch, + /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter. + pending_broadcasts: std.ArrayList(u8), + broadcast: std.ArrayList(u8), + broadcast_undelivered: u32, + + const Instance = struct { + child: process.Child, + message: std.ArrayListAligned(u8, .@"4"), + broadcast_written: usize, + stderr: std.ArrayList(u8), + stdin_vec: [1][]u8, + stdout_vec: [1][]u8, + stderr_vec: [1][]u8, + progress_node: std.Progress.Node, + + fn messageHeader(instance: *Instance) InHeader { + assert(instance.message.items.len >= @sizeOf(InHeader)); + const header_ptr: *InHeader = @ptrCast(instance.message.items); + var header = header_ptr.*; + if (std.builtin.Endian.native != .little) { + std.mem.byteSwapAllFields(InHeader, &header); + } + return header; + } + }; + + const PendingBroadcastFooter = struct { + from_id: u32, + body_len: u32, + }; + + const InHeader = std.zig.Server.Message.Header; + const OutHeader = std.zig.Client.Message.Header; + + const stdin_i = 0; + const stdout_i = 1; + const stderr_i = 2; + + fn init( + run: *Run, + ctx: FuzzContext, + progress_node: std.Progress.Node, + spawn_options: process.SpawnOptions, + ) !FuzzTestRunner { + const step_owner = run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + + const n_instances = switch (ctx.fuzz.mode) { + .forever => step_owner.graph.max_jobs orelse @min( + std.Thread.getCpuCount() catch 1, + (std.math.maxInt(u32) - 2) / 3, + ), + .limit => 1, + }; + const instances = try gpa.alloc(Instance, n_instances); + errdefer gpa.free(instances); + const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3); + errdefer gpa.free(batch_storage); + + @memset(instances, .{ + .child = undefined, + .message = .empty, + .broadcast_written = undefined, + .stderr = .empty, + .stdin_vec = undefined, + .stdout_vec = undefined, + .stderr_vec = undefined, + .progress_node = undefined, + }); + for (0.., instances) |id, *instance| { + errdefer for (instances[0..id]) |*spawned| { + spawned.child.kill(io); + spawned.progress_node.end(); + }; + instance.child = try process.spawn(io, spawn_options); + instance.progress_node = progress_node.start("starting fuzzer", 0); + } + + return .{ + .run = run, + .ctx = ctx, + .coverage_id = null, + + .instances = instances, + .batch = .init(batch_storage), + .pending_broadcasts = .empty, + .broadcast = .empty, + .broadcast_undelivered = 0, + }; + } + + fn deinit(f: *FuzzTestRunner) void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + + f.batch.cancel(io); + gpa.free(f.batch.storage); + var total_rss: usize = 0; + for (f.instances) |*instance| { + instance.child.kill(io); + instance.message.deinit(gpa); + instance.stderr.deinit(gpa); + instance.progress_node.end(); + total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; + } + f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); + gpa.free(f.instances); + } + + fn startInstances(f: *FuzzTestRunner) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + + for (0.., f.instances) |id, *instance| { + const id32: u32 = @intCast(id); + (switch (f.ctx.fuzz.mode) { + .forever => sendRunFuzzTestMessage( + io, + instance.child.stdin.?, + f.run.fuzz_tests.items, + .forever, + id32, + ), + .limit => |limit| sendRunFuzzTestMessage( + io, + instance.child.stdin.?, + f.run.fuzz_tests.items, + .iterations, + limit.amount, + ), + }) catch |write_err| { + // The runner unexpectedly closed stdin, which means it crashed during initialization. + // Clean up everything and wait for the child to exit. + instance.child.stdin.?.close(io); + instance.child.stdin = null; + const term = try instance.child.wait(io); + return f.run.step.fail( + "unable to write stdin ({t}); test process unexpectedly {f}", + .{ write_err, fmtTerm(term) }, + ); + }; + + try f.addStdoutRead(id32, @sizeOf(InHeader)); + try f.addStderrRead(id32); + } + } + + fn listen(f: *FuzzTestRunner) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + + while (true) { + try f.batch.awaitConcurrent(io, .none); + while (f.batch.next()) |completion| { + const id = completion.index / 3; + const result = completion.result; + switch (completion.index % 3) { + 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) { + // Avoid calling `instanceEos` until EndOfStream is seen with stderr so + // that all stderr is collected. + error.BrokenPipe => continue, + else => |write_e| return write_e, + }), + 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) { + // Avoid calling `instanceEos` until EndOfStream is seen with stderr so + // that all stderr is collected. + error.EndOfStream => continue, + else => |read_e| return read_e, + }), + 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { + error.EndOfStream => return f.instanceEos(id), + else => |read_e| return read_e, + }), + else => unreachable, + } + } + } + } + + fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + const instance = &f.instances[id]; + + instance.message.items.len += n; + const total_read = instance.message.items.len; + if (total_read < @sizeOf(InHeader)) { + try f.addStdoutRead(id, @sizeOf(InHeader)); + return; + } + + const header = instance.messageHeader(); + const body = instance.message.items[@sizeOf(InHeader)..]; + if (body.len != header.bytes_len) { + try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len); + return; + } + + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + }, + .coverage_id => { + var body_r: Io.Reader = .fixed(body); + f.coverage_id = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable; + + const fuzz = f.ctx.fuzz; + fuzz.queue_mutex.lockUncancelable(io); + defer fuzz.queue_mutex.unlock(io); + try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ + .id = f.coverage_id.?, + .cumulative = .{ + .runs = cumulative_runs, + .unique = cumulative_unique, + .coverage = cumulative_coverage, + }, + .run = f.run, + } }); + fuzz.queue_cond.signal(io); + }, + .fuzz_start_addr => { + var body_r: Io.Reader = .fixed(body); + const fuzz = f.ctx.fuzz; + const addr = body_r.takeInt(u64, .little) catch unreachable; + + fuzz.queue_mutex.lockUncancelable(io); + defer fuzz.queue_mutex.unlock(io); + try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ + .addr = addr, + .coverage_id = f.coverage_id.?, + } }); + fuzz.queue_cond.signal(io); + }, + .fuzz_test_change => { + const test_i = std.mem.readInt(u32, body[0..4], .little); + instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); + }, + .broadcast_fuzz_input => { + if (f.instances.len == 1) { + // No other processes to broadcast to. + } else if (f.broadcast_undelivered == 0) { + try f.instanceBroadcast(id, body); + } else { + const footer: PendingBroadcastFooter = .{ + .from_id = id, + .body_len = @intCast(body.len), + }; + // There is another broadcast in progress so add this one to the queue. + const size = @sizeOf(PendingBroadcastFooter) + body.len; + try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); + f.pending_broadcasts.appendSliceAssumeCapacity(body); + f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); + } + }, + else => {}, // ignore other messages + } + + instance.message.clearRetainingCapacity(); + try f.addStdoutRead(id, @sizeOf(InHeader)); + } + + fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void { + const instance = &f.instances[id]; + instance.stderr.items.len += n; + try f.addStderrRead(id); + } + + fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void { + const instance = &f.instances[id]; + + instance.broadcast_written += n; + if (instance.broadcast_written == f.broadcast.items.len) { + f.broadcast_undelivered -= 1; + if (f.broadcast_undelivered == 0) { + try f.broadcastComplete(); + } + } else { + f.addStdinWrite(id); + } + } + + fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const instance = &f.instances[id]; + + try instance.message.ensureTotalCapacity(gpa, end); + const start = instance.message.items.len; + instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]}; + f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{ + .file = instance.child.stdout.?, + .data = &instance.stdout_vec, + } }); + } + + fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const instance = &f.instances[id]; + + try instance.stderr.ensureUnusedCapacity(gpa, 1); + instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()}; + f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{ + .file = instance.child.stderr.?, + .data = &instance.stderr_vec, + } }); + } + + fn addStdinWrite(f: *FuzzTestRunner, id: u32) void { + const instance = &f.instances[id]; + + assert(f.broadcast.items.len != instance.broadcast_written); + instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]}; + f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{ + .file = instance.child.stdin.?, + .data = &instance.stdin_vec, + } }); + } + + fn instanceEos(f: *FuzzTestRunner, id: u32) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + const instance = &f.instances[id]; + + instance.child.stdin.?.close(io); + instance.child.stdin = null; + const term = try instance.child.wait(io); + if (!termMatches(.{ .exited = 0 }, term)) { + f.run.step.result_stderr = try f.mergedStderr(); + try f.saveCrash(id, term); + return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); + } + } + + fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { + const step = &f.run.step; + const b = step.owner; + const io = b.graph.io; + + if (f.coverage_id == null) return; + + // Search for the input file corresponding to the instance + const InputHeader = Build.abi.fuzz.MmapInputHeader; + var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; + var in_r: Io.File.Reader = undefined; + var in_f: Io.File = undefined; + var in_name_buf: [12]u8 = undefined; + var in_name: []const u8 = undefined; + var i: u32 = 0; + const header: InputHeader = while (true) : ({ + if (i == std.math.maxInt(u32)) return; + i += 1; + }) { + const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; + in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; + in_f = b.cache_root.handle.openFile(io, in_name, .{ + .lock = .exclusive, + .lock_nonblocking = true, + }) catch |e| switch (e) { + error.FileNotFound => return, + error.WouldBlock => continue, // Can not be from + // the crashed instance since it is still locked. + else => return step.fail("failed to open file '{f}{s}': {t}", .{ + b.cache_root, in_name, e, + }), + }; + + in_r = in_f.readerStreaming(io, &in_r_buf); + const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { + in_f.close(io); + switch (e) { + error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ + b.cache_root, in_name, in_r.err.?, + }), + error.EndOfStream => continue, + } + }; + + if (header.pc_digest == f.coverage_id.? and + header.instance_id == id and + header.test_i < f.run.fuzz_tests.items.len) + { + break header; + } + + in_f.close(io); + }; + defer in_f.close(io); + + // Save it to a seperate file + const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; + const out = b.cache_root.handle.createFile(io, crash_name, .{ + .lock = .exclusive, // Multiple run steps could have found a crash at the same time + }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ + b.cache_root, crash_name, e, + }); + defer out.close(io); + + var out_w_buf: [512]u8 = undefined; + var out_w = out.writerStreaming(io, &out_w_buf); + _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { + error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ + b.cache_root, in_name, in_r.err.?, + }), + error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ + b.cache_root, crash_name, out_w.err.?, + }), + }; + + return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ + f.run.fuzz_tests.items[header.test_i], + fmtTerm(term), + b.cache_root, + crash_name, + }); + } + + fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void { + assert(f.instances.len > 1); + assert(f.broadcast_undelivered == 0); // no other broadcast is progress + assert(f.broadcast.items.len == 0); + assert(from_id < f.instances.len); + + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + + var out_header: OutHeader = .{ + .tag = .new_fuzz_input, + .bytes_len = @intCast(bytes.len), + }; + if (std.builtin.Endian.native != .little) { + std.mem.byteSwapAllFields(OutHeader, &out_header); + } + try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len); + f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header)); + f.broadcast.appendSliceAssumeCapacity(bytes); + + f.broadcast_undelivered = @intCast(f.instances.len - 1); + for (0.., f.instances) |to_id, *instance| { + if (to_id == from_id) continue; + instance.broadcast_written = 0; + f.addStdinWrite(@intCast(to_id)); + } + } + + fn broadcastComplete(f: *FuzzTestRunner) !void { + assert(f.instances.len > 1); + assert(f.broadcast_undelivered == 0); + f.broadcast.clearRetainingCapacity(); + + const pending = &f.pending_broadcasts; + if (pending.items.len != 0) { + // Another broadcast is pending; copy it over to `broadcast` + + const footer_len = @sizeOf(PendingBroadcastFooter); + const footer_bytes = pending.items[pending.items.len - footer_len ..]; + const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes); + pending.items.len -= footer_len; + + const body = pending.items[pending.items.len - footer.body_len ..]; + try f.instanceBroadcast(footer.from_id, body); + pending.items.len -= body.len; + } + } + + fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { + const step_owner = f.run.step.owner; + const arena = step_owner.allocator; + + // Collect any available stderr + while (f.batch.next()) |completion| { + if (completion.index % 3 != 2) continue; + const len = completion.result.file_read_streaming catch continue; + f.instances[completion.index / 3].stderr.items.len += len; + } + + var stderr_len: usize = 0; + for (f.instances) |*instance| stderr_len += instance.stderr.items.len; + const stderr = try arena.alloc(u8, stderr_len); + + stderr_len = 0; + for (f.instances) |*instance| { + @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items); + stderr_len += instance.stderr.items.len; + } + return stderr; + } +}; + +fn evalFuzzTest( + run: *Run, + spawn_options: process.SpawnOptions, + options: Step.MakeOptions, + fuzz_context: FuzzContext, +) !void { + var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); + defer f.deinit(); + try f.startInstances(); + try f.listen(); +} + +const StdioPollEnum = enum { stdout, stderr }; + +fn evalZigTest( + run: *Run, + spawn_options: process.SpawnOptions, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !void { + if (fuzz_context != null) { + try evalFuzzTest(run, spawn_options, options, fuzz_context.?); + return; + } + + const step_owner = run.step.owner; + const gpa = step_owner.allocator; + const arena = step_owner.allocator; + const io = step_owner.graph.io; + + // We will update this every time a child runs. + run.step.result_peak_rss = 0; + + var test_results: Step.TestResults = .{ + .test_count = 0, + .skip_count = 0, + .fail_count = 0, + .crash_count = 0, + .timeout_count = 0, + .leak_count = 0, + .log_err_count = 0, + }; + var test_metadata: ?TestMetadata = null; + + while (true) { + var child = try process.spawn(io, spawn_options); + var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; + var multi_reader: Io.File.MultiReader = undefined; + multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); + var child_killed = false; + defer if (!child_killed) { + child.kill(io); + multi_reader.deinit(); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + }; + + switch (try waitZigTest( + run, + &child, + options, + &multi_reader, + &test_metadata, + &test_results, + )) { + .write_failed => |err| { + // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured + // all available stderr to make our error output as useful as possible. + const stderr_fr = multi_reader.fileReader(1); + while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) { + error.ReadFailed => return stderr_fr.err.?, + error.EndOfStream => {}, + } + run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); + + // Clean up everything and wait for the child to exit. + child.stdin.?.close(io); + child.stdin = null; + multi_reader.deinit(); + child_killed = true; + const term = try child.wait(io); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + + // The individual unit test results are irrelevant: the test runner itself broke! + // Fail immediately without populating `s.test_results`. + return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); + }, + .no_poll => |no_poll| { + // This might be a success (we requested exit and the child dutifully closed stdout) or + // a crash of some kind. Either way, the child will terminate by itself -- wait for it. + const stderr_reader = multi_reader.reader(1); + const stderr_owned = try arena.dupe(u8, stderr_reader.buffered()); + + // Clean up everything and wait for the child to exit. + child.stdin.?.close(io); + child.stdin = null; + multi_reader.deinit(); + child_killed = true; + const term = try child.wait(io); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + + if (no_poll.active_test_index) |test_index| { + // A test was running, so this is definitely a crash. Report it against that + // test, and continue to the next test. + test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; + test_results.crash_count += 1; + try run.step.addError("'{s}' {f}{s}{s}", .{ + test_metadata.?.testName(test_index), + fmtTerm(term), + if (stderr_owned.len != 0) " with stderr:\n" else "", + std.mem.trim(u8, stderr_owned, "\n"), + }); + continue; + } + + // Report an error if the child terminated uncleanly or if we were still trying to run more tests. + run.step.result_stderr = stderr_owned; + const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); + if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { + // The individual unit test results are irrelevant: the test runner itself broke! + // Fail immediately without populating `s.test_results`. + return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); + } + + // We're done with all of the tests! Commit the test results and return. + run.step.test_results = test_results; + if (test_metadata) |tm| { + run.cached_test_metadata = tm.toCachedTestMetadata(); + if (options.web_server) |ws| { + if (run.step.owner.graph.time_report) { + ws.updateTimeReportRunTest( + run, + &run.cached_test_metadata.?, + tm.ns_per_test, + ); + } + } + } + return; + }, + .timeout => |timeout| { + const stderr_reader = multi_reader.reader(1); + const stderr = stderr_reader.buffered(); + stderr_reader.tossBuffered(); + if (timeout.active_test_index) |test_index| { + // A test was running. Report the timeout against that test, and continue on to + // the next test. + test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; + test_results.timeout_count += 1; + try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ + test_metadata.?.testName(test_index), + Io.Duration{ .nanoseconds = timeout.ns_elapsed }, + if (stderr.len != 0) " with stderr:\n" else "", + std.mem.trim(u8, stderr, "\n"), + }); + continue; + } + // Just log an error and let the child be killed. + run.step.result_stderr = try arena.dupe(u8, stderr); + // The individual unit test results in `results` are irrelevant: the test runner + // is broken! Fail immediately without populating `s.test_results`. + return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); + }, + } + comptime unreachable; + } +} + +const TestMetadata = struct { + names: []const u32, + ns_per_test: []u64, + expected_panic_msgs: []const u32, + string_bytes: []const u8, + next_index: u32, + prog_node: std.Progress.Node, + + fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { + return .{ + .names = tm.names, + .string_bytes = tm.string_bytes, + }; + } + + fn testName(tm: TestMetadata, index: u32) []const u8 { + return tm.toCachedTestMetadata().testName(index); + } +}; + +pub const CachedTestMetadata = struct { + names: []const u32, + string_bytes: []const u8, + + pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { + return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); + } +}; + +fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { + while (metadata.next_index < metadata.names.len) { + const i = metadata.next_index; + metadata.next_index += 1; + + if (metadata.expected_panic_msgs[i] != 0) continue; + + const name = metadata.testName(i); + if (sub_prog_node.*) |n| n.end(); + sub_prog_node.* = metadata.prog_node.start(name, 0); + + try sendRunTestMessage(io, in, .run_test, i); + return; + } else { + metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done + try sendMessage(io, in, .exit); + } +} + +fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 4, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u32, index, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +fn sendRunFuzzTestMessage( + io: Io, + file: Io.File, + test_names: []const []const u8, + kind: std.Build.abi.fuzz.LimitKind, + amount_or_instance: u64, +) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = .start_fuzzing, + .bytes_len = 1 + 8 + 4 + count: { + var c: u32 = @intCast(test_names.len * 4); + for (test_names) |name| { + c += @intCast(name.len); + } + break :count c; + }, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + for (test_names) |test_name| { + w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeAll(test_name) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + } +} + +fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { + const b = run.step.owner; + const io = b.graph.io; + const arena = b.allocator; + const gpa = b.allocator; + + var child = try process.spawn(io, spawn_options); + defer child.kill(io); + + switch (run.stdin) { + .bytes => |bytes| { + child.stdin.?.writeStreamingAll(io, bytes) catch |err| { + return run.step.fail("unable to write stdin: {t}", .{err}); + }; + child.stdin.?.close(io); + child.stdin = null; + }, + .lazy_path => |lazy_path| { + const path = lazy_path.getPath3(b, &run.step); + const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { + return run.step.fail("unable to open stdin file: {t}", .{err}); + }; + defer file.close(io); + // TODO https://github.com/ziglang/zig/issues/23955 + var read_buffer: [1024]u8 = undefined; + var file_reader = file.reader(io, &read_buffer); + var write_buffer: [1024]u8 = undefined; + var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); + _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { + error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ + path, file_reader.err.?, + }), + error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ + stdin_writer.err.?, + }), + }; + stdin_writer.interface.flush() catch |err| switch (err) { + error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ + stdin_writer.err.?, + }), + }; + child.stdin.?.close(io); + child.stdin = null; + }, + .none => {}, + } + + var stdout_bytes: ?[]const u8 = null; + var stderr_bytes: ?[]const u8 = null; + + if (child.stdout) |stdout| { + if (child.stderr) |stderr| { + var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; + var multi_reader: Io.File.MultiReader = undefined; + multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); + defer multi_reader.deinit(); + + const stdout_reader = multi_reader.reader(0); + const stderr_reader = multi_reader.reader(1); + + while (multi_reader.fill(64, .none)) |_| { + if (run.stdio_limit.toInt()) |limit| { + if (stdout_reader.buffered().len > limit) + return error.StdoutStreamTooLong; + if (stderr_reader.buffered().len > limit) + return error.StderrStreamTooLong; + } + } else |err| switch (err) { + error.Timeout => unreachable, + error.EndOfStream => {}, + else => |e| return e, + } + + try multi_reader.checkAnyError(); + + // TODO: this string can leak since alloc below can return error. + stdout_bytes = try multi_reader.toOwnedSlice(0); + // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. + stderr_bytes = try multi_reader.toOwnedSlice(1); + } else { + var stdout_reader = stdout.readerStreaming(io, &.{}); + stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return stdout_reader.err.?, + error.StreamTooLong => return error.StdoutStreamTooLong, + }; + } + } else if (child.stderr) |stderr| { + var stderr_reader = stderr.readerStreaming(io, &.{}); + stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return stderr_reader.err.?, + error.StreamTooLong => return error.StderrStreamTooLong, + }; + } + + if (stderr_bytes) |bytes| if (bytes.len > 0) { + // Treat stderr as an error message. + const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { + .check => |checks| !checksContainStderr(checks.items), + else => true, + }; + if (stderr_is_diagnostic) { + run.step.result_stderr = bytes; + } + }; + + run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; + + return .{ + .term = try child.wait(io), + .stdout = stdout_bytes, + .stderr = stderr_bytes, + }; +} + +const IndexedOutput = struct { + index: usize, + tag: @typeInfo(Arg).@"union".tag_type.?, + output: *Output, +}; + +pub fn rerunInFuzzMode( + run: *Run, + fuzz: *std.Build.Fuzz, + prog_node: std.Progress.Node, +) !void { + const step = &run.step; + const b = step.owner; + const io = b.graph.io; + const arena = b.allocator; + var argv_list: std.ArrayList([]const u8) = .empty; + for (run.argv.items) |arg| { + switch (arg) { + .bytes => |bytes| { + try argv_list.append(arena, bytes); + }, + .lazy_path => |file| { + const file_path = file.lazy_path.getPath3(b, step); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); + }, + .decorated_directory => |dd| { + const file_path = dd.lazy_path.getPath3(b, step); + try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); + }, + .file_content => |file_plp| { + const file_path = file_plp.lazy_path.getPath3(b, step); + + var result: std.Io.Writer.Allocating = .init(arena); + errdefer result.deinit(); + result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + + const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); + defer file.close(io); + + var buf: [1024]u8 = undefined; + var file_reader = file.reader(io, &buf); + _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { + error.ReadFailed => return file_reader.err.?, + error.WriteFailed => return error.OutOfMemory, + }; + + try argv_list.append(arena, result.written()); + }, + .artifact => |pa| { + const artifact = pa.artifact; + const file_path: []const u8 = p: { + if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); + break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; + }; + try argv_list.append(arena, b.fmt("{s}{s}", .{ + pa.prefix, + run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), + })); + }, + .output_file, .output_directory => unreachable, + } + } + + if (run.step.result_failed_command) |cmd| { + fuzz.gpa.free(cmd); + run.step.result_failed_command = null; + } + + const has_side_effects = false; + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ + .progress_node = prog_node, + .watch = undefined, // not used by `runCommand` + .web_server = null, // only needed for time reports + .unit_test_timeout_ns = null, // don't time out fuzz tests for now + .gpa = fuzz.gpa, + }, .{ + .fuzz = fuzz, + }); +} + +fn populateGeneratedPaths( + arena: std.mem.Allocator, + output_placeholders: []const IndexedOutput, + captured_stdout: ?*CapturedStdIo, + captured_stderr: ?*CapturedStdIo, + cache_root: Cache.Directory, + digest: *const Cache.HexDigest, +) !void { + for (output_placeholders) |placeholder| { + placeholder.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, placeholder.output.basename, + }); + } + + if (captured_stdout) |captured| { + captured.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, captured.output.basename, + }); + } + + if (captured_stderr) |captured| { + captured.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, captured.output.basename, + }); + } +} + +fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void { + if (term) |t| switch (t) { + .exited => |code| try w.print("exited with code {d}", .{code}), + .signal => |sig| try w.print("terminated with signal {t}", .{sig}), + .stopped => |sig| try w.print("stopped with signal {t}", .{sig}), + .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}), + } else { + try w.writeAll("exited with any code"); + } +} +fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) { + return .{ .data = term }; +} + +const FuzzContext = struct { + fuzz: *std.Build.Fuzz, +}; + +fn runCommand( + run: *Run, + argv: []const []const u8, + has_side_effects: bool, + output_dir_path: []const u8, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !void { + const step = &run.step; + const b = step.owner; + const arena = b.allocator; + const gpa = options.gpa; + const io = b.graph.io; + + const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; + + try step.handleChildProcUnsupported(); + try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); + + const allow_skip = switch (run.stdio) { + .check, .zig_test => run.skip_foreign_checks, + else => false, + }; + + var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); + defer interp_argv.deinit(); + + var environ_map: EnvMap = env: { + const orig = run.environ_map orelse &b.graph.environ_map; + break :env try orig.clone(gpa); + }; + defer environ_map.deinit(); + + const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { + // InvalidExe: cpu arch mismatch + // FileNotFound: can happen with a wrong dynamic linker path + if (err == error.InvalidExe or err == error.FileNotFound) interpret: { + // TODO: learn the target from the binary directly rather than from + // relying on it being a Compile step. This will make this logic + // work even for the edge case that the binary was produced by a + // third party. + const exe = switch (run.argv.items[0]) { + .artifact => |exe| exe.artifact, + else => break :interpret, + }; + switch (exe.kind) { + .exe, .@"test" => {}, + else => break :interpret, + } + + const root_target = exe.rootModuleTarget(); + const need_cross_libc = exe.is_linking_libc and + (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); + const other_target = exe.root_module.resolved_target.?.result; + switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ + .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, + .link_libc = exe.is_linking_libc, + })) { + .native, .rosetta => { + if (allow_skip) return error.MakeSkipped; + break :interpret; + }, + .wine => |bin_name| { + if (b.enable_wine) { + try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); + + // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but + // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. + if (environ_map.get("WINEDEBUG") == null) { + try environ_map.put("WINEDEBUG", "-all"); + } + } else { + return failForeign(run, "-fwine", argv[0], exe); + } + }, + .qemu => |bin_name| { + if (b.enable_qemu) { + try interp_argv.append(bin_name); + + if (need_cross_libc) { + if (b.libc_runtimes_dir) |dir| { + try interp_argv.append("-L"); + try interp_argv.append(b.pathJoin(&.{ + dir, + try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( + b.allocator, + root_target.cpu.arch, + root_target.os.tag, + root_target.abi, + ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( + b.allocator, + root_target.cpu.arch, + root_target.abi, + ) else unreachable, + })); + } else return failForeign(run, "--libc-runtimes", argv[0], exe); + } + + try interp_argv.appendSlice(argv); + } else return failForeign(run, "-fqemu", argv[0], exe); + }, + .darling => |bin_name| { + if (b.enable_darling) { + try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); + } else { + return failForeign(run, "-fdarling", argv[0], exe); + } + }, + .wasmtime => |bin_name| { + if (b.enable_wasmtime) { + try interp_argv.append(bin_name); + try interp_argv.append("--dir=."); + // Wasmtime doeesn't inherit environment variables from the parent process + // by default. '-S inherit-env' was added in Wasmtime version 20. + try interp_argv.append("-Sinherit-env"); + try interp_argv.append(argv[0]); + try interp_argv.appendSlice(argv[1..]); + } else { + return failForeign(run, "-fwasmtime", argv[0], exe); + } + }, + .bad_dl => |foreign_dl| { + if (allow_skip) return error.MakeSkipped; + + const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; + + return step.fail( + \\the host system is unable to execute binaries from the target + \\ because the host dynamic linker is '{s}', + \\ while the target dynamic linker is '{s}'. + \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step + , .{ host_dl, foreign_dl }); + }, + .bad_os_or_cpu => { + if (allow_skip) return error.MakeSkipped; + + const host_name = try b.graph.host.result.zigTriple(b.allocator); + const foreign_name = try root_target.zigTriple(b.allocator); + + return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ + host_name, foreign_name, + }); + }, + } + + if (root_target.os.tag == .windows) { + // On Windows we don't have rpaths so we have to add .dll search paths to PATH + run.addPathForDynLibs(exe); + } + + gpa.free(step.result_failed_command.?); + step.result_failed_command = null; + try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); + + break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { + if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; + if (e == error.MakeFailed) return error.MakeFailed; // error already reported + return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); + }; + } + if (err == error.MakeFailed) return error.MakeFailed; // error already reported + + return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); + }; + + const generic_result = opt_generic_result orelse { + assert(run.stdio == .zig_test); + // Specific errors have already been reported, and test results are populated. All we need + // to do is report step failure if any test failed. + if (!step.test_results.isSuccess()) return error.MakeFailed; + return; + }; + + assert(fuzz_context == null); + assert(run.stdio != .zig_test); + + // Capture stdout and stderr to GeneratedFile objects. + const Stream = struct { + captured: ?*CapturedStdIo, + bytes: ?[]const u8, + }; + for ([_]Stream{ + .{ + .captured = run.captured_stdout, + .bytes = generic_result.stdout, + }, + .{ + .captured = run.captured_stderr, + .bytes = generic_result.stderr, + }, + }) |stream| { + if (stream.captured) |captured| { + const output_components = .{ output_dir_path, captured.output.basename }; + const output_path = try b.cache_root.join(arena, &output_components); + captured.output.generated_file.path = output_path; + + const sub_path = b.pathJoin(&output_components); + const sub_path_dirname = Dir.path.dirname(sub_path).?; + b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, sub_path_dirname, @errorName(err), + }); + }; + const data = switch (captured.trim_whitespace) { + .none => stream.bytes.?, + .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), + .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), + .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), + }; + b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { + return step.fail("unable to write file '{f}{s}': {s}", .{ + b.cache_root, sub_path, @errorName(err), + }); + }; + } + } + + switch (run.stdio) { + .zig_test => unreachable, + .check => |checks| for (checks.items) |check| switch (check) { + .expect_stderr_exact => |expected_bytes| { + if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { + return step.fail( + \\========= expected this stderr: ========= + \\{s} + \\========= but found: ==================== + \\{s} + , .{ + expected_bytes, + generic_result.stderr.?, + }); + } + }, + .expect_stderr_match => |match| { + if (mem.find(u8, generic_result.stderr.?, match) == null) { + return step.fail( + \\========= expected to find in stderr: ========= + \\{s} + \\========= but stderr does not contain it: ===== + \\{s} + , .{ + match, + generic_result.stderr.?, + }); + } + }, + .expect_stdout_exact => |expected_bytes| { + if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { + return step.fail( + \\========= expected this stdout: ========= + \\{s} + \\========= but found: ==================== + \\{s} + , .{ + expected_bytes, + generic_result.stdout.?, + }); + } + }, + .expect_stdout_match => |match| { + if (mem.find(u8, generic_result.stdout.?, match) == null) { + return step.fail( + \\========= expected to find in stdout: ========= + \\{s} + \\========= but stdout does not contain it: ===== + \\{s} + , .{ + match, + generic_result.stdout.?, + }); + } + }, + .expect_term => |expected_term| { + if (!termMatches(expected_term, generic_result.term)) { + return step.fail("process {f} (expected {f})", .{ + fmtTerm(generic_result.term), + fmtTerm(expected_term), + }); + } + }, + }, + else => { + // On failure, report captured stderr like normal standard error output. + const bad_exit = switch (generic_result.term) { + .exited => |code| code != 0, + .signal, .stopped, .unknown => true, + }; + if (bad_exit) { + if (generic_result.stderr) |bytes| { + run.step.result_stderr = bytes; + } + } + + try step.handleChildProcessTerm(generic_result.term); + }, + } +} + +const EvalGenericResult = struct { + term: process.Child.Term, + stdout: ?[]const u8, + stderr: ?[]const u8, +}; + +fn spawnChildAndCollect( + run: *Run, + argv: []const []const u8, + environ_map: *EnvMap, + has_side_effects: bool, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !?EvalGenericResult { + const b = run.step.owner; + const graph = b.graph; + const io = graph.io; + + if (fuzz_context != null) { + assert(!has_side_effects); + assert(run.stdio == .zig_test); + } + + const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; + + // If an error occurs, it's caused by this command: + assert(run.step.result_failed_command == null); + run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ + .child = environ_map, + .parent = &graph.environ_map, + }, argv); + + var spawn_options: process.SpawnOptions = .{ + .argv = argv, + .cwd = child_cwd, + .environ_map = environ_map, + .request_resource_usage_statistics = true, + .stdin = if (run.stdin != .none) s: { + assert(run.stdio != .inherit); + break :s .pipe; + } else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .ignore, + .inherit => .inherit, + .check => .ignore, + .zig_test => .pipe, + }, + .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .ignore, + .inherit => .inherit, + .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, + .zig_test => .pipe, + }, + .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .pipe, + .inherit => .inherit, + .check => .pipe, + .zig_test => .pipe, + }, + }; + + if (run.stdio == .zig_test) { + const started: Io.Clock.Timestamp = .now(io, .awake); + const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| e, + }; + run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + try result; + return null; + } else { + const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; + if (!run.disable_zig_progress and !inherit) { + spawn_options.progress_node = options.progress_node; + } + const terminal_mode: Io.Terminal.Mode = if (inherit) m: { + const stderr = try io.lockStderr(&.{}, graph.stderr_mode); + break :m stderr.terminal_mode; + } else .no_color; + defer if (inherit) io.unlockStderr(); + try setColorEnvironmentVariables(run, environ_map, terminal_mode); + + const started: Io.Clock.Timestamp = .now(io, .awake); + const result = evalGeneric(run, spawn_options) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| e, + }; + run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + return try result; + } +} + +fn hashStdIo(hh: *Cache.HashHelper, stdio: StdIo) void { + switch (stdio) { + .infer_from_args, .inherit, .zig_test => {}, + .check => |checks| for (checks.items) |check| { + hh.add(@as(std.meta.Tag(StdIo.Check), check)); + switch (check) { + .expect_stderr_exact, + .expect_stderr_match, + .expect_stdout_exact, + .expect_stdout_match, + => |s| hh.addBytes(s), + + .expect_term => |term| { + hh.add(@as(std.meta.Tag(process.Child.Term), term)); + switch (term) { + inline .exited, .signal, .stopped => |x| hh.add(x), + .unknown => |x| hh.add(x), + } + }, + } + }, + } +} +fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { + return if (expected) |e| switch (e) { + .exited => |expected_code| switch (actual) { + .exited => |actual_code| expected_code == actual_code, + else => false, + }, + .signal => |expected_sig| switch (actual) { + .signal => |actual_sig| expected_sig == actual_sig, + else => false, + }, + .stopped => |expected_sig| switch (actual) { + .stopped => |actual_sig| expected_sig == actual_sig, + else => false, + }, + .unknown => |expected_code| switch (actual) { + .unknown => |actual_code| expected_code == actual_code, + else => false, + }, + } else switch (actual) { + .exited => true, + else => false, + }; +} + +fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { + color: switch (run.color) { + .manual => {}, + .enable => { + try environ_map.put("CLICOLOR_FORCE", "1"); + _ = environ_map.swapRemove("NO_COLOR"); + }, + .disable => { + try environ_map.put("NO_COLOR", "1"); + _ = environ_map.swapRemove("CLICOLOR_FORCE"); + }, + .inherit => switch (terminal_mode) { + .no_color, .windows_api => continue :color .disable, + .escape_codes => continue :color .enable, + }, + .auto => { + const capture_stderr = run.captured_stderr != null or switch (run.stdio) { + .check => |checks| checksContainStderr(checks.items), + .infer_from_args, .inherit, .zig_test => false, + }; + if (capture_stderr) { + continue :color .disable; + } else { + continue :color .inherit; + } + }, + } +} + +fn checksContainStdout(checks: []const StdIo.Check) bool { + for (checks) |check| switch (check) { + .expect_stderr_exact, + .expect_stderr_match, + .expect_term, + => continue, + + .expect_stdout_exact, + .expect_stdout_match, + => return true, + }; + return false; +} + +fn checksContainStderr(checks: []const StdIo.Check) bool { + for (checks) |check| switch (check) { + .expect_stdout_exact, + .expect_stdout_match, + .expect_term, + => continue, + + .expect_stderr_exact, + .expect_stderr_match, + => return true, + }; + return false; +} + +/// Returns whether the Run step has side effects *other than* updating the output arguments. +fn hasSideEffects(run: Run) bool { + if (run.has_side_effects) return true; + return switch (run.stdio) { + .infer_from_args => !run.hasAnyOutputArgs(), + .inherit => true, + .check => false, + .zig_test => false, + }; +} + +fn hasAnyOutputArgs(run: Run) bool { + if (run.captured_stdout != null) return true; + if (run.captured_stderr != null) return true; + for (run.argv.items) |arg| switch (arg) { + .output_file, .output_directory => return true, + else => continue, + }; + return false; +} + +/// If `path` is cwd-relative, make it relative to the cwd of the child instead. +/// +/// Whenever a path is included in the argv of a child, it should be put through this function first +/// to make sure the child doesn't see paths relative to a cwd other than its own. +fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { + const b = run.step.owner; + const graph = b.graph; + const arena = graph.arena; + + const path_str = path.toString(arena) catch @panic("OOM"); + if (Dir.path.isAbsolute(path_str)) { + // Absolute paths don't need changing. + return path_str; + } + const child_cwd_rel: []const u8 = rel: { + const child_lazy_cwd = run.cwd orelse break :rel path_str; + const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); + // Convert it from relative to *our* cwd, to relative to the *child's* cwd. + break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); + }; + // Not every path can be made relative, e.g. if the path and the child cwd are on different + // disk designators on Windows. In that case, `relative` will return an absolute path which we can + // just return. + if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel; + + // We're not done yet. In some cases this path must be prefixed with './': + // * On POSIX, the executable name cannot be a single component like 'foo' + // * Some executables might treat a leading '-' like a flag, which we must avoid + // There's no harm in it, so just *always* apply this prefix. + return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); +} + +fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { + const b = run.step.owner; + const compiles = artifact.getCompileDependencies(true); + for (compiles) |compile| { + if (compile.root_module.resolved_target.?.result.os.tag == .windows and + compile.isDynamicLibrary()) + { + addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); + } + } +} + +fn failForeign( + run: *Run, + suggested_flag: []const u8, + argv0: []const u8, + exe: *Step.Compile, +) error{ MakeFailed, MakeSkipped, OutOfMemory } { + switch (run.stdio) { + .check, .zig_test => { + if (run.skip_foreign_checks) + return error.MakeSkipped; + + const b = run.step.owner; + const host_name = try b.graph.host.result.zigTriple(b.allocator); + const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); + + return run.step.fail( + \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) + \\ consider using {s} or enabling skip_foreign_checks in the Run step + , .{ argv0, foreign_name, host_name, suggested_flag }); + }, + else => { + return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); + }, + } +} diff --git a/lib/compiler/maker/Step/WriteFile.zig b/lib/compiler/maker/Step/WriteFile.zig new file mode 100644 index 0000000000000000000000000000000000000000..d594f8983fe5ead949a5efdd0ba106c176777d34 --- /dev/null +++ b/lib/compiler/maker/Step/WriteFile.zig @@ -0,0 +1,206 @@ + +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; + const b = step.owner; + const graph = b.graph; + const io = graph.io; + const arena = b.allocator; + const gpa = graph.cache.gpa; + const write_file: *WriteFile = @fieldParentPtr("step", step); + + const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); + var open_dirs_count: usize = 0; + defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); + + switch (write_file.mode) { + .whole_cached => { + step.clearWatchInputs(); + + // The cache is used here not really as a way to speed things up - because writing + // the data to a file would probably be very fast - but as a way to find a canonical + // location to put build artifacts. + + // If, for example, a hard-coded path was used as the location to put WriteFile + // files, then two WriteFiles executing in parallel might clobber each other. + + var man = b.graph.cache.obtain(); + defer man.deinit(); + + for (write_file.files.items) |file| { + man.hash.addBytes(file.sub_path); + + switch (file.contents) { + .bytes => |bytes| { + man.hash.addBytes(bytes); + }, + .copy => |lazy_path| { + const path = lazy_path.getPath3(b, step); + _ = try man.addFilePath(path, null); + try step.addWatchInput(lazy_path); + }, + } + } + + for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { + man.hash.addBytes(dir.sub_path); + for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); + if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); + + const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); + const src_dir_path = dir.source.getPath3(b, step); + + var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{f}': {s}", .{ + src_dir_path, @errorName(err), + }); + }; + open_dir_cache_elem.* = src_dir; + open_dirs_count += 1; + + var it = try src_dir.walk(gpa); + defer it.deinit(); + while (try it.next(io)) |entry| { + if (!dir.options.pathIncluded(entry.path)) continue; + + switch (entry.kind) { + .directory => { + if (need_derived_inputs) { + const entry_path = try src_dir_path.join(arena, entry.path); + try step.addDirectoryWatchInputFromPath(entry_path); + } + }, + .file => { + const entry_path = try src_dir_path.join(arena, entry.path); + _ = try man.addFilePath(entry_path, null); + }, + else => continue, + } + } + } + + if (try step.cacheHit(&man)) { + const digest = man.final(); + write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); + assert(step.result_cached); + return; + } + + const digest = man.final(); + const cache_path = "o" ++ Dir.path.sep_str ++ digest; + + write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); + + try operate(write_file, open_dir_cache, .{ + .root_dir = b.cache_root, + .sub_path = cache_path, + }); + + try step.writeManifest(&man); + }, + .tmp => { + step.result_cached = false; + + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + + write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); + + try operate(write_file, open_dir_cache, .{ + .root_dir = b.cache_root, + .sub_path = tmp_dir_sub_path, + }); + }, + .mutate => |lp| { + step.result_cached = false; + const root_path = try lp.getPath4(b, step); + write_file.generated_directory.path = try root_path.toString(arena); + try operate(write_file, open_dir_cache, root_path); + }, + } +} + +fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { + const step = &write_file.step; + const b = step.owner; + const io = b.graph.io; + const gpa = b.graph.cache.gpa; + const arena = b.allocator; + + var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| + return step.fail("unable to make path {f}: {t}", .{ root_path, err }); + defer cache_dir.close(io); + + for (write_file.files.items) |file| { + if (Dir.path.dirname(file.sub_path)) |dirname| { + cache_dir.createDirPath(io, dirname) catch |err| { + return step.fail("unable to make path '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, dirname, err, + }); + }; + } + switch (file.contents) { + .bytes => |bytes| { + cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { + return step.fail("unable to write file '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, file.sub_path, err, + }); + }; + }, + .copy => |file_source| { + const source_path = file_source.getPath2(b, step); + const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { + return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ + source_path, root_path, Dir.path.sep, file.sub_path, err, + }); + }; + // At this point we already will mark the step as a cache miss. + // But this is kind of a partial cache hit since individual + // file copies may be avoided. Oh well, this information is + // discarded. + _ = prev_status; + }, + } + } + + for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { + const src_dir_path = dir.source.getPath3(b, step); + const dest_dirname = dir.sub_path; + + if (dest_dirname.len != 0) { + cache_dir.createDirPath(io, dest_dirname) catch |err| { + return step.fail("unable to make path '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, dest_dirname, err, + }); + }; + } + + var it = try already_open_dir.walk(gpa); + defer it.deinit(); + while (try it.next(io)) |entry| { + if (!dir.options.pathIncluded(entry.path)) continue; + + const src_entry_path = try src_dir_path.join(arena, entry.path); + const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); + switch (entry.kind) { + .directory => try cache_dir.createDirPath(io, dest_path), + .file => { + const prev_status = Io.Dir.updateFile( + src_entry_path.root_dir.handle, + io, + src_entry_path.sub_path, + cache_dir, + dest_path, + .{}, + ) catch |err| { + return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ + src_entry_path, root_path, Dir.path.sep, dest_path, err, + }); + }; + _ = prev_status; + }, + else => continue, + } + } + } +} diff --git a/lib/std/Build/Watch.zig b/lib/compiler/maker/Watch.zig similarity index 100% rename from lib/std/Build/Watch.zig rename to lib/compiler/maker/Watch.zig diff --git a/lib/std/Build/Watch/FsEvents.zig b/lib/compiler/maker/Watch/FsEvents.zig similarity index 100% rename from lib/std/Build/Watch/FsEvents.zig rename to lib/compiler/maker/Watch/FsEvents.zig diff --git a/lib/std/Build/WebServer.zig b/lib/compiler/maker/WebServer.zig similarity index 100% rename from lib/std/Build/WebServer.zig rename to lib/compiler/maker/WebServer.zig diff --git a/lib/std/Build.zig b/lib/std/Build.zig index f0eae560d432684103f13b4d43bcf64676c57155..f0edfb7b817eb2dbc0d03dff4a0267c2da519ae6 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -19,15 +19,14 @@ const ArrayList = std.ArrayList; pub const Cache = @import("Build/Cache.zig"); pub const Step = @import("Build/Step.zig"); pub const Module = @import("Build/Module.zig"); -pub const Watch = @import("Build/Watch.zig"); -pub const Fuzz = @import("Build/Fuzz.zig"); -pub const WebServer = @import("Build/WebServer.zig"); pub const abi = @import("Build/abi.zig"); +/// The serialized output of configure phase ingested by make phase. +pub const Configuration = @import("zig/Configuration.zig"); /// Shared state among all Build instances. graph: *Graph, -install_tls: TopLevelStep, -uninstall_tls: TopLevelStep, +install_tls: Step.TopLevel, +uninstall_tls: Step.TopLevel, allocator: Allocator, user_input_options: UserInputOptionsMap, available_options_map: AvailableOptionsMap, @@ -39,28 +38,17 @@ verbose_air: bool, verbose_llvm_ir: ?[]const u8, verbose_llvm_bc: ?[]const u8, verbose_llvm_cpu_features: bool, -reference_trace: ?u32 = null, invalid_user_input: bool, default_step: *Step, -top_level_steps: std.StringArrayHashMapUnmanaged(*TopLevelStep), +top_level_steps: std.StringArrayHashMapUnmanaged(*Step.TopLevel), install_prefix: []const u8, -dest_dir: ?[]const u8, -lib_dir: []const u8, -exe_dir: []const u8, -h_dir: []const u8, -install_path: []const u8, -sysroot: ?[]const u8 = null, -search_prefixes: ArrayList([]const u8), -libc_file: ?[]const u8 = null, /// Path to the directory containing build.zig. build_root: Cache.Directory, cache_root: Cache.Directory, pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, -args: ?[]const []const u8 = null, debug_log_scopes: []const []const u8 = &.{}, debug_compile_errors: bool = false, debug_incremental: bool = false, -debug_pkg_config: bool = false, /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, /// in particular at `Step` creation. /// Set to 0 to disable stack collection. @@ -76,12 +64,6 @@ enable_rosetta: bool = false, enable_wasmtime: bool = false, /// Use system Wine installation to run cross compiled Windows build artifacts. enable_wine: bool = false, -/// After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, -/// this will be the directory $glibc-build-dir/install/glibcs -/// Given the example of the aarch64 target, this is the directory -/// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. -/// Also works for dynamic musl. -libc_runtimes_dir: ?[]const u8 = null, dep_prefix: []const u8 = "", @@ -94,8 +76,6 @@ pkg_hash: []const u8, /// A mapping from dependency names to package hashes. available_deps: AvailableDeps, -release_mode: ReleaseMode, - build_id: ?std.zig.BuildId = null, pub const ReleaseMode = enum { @@ -116,14 +96,13 @@ pub const Graph = struct { system_package_mode: bool = false, debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, cache: Cache, - zig_exe: [:0]const u8, + zig_exe: []const u8, environ_map: process.Environ.Map, global_cache_root: Cache.Directory, zig_lib_directory: Cache.Directory, needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .empty, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, - incremental: ?bool = null, random_seed: u32 = 0, dependency_cache: InitializedDepMap = .empty, allow_so_scripts: ?bool = null, @@ -134,11 +113,12 @@ pub const Graph = struct { /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also /// respects the '--color' flag. stderr_mode: ?Io.Terminal.Mode = null, + release_mode: ReleaseMode = .off, }; const AvailableDeps = []const struct { []const u8, []const u8 }; -const SystemLibraryMode = enum { +pub const SystemLibraryMode = enum { /// User asked for the library to be disabled. /// The build runner has not confirmed whether the setting is recognized yet. user_disabled, @@ -245,19 +225,6 @@ const TypeId = enum { lazy_path_list, }; -const TopLevelStep = struct { - pub const base_id: Step.Id = .top_level; - - step: Step, - description: []const u8, -}; - -pub const DirList = struct { - lib_dir: ?[]const u8 = null, - exe_dir: ?[]const u8 = null, - include_dir: ?[]const u8 = null, -}; - pub fn create( graph: *Graph, build_root: Cache.Directory, @@ -285,15 +252,10 @@ pub fn create( .available_options_list = std.array_list.Managed(AvailableOption).init(arena), .top_level_steps = .{}, .default_step = undefined, - .search_prefixes = .empty, .install_prefix = undefined, - .lib_dir = undefined, - .exe_dir = undefined, - .h_dir = undefined, - .dest_dir = graph.environ_map.get("DESTDIR"), .install_tls = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = "install", .owner = b, }), @@ -301,21 +263,17 @@ pub fn create( }, .uninstall_tls = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = "uninstall", .owner = b, - .makeFn = makeUninstall, }), .description = "Remove build artifacts from prefix path", }, - .install_path = undefined, - .args = null, .modules = .empty, .named_writefiles = .empty, .named_lazy_paths = .empty, .pkg_hash = "", .available_deps = available_deps, - .release_mode = .off, }; try b.top_level_steps.put(arena, b.install_tls.step.name, &b.install_tls); try b.top_level_steps.put(arena, b.uninstall_tls.step.name, &b.uninstall_tls); @@ -330,19 +288,6 @@ fn createChild( pkg_hash: []const u8, pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap, -) error{OutOfMemory}!*Build { - const child = try createChildOnly(parent, dep_name, build_root, pkg_hash, pkg_deps, user_input_options); - try determineAndApplyInstallPrefix(child); - return child; -} - -fn createChildOnly( - parent: *Build, - dep_name: []const u8, - build_root: Cache.Directory, - pkg_hash: []const u8, - pkg_deps: AvailableDeps, - user_input_options: UserInputOptionsMap, ) error{OutOfMemory}!*Build { const allocator = parent.allocator; const child = try allocator.create(Build); @@ -351,7 +296,7 @@ fn createChildOnly( .allocator = allocator, .install_tls = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = "install", .owner = child, }), @@ -359,10 +304,9 @@ fn createChildOnly( }, .uninstall_tls = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = "uninstall", .owner = child, - .makeFn = makeUninstall, }), .description = "Remove build artifacts from prefix path", }, @@ -376,38 +320,31 @@ fn createChildOnly( .verbose_llvm_ir = parent.verbose_llvm_ir, .verbose_llvm_bc = parent.verbose_llvm_bc, .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, - .reference_trace = parent.reference_trace, .invalid_user_input = false, .default_step = undefined, .top_level_steps = .{}, .install_prefix = undefined, - .dest_dir = parent.dest_dir, .lib_dir = parent.lib_dir, .exe_dir = parent.exe_dir, .h_dir = parent.h_dir, .install_path = parent.install_path, .sysroot = parent.sysroot, - .search_prefixes = parent.search_prefixes, - .libc_file = parent.libc_file, .build_root = build_root, .cache_root = parent.cache_root, .debug_log_scopes = parent.debug_log_scopes, .debug_compile_errors = parent.debug_compile_errors, .debug_incremental = parent.debug_incremental, - .debug_pkg_config = parent.debug_pkg_config, .enable_darling = parent.enable_darling, .enable_qemu = parent.enable_qemu, .enable_rosetta = parent.enable_rosetta, .enable_wasmtime = parent.enable_wasmtime, .enable_wine = parent.enable_wine, - .libc_runtimes_dir = parent.libc_runtimes_dir, .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }), .modules = .empty, .named_writefiles = .empty, .named_lazy_paths = .empty, .pkg_hash = pkg_hash, .available_deps = pkg_deps, - .release_mode = parent.release_mode, }; try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls); try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls); @@ -702,59 +639,6 @@ fn hashUserInputOptionsMap(allocator: Allocator, user_input_options: UserInputOp user_option.hash(hasher); } -fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { - // Create an installation directory local to this package. This will be used when - // dependant packages require a standard prefix, such as include directories for C headers. - var hash = b.graph.cache.hash; - // Random bytes to make unique. Refresh this with new random bytes when - // implementation is modified in a non-backwards-compatible way. - hash.add(@as(u32, 0xd8cb0055)); - hash.addBytes(b.dep_prefix); - - var wyhash = std.hash.Wyhash.init(0); - hashUserInputOptionsMap(b.allocator, b.user_input_options, &wyhash); - hash.add(wyhash.final()); - - const digest = hash.final(); - const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); - b.resolveInstallPrefix(install_prefix, .{}); -} - -/// This function is intended to be called by lib/build_runner.zig, not a build.zig file. -pub fn resolveInstallPrefix(b: *Build, install_prefix: ?[]const u8, dir_list: DirList) void { - if (b.dest_dir) |dest_dir| { - b.install_prefix = install_prefix orelse "/usr"; - b.install_path = b.pathJoin(&.{ dest_dir, b.install_prefix }); - } else { - b.install_prefix = install_prefix orelse - (b.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); - b.install_path = b.install_prefix; - } - - var lib_list = [_][]const u8{ b.install_path, "lib" }; - var exe_list = [_][]const u8{ b.install_path, "bin" }; - var h_list = [_][]const u8{ b.install_path, "include" }; - - if (dir_list.lib_dir) |dir| { - if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; - lib_list[1] = dir; - } - - if (dir_list.exe_dir) |dir| { - if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; - exe_list[1] = dir; - } - - if (dir_list.include_dir) |dir| { - if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; - h_list[1] = dir; - } - - b.lib_dir = b.pathJoin(&lib_list); - b.exe_dir = b.pathJoin(&exe_list); - b.h_dir = b.pathJoin(&h_list); -} - /// Create a set of key-value pairs that can be converted into a Zig source /// file and then inserted into a Zig compilation's module table for importing. /// In other words, this provides a way to expose build.zig values to Zig @@ -1121,15 +1005,6 @@ pub fn getUninstallStep(b: *Build) *Step { return &b.uninstall_tls.step; } -fn makeUninstall(uninstall_step: *Step, options: Step.MakeOptions) anyerror!void { - _ = options; - const uninstall_tls: *TopLevelStep = @fieldParentPtr("step", uninstall_step); - const b: *Build = @fieldParentPtr("uninstall_tls", uninstall_tls); - - _ = b; - @panic("TODO implement https://github.com/ziglang/zig/issues/14943"); -} - /// Creates a configuration option to be passed to the build.zig script. /// When a user directly runs `zig build`, they can set these options with `-D` arguments. /// When a project depends on a Zig package as a dependency, it programmatically sets @@ -1350,10 +1225,10 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw } pub fn step(b: *Build, name: []const u8, description: []const u8) *Step { - const step_info = b.allocator.create(TopLevelStep) catch @panic("OOM"); + const step_info = b.allocator.create(Step.TopLevel) catch @panic("OOM"); step_info.* = .{ .step = .init(.{ - .id = TopLevelStep.base_id, + .tag = .top_level, .name = name, .owner = b, }), @@ -1373,8 +1248,10 @@ pub const StandardOptimizeOptionOptions = struct { }; pub fn standardOptimizeOption(b: *Build, options: StandardOptimizeOptionOptions) std.builtin.OptimizeMode { + const graph = b.graph; + if (options.preferred_optimize_mode) |mode| { - if (b.option(bool, "release", "optimize for end users") orelse (b.release_mode != .off)) { + if (b.option(bool, "release", "optimize for end users") orelse (graph.release_mode != .off)) { return mode; } else { return .Debug; @@ -1389,7 +1266,7 @@ pub fn standardOptimizeOption(b: *Build, options: StandardOptimizeOptionOptions) return mode; } - return switch (b.release_mode) { + return switch (graph.release_mode) { .off => .Debug, .any => { std.debug.print("the project does not declare a preferred optimization mode. choose: --release=fast, --release=safe, or --release=small\n", .{}); @@ -1824,36 +1701,11 @@ fn tryFindProgram(b: *Build, full_path: []const u8) ?[]const u8 { return null; } -pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) error{FileNotFound}![]const u8 { - // TODO report error for ambiguous situations - for (b.search_prefixes.items) |search_prefix| { - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - return tryFindProgram(b, b.pathJoin(&.{ search_prefix, "bin", name })) orelse continue; - } - } - if (b.graph.environ_map.get("PATH")) |PATH| { - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - var it = mem.tokenizeScalar(u8, PATH, fs.path.delimiter); - while (it.next()) |p| { - return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; - } - } - } - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - for (paths) |p| { - return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; - } - } - return error.FileNotFound; +pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) LazyPath { + _ = b; + _ = names; + _ = paths; + @panic("TODO rework findProgram to be based on LazyPath"); } pub fn runAllowFail( @@ -1918,10 +1770,6 @@ pub fn run(b: *Build, argv: []const []const u8) []u8 { ); } -pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { - b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); -} - pub fn getInstallPath(b: *Build, dir: InstallDir, dest_rel_path: []const u8) []const u8 { assert(!fs.path.isAbsolute(dest_rel_path)); // Install paths must be relative to the prefix const base_dir = switch (dir) { diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 51c1514f8597790dd8c02f53eae2e23dd9a26fd7..7b8a04d66e923953d5fec8fec03e938e6e9f22ac 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -10,24 +10,11 @@ const Cache = Build.Cache; const Path = Cache.Path; const ArrayList = std.ArrayList; -id: Id, +tag: std.Build.Configuration.Step.Tag, name: []const u8, owner: *Build, -makeFn: MakeFn, -dependencies: std.array_list.Managed(*Step), -/// This field is empty during execution of the user's build script, and -/// then populated during dependency loop checking in the build runner. -dependants: ArrayList(*Step), -/// Collects the set of files that retrigger this step to run. -/// -/// This is used by the build system's implementation of `--watch` but it can -/// also be potentially useful for IDEs to know what effects editing a -/// particular file has. -/// -/// Populated within `make`. Implementation may choose to clear and repopulate, -/// retain previous value, or update. -inputs: Inputs, +dependencies: ArrayList(*Step), /// Set this field to declare an upper bound on the amount of bytes of memory it will /// take to run the step. Zero means no limit. @@ -51,77 +38,11 @@ inputs: Inputs, max_rss: usize, state: State, -pending_deps: u32, - -result_error_msgs: ArrayList([]const u8), -result_error_bundle: std.zig.ErrorBundle, -result_stderr: []const u8, -result_cached: bool, -result_duration_ns: ?u64, -/// 0 means unavailable or not reported. -result_peak_rss: usize, -/// If the step is failed and this field is populated, this is the command which failed. -/// This field may be populated even if the step succeeded. -result_failed_command: ?[]const u8, -test_results: TestResults, /// The return address associated with creation of this step that can be useful /// to print along with debugging messages. debug_stack_trace: std.debug.StackTrace, -pub const TestResults = struct { - /// The total number of tests in the step. Every test has a "status" from the following: - /// * passed - /// * skipped - /// * failed cleanly - /// * crashed - /// * timed out - test_count: u32 = 0, - - /// The number of tests which were skipped (`error.SkipZigTest`). - skip_count: u32 = 0, - /// The number of tests which failed cleanly. - fail_count: u32 = 0, - /// The number of tests which terminated unexpectedly, i.e. crashed. - crash_count: u32 = 0, - /// The number of tests which timed out. - timeout_count: u32 = 0, - - /// The number of detected memory leaks. The associated test may still have passed; indeed, *all* - /// individual tests may have passed. However, the step as a whole fails if any test has leaks. - leak_count: u32 = 0, - /// The number of detected error logs. The associated test may still have passed; indeed, *all* - /// individual tests may have passed. However, the step as a whole fails if any test logs errors. - log_err_count: u32 = 0, - - pub fn isSuccess(tr: TestResults) bool { - // all steps are success or skip - return tr.fail_count == 0 and - tr.crash_count == 0 and - tr.timeout_count == 0 and - // no (otherwise successful) step leaked memory or logged errors - tr.leak_count == 0 and - tr.log_err_count == 0; - } - - /// Computes the number of tests which passed from the other values. - pub fn passCount(tr: TestResults) u32 { - return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count; - } -}; - -pub const MakeOptions = struct { - progress_node: std.Progress.Node, - watch: bool, - web_server: ?*Build.WebServer, - /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. - unit_test_timeout_ns: ?u64, - /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. - gpa: Allocator, -}; - -pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; - pub const State = enum { precheck_unstarted, precheck_started, @@ -132,57 +53,29 @@ pub const State = enum { /// be re-evaluated. precheck_done, dependency_failure, - success, - failure, - /// This state indicates that the step did not complete, however, it also did not fail, - /// and it is safe to continue executing its dependencies. - skipped, - /// This step was skipped because it specified a max_rss that exceeded the runner's maximum. - /// It is not safe to run its dependencies. - skipped_oom, }; -pub const Id = enum { - top_level, - compile, - install_artifact, - install_file, - install_dir, - remove_dir, - fail, - fmt, - translate_c, - write_file, - update_source_files, - run, - check_file, - check_object, - config_header, - objcopy, - options, - custom, +pub const Tag = std.Build.Configuration.Step.Tag; - pub fn Type(comptime id: Id) type { - return switch (id) { - .top_level => Build.TopLevelStep, - .compile => Compile, - .install_artifact => InstallArtifact, - .install_file => InstallFile, - .install_dir => InstallDir, - .fail => Fail, - .fmt => Fmt, - .translate_c => TranslateC, - .write_file => WriteFile, - .update_source_files => UpdateSourceFiles, - .run => Run, - .check_file => CheckFile, - .config_header => ConfigHeader, - .objcopy => ObjCopy, - .options => Options, - .custom => @compileError("no type available for custom step"), - }; - } -}; +pub fn Type(comptime tag: Tag) type { + return switch (tag) { + .top_level => Build.TopLevelStep, + .compile => Compile, + .install_artifact => InstallArtifact, + .install_file => InstallFile, + .install_dir => InstallDir, + .fail => Fail, + .fmt => Fmt, + .translate_c => TranslateC, + .write_file => WriteFile, + .update_source_files => UpdateSourceFiles, + .run => Run, + .check_file => CheckFile, + .config_header => ConfigHeader, + .objcopy => ObjCopy, + .options => Options, + }; +} pub const CheckFile = @import("Step/CheckFile.zig"); pub const ConfigHeader = @import("Step/ConfigHeader.zig"); @@ -199,32 +92,17 @@ pub const TranslateC = @import("Step/TranslateC.zig"); pub const WriteFile = @import("Step/WriteFile.zig"); pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); -pub const Inputs = struct { - table: Table, +pub const TopLevel = struct { + pub const base_tag: Step.Tag = .top_level; - pub const init: Inputs = .{ - .table = .{}, - }; - - pub const Table = std.ArrayHashMapUnmanaged(Build.Cache.Path, Files, Build.Cache.Path.TableAdapter, false); - /// The special file name "." means any changes inside the directory. - pub const Files = ArrayList([]const u8); - - pub fn populated(inputs: *Inputs) bool { - return inputs.table.count() != 0; - } - - pub fn clear(inputs: *Inputs, gpa: Allocator) void { - for (inputs.table.values()) |*files| files.deinit(gpa); - inputs.table.clearRetainingCapacity(); - } + step: Step, + description: []const u8, }; pub const StepOptions = struct { - id: Id, + tag: Tag, name: []const u8, owner: *Build, - makeFn: MakeFn = makeNoOp, first_ret_addr: ?usize = null, max_rss: usize = 0, }; @@ -233,90 +111,27 @@ pub fn init(options: StepOptions) Step { const arena = options.owner.allocator; return .{ - .id = options.id, + .tag = options.tag, .name = arena.dupe(u8, options.name) catch @panic("OOM"), .owner = options.owner, - .makeFn = options.makeFn, - .dependencies = std.array_list.Managed(*Step).init(arena), - .dependants = .empty, - .inputs = Inputs.init, + .dependencies = .empty, .state = .precheck_unstarted, - .pending_deps = undefined, // initialized by build runner .max_rss = options.max_rss, .debug_stack_trace = blk: { const addr_buf = arena.alloc(usize, options.owner.debug_stack_frames_count) catch @panic("OOM"); const first_ret_addr = options.first_ret_addr orelse @returnAddress(); break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf); }, - .result_error_msgs = .empty, - .result_error_bundle = std.zig.ErrorBundle.empty, - .result_stderr = "", - .result_cached = false, - .result_duration_ns = null, - .result_peak_rss = 0, - .result_failed_command = null, - .test_results = .{}, }; } -/// If the Step's `make` function reports `error.MakeFailed`, it indicates they -/// have already reported the error. Otherwise, we add a simple error report -/// here. -pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { - const arena = s.owner.allocator; - const graph = s.owner.graph; - const io = graph.io; - - var start_ts: ?Io.Timestamp = t: { - if (!graph.time_report) break :t null; - if (s.id == .compile) break :t null; - if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; - break :t Io.Clock.awake.now(io); - }; - const make_result = s.makeFn(s, options); - if (start_ts) |*ts| { - const duration = ts.untilNow(io, .awake); - options.web_server.?.updateTimeReportGeneric(s, duration); - } - - make_result catch |err| switch (err) { - error.MakeFailed, error.MakeSkipped => |e| return e, - else => { - s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); - return error.MakeFailed; - }, - }; - - if (!s.test_results.isSuccess()) { - return error.MakeFailed; - } - - if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { - const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ - s.result_peak_rss, s.max_rss, - }) catch @panic("OOM"); - s.result_error_msgs.append(arena, msg) catch @panic("OOM"); - } -} - pub fn dependOn(step: *Step, other: *Step) void { - step.dependencies.append(other) catch @panic("OOM"); -} - -fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { - _ = options; - - var all_cached = true; - - for (step.dependencies.items) |dep| { - all_cached = all_cached and dep.result_cached; - } - - step.result_cached = all_cached; + const arena = step.owner.allocator; + step.dependencies.append(arena, other) catch @panic("OOM"); } pub fn cast(step: *Step, comptime T: type) ?*T { - if (step.id == T.base_id) { + if (step.tag == T.base_tag) { return @fieldParentPtr("step", step); } return null; @@ -337,670 +152,6 @@ pub fn dump(step: *Step, t: Io.Terminal) void { } } -/// Populates `s.result_failed_command`. -pub fn captureChildProcess( - s: *Step, - gpa: Allocator, - progress_node: std.Progress.Node, - argv: []const []const u8, -) !std.process.RunResult { - const graph = s.owner.graph; - const arena = graph.arena; - const io = graph.io; - - // If an error occurs, it's happened in this command: - assert(s.result_failed_command == null); - s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); - - try handleChildProcUnsupported(s); - try handleVerbose(s.owner, .inherit, argv); - - const result = std.process.run(arena, io, .{ - .argv = argv, - .environ_map = &graph.environ_map, - .progress_node = progress_node, - }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); - - if (result.stderr.len > 0) { - try s.result_error_msgs.append(arena, result.stderr); - } - - return result; -} - -pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { - try step.addError(fmt, args); - return error.MakeFailed; -} - -pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { - const arena = step.owner.allocator; - const msg = try std.fmt.allocPrint(arena, fmt, args); - try step.result_error_msgs.append(arena, msg); -} - -pub const ZigProcess = struct { - child: std.process.Child, - multi_reader_buffer: Io.File.MultiReader.Buffer(2), - multi_reader: Io.File.MultiReader, - progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn, - - pub const StreamEnum = enum { stdout, stderr }; - - pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void { - zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null; - } - - pub fn deinit(zp: *ZigProcess, io: Io) void { - zp.child.kill(io); - zp.multi_reader.deinit(); - zp.* = undefined; - } -}; - -/// Assumes that argv contains `--listen=-` and that the process being spawned -/// is the zig compiler - the same version that compiled the build runner. -/// Populates `s.result_failed_command`. -pub fn evalZigProcess( - s: *Step, - argv: []const []const u8, - prog_node: std.Progress.Node, - watch: bool, - web_server: ?*Build.WebServer, - gpa: Allocator, -) !?Path { - const b = s.owner; - const io = b.graph.io; - - // If an error occurs, it's happened in this command: - assert(s.result_failed_command == null); - s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); - - if (s.getZigProcess()) |zp| update: { - assert(watch); - if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index); - zp.progress_ipc_index = null; - var exited = false; - defer if (exited) { - s.cast(Compile).?.zig_process = null; - zp.deinit(io); - gpa.destroy(zp); - } else zp.saveState(prog_node); - const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { - error.BrokenPipe, error.EndOfStream => |reason| { - std.log.info("{s} restart required: {t}", .{ argv[0], reason }); - // Process restart required. - const term = zp.child.wait(io) catch |e| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); - }; - _ = term; - exited = true; - break :update; - }, - else => |e| return e, - }; - - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); - } - - if (s.result_error_msgs.items.len > 0 and result == null) { - // Crash detected. - const term = zp.child.wait(io) catch |e| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); - }; - s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; - exited = true; - try handleChildProcessTerm(s, term); - return error.MakeFailed; - } - - return result; - } - assert(argv.len != 0); - - try handleChildProcUnsupported(s); - try handleVerbose(s.owner, .inherit, argv); - - const zp = try gpa.create(ZigProcess); - defer if (!watch) gpa.destroy(zp); - - zp.child = std.process.spawn(io, .{ - .argv = argv, - .environ_map = &b.graph.environ_map, - .stdin = .pipe, - .stdout = .pipe, - .stderr = .pipe, - .request_resource_usage_statistics = true, - .progress_node = prog_node, - }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); - - zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ - zp.child.stdout.?, zp.child.stderr.?, - }); - if (watch) s.cast(Compile).?.zig_process = zp; - defer if (!watch) zp.deinit(io); - - const result = result: { - defer if (watch) zp.saveState(prog_node); - break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); - }; - - if (!watch) { - // Send EOF to stdin. - zp.child.stdin.?.close(io); - zp.child.stdin = null; - - const term = zp.child.wait(io) catch |err| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); - }; - s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; - - // Special handling for Compile step that is expecting compile errors. - if (s.cast(Compile)) |compile| switch (term) { - .exited => { - // Note that the exit code may be 0 in this case due to the - // compiler server protocol. - if (compile.expect_errors != null) { - return error.NeedCompileErrorCheck; - } - }, - else => {}, - }; - - try handleChildProcessTerm(s, term); - } - - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); - } - - return result; -} - -/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. -pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { - const b = s.owner; - const io = b.graph.io; - const src_path = src_lazy_path.getPath3(b, s); - try handleVerbose(b, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); - return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| - return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); -} - -/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. -pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { - const b = s.owner; - const io = b.graph.io; - try handleVerbose(b, .inherit, &.{ "install", "-d", dest_path }); - return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| - return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); -} - -fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build.WebServer, gpa: Allocator) !?Path { - const b = s.owner; - const arena = b.allocator; - const io = b.graph.io; - - const start_ts = Io.Clock.awake.now(io); - - try sendMessage(io, zp.child.stdin.?, .update); - if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); - - var result: ?Path = null; - var eos_err: error{EndOfStream}!void = {}; - - const stdout = zp.multi_reader.fileReader(0); - - while (true) { - const Header = std.zig.Server.Message.Header; - const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) { - error.EndOfStream => break, - error.ReadFailed => return stdout.err.?, - }; - const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) { - error.EndOfStream => |e| { - // Better to report the crash with stderr below, but we set - // this in case the child exits successfully while violating - // this protocol. - eos_err = e; - break; - }, - error.ReadFailed => return stdout.err.?, - }; - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) { - return s.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - } - }, - .error_bundle => { - s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); - // This message indicates the end of the update. - if (watch) break; - }, - .emit_digest => { - const EmitDigest = std.zig.Server.Message.EmitDigest; - const emit_digest: *align(1) const EmitDigest = @ptrCast(body); - s.result_cached = emit_digest.flags.cache_hit; - const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; - result = .{ - .root_dir = b.cache_root, - .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), - }; - }, - .file_system_inputs => { - s.clearWatchInputs(); - var it = std.mem.splitScalar(u8, body, 0); - while (it.next()) |prefixed_path| { - const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); - const sub_path = try arena.dupe(u8, prefixed_path[1..]); - const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; - switch (prefix_index) { - .cwd => { - const path: Build.Cache.Path = .{ - .root_dir = Build.Cache.Directory.cwd(), - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .zig_lib => zl: { - if (s.cast(Step.Compile)) |compile| { - if (compile.zig_lib_dir) |zig_lib_dir| { - const lp = try zig_lib_dir.join(arena, sub_path); - try addWatchInput(s, lp); - break :zl; - } - } - const path: Build.Cache.Path = .{ - .root_dir = s.owner.graph.zig_lib_directory, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .local_cache => { - const path: Build.Cache.Path = .{ - .root_dir = b.cache_root, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .global_cache => { - const path: Build.Cache.Path = .{ - .root_dir = s.owner.graph.global_cache_root, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - } - } - }, - .time_report => if (web_server) |ws| { - const TimeReport = std.zig.Server.Message.TimeReport; - const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); - ws.updateTimeReportCompile(.{ - .compile = s.cast(Step.Compile).?, - .use_llvm = tr.flags.use_llvm, - .stats = tr.stats, - .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), - .llvm_pass_timings_len = tr.llvm_pass_timings_len, - .files_len = tr.files_len, - .decls_len = tr.decls_len, - .trailing = body[@sizeOf(TimeReport)..], - }); - }, - else => {}, // ignore other messages - } - } - - s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()); - - const stderr_contents = zp.multi_reader.reader(1).buffered(); - if (stderr_contents.len > 0) { - try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); - } - - try eos_err; - - return result; -} - -pub fn getZigProcess(s: *Step) ?*ZigProcess { - return switch (s.id) { - .compile => s.cast(Compile).?.zig_process, - else => null, - }; -} - -fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 0, - }; - var w = file.writer(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -pub fn handleVerbose( - b: *Build, - cwd: std.process.Child.Cwd, - argv: []const []const u8, -) error{OutOfMemory}!void { - return handleVerbose2(b, cwd, null, argv); -} - -pub fn handleVerbose2( - b: *Build, - cwd: std.process.Child.Cwd, - opt_env: ?*const std.process.Environ.Map, - argv: []const []const u8, -) error{OutOfMemory}!void { - if (b.verbose) { - const graph = b.graph; - // Intention of verbose is to print all sub-process command lines to - // stderr before spawning them. - const text = try allocPrintCmd(b.allocator, cwd, if (opt_env) |env| .{ - .child = env, - .parent = &graph.environ_map, - } else null, argv); - std.debug.print("{s}\n", .{text}); - } -} - -/// Asserts that the caller has already populated `s.result_failed_command`. -pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { - if (!std.process.can_spawn) { - return s.fail("unable to spawn process: host cannot spawn child processes", .{}); - } -} - -/// Asserts that the caller has already populated `s.result_failed_command`. -pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { - assert(s.result_failed_command != null); - return switch (term) { - .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), - .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), - .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), - .unknown => s.fail("process terminated unexpectedly", .{}), - }; -} - -pub fn allocPrintCmd( - gpa: Allocator, - cwd: std.process.Child.Cwd, - opt_env: ?struct { - child: *const std.process.Environ.Map, - parent: *const std.process.Environ.Map, - }, - argv: []const []const u8, -) Allocator.Error![]u8 { - const shell = struct { - fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { - for (string) |c| { - if (switch (c) { - else => true, - '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, - '=' => is_argv0, - }) break; - } else return writer.writeAll(string); - - try writer.writeByte('"'); - for (string) |c| { - if (switch (c) { - std.ascii.control_code.nul => break, - '!', '"', '$', '\\', '`' => true, - else => !std.ascii.isPrint(c), - }) try writer.writeByte('\\'); - switch (c) { - std.ascii.control_code.nul => unreachable, - std.ascii.control_code.bel => try writer.writeByte('a'), - std.ascii.control_code.bs => try writer.writeByte('b'), - std.ascii.control_code.ht => try writer.writeByte('t'), - std.ascii.control_code.lf => try writer.writeByte('n'), - std.ascii.control_code.vt => try writer.writeByte('v'), - std.ascii.control_code.ff => try writer.writeByte('f'), - std.ascii.control_code.cr => try writer.writeByte('r'), - std.ascii.control_code.esc => try writer.writeByte('E'), - ' '...'~' => try writer.writeByte(c), - else => try writer.print("{o:0>3}", .{c}), - } - } - try writer.writeByte('"'); - } - }; - - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - const writer = &aw.writer; - switch (cwd) { - .inherit => {}, - .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, - .dir => @panic("TODO"), - } - if (opt_env) |env| { - var it = env.child.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; - if (env.parent.get(key)) |process_value| { - if (std.mem.eql(u8, value, process_value)) continue; - } - writer.print("{s}=", .{key}) catch return error.OutOfMemory; - shell.escape(writer, value, false) catch return error.OutOfMemory; - writer.writeByte(' ') catch return error.OutOfMemory; - } - } - shell.escape(writer, argv[0], true) catch return error.OutOfMemory; - for (argv[1..]) |arg| { - writer.writeByte(' ') catch return error.OutOfMemory; - shell.escape(writer, arg, false) catch return error.OutOfMemory; - } - return aw.toOwnedSlice(); -} - -/// Prefer `cacheHitAndWatch` unless you already added watch inputs -/// separately from using the cache system. -pub fn cacheHit(s: *Step, man: *Build.Cache.Manifest) !bool { - s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); - return s.result_cached; -} - -/// Clears previous watch inputs, if any, and then populates watch inputs from -/// the full set of files picked up by the cache manifest. -/// -/// Must be accompanied with `writeManifestAndWatch`. -pub fn cacheHitAndWatch(s: *Step, man: *Build.Cache.Manifest) !bool { - const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); - s.result_cached = is_hit; - // The above call to hit() populates the manifest with files, so in case of - // a hit, we need to populate watch inputs. - if (is_hit) try setWatchInputsFromManifest(s, man); - return is_hit; -} - -fn failWithCacheError( - s: *Step, - man: *const Build.Cache.Manifest, - err: Build.Cache.Manifest.HitError, -) error{ OutOfMemory, Canceled, MakeFailed } { - switch (err) { - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ - man.diagnostic, e, - }), - .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ - prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, - }); - }, - }, - error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), - } -} - -/// Prefer `writeManifestAndWatch` unless you already added watch inputs -/// separately from using the cache system. -pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void { - if (s.test_results.isSuccess()) { - man.writeManifest() catch |err| { - try s.addError("unable to write cache manifest: {t}", .{err}); - }; - } -} - -/// Clears previous watch inputs, if any, and then populates watch inputs from -/// the full set of files picked up by the cache manifest. -/// -/// Must be accompanied with `cacheHitAndWatch`. -pub fn writeManifestAndWatch(s: *Step, man: *Build.Cache.Manifest) !void { - try writeManifest(s, man); - try setWatchInputsFromManifest(s, man); -} - -fn setWatchInputsFromManifest(s: *Step, man: *Build.Cache.Manifest) !void { - const arena = s.owner.allocator; - const prefixes = man.cache.prefixes(); - clearWatchInputs(s); - for (man.files.keys()) |file| { - // The file path data is freed when the cache manifest is cleaned up at the end of `make`. - const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); - try addWatchInputFromPath(s, .{ - .root_dir = prefixes[file.prefixed_path.prefix], - .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, std.fs.path.basename(sub_path)); - } -} - -/// For steps that have a single input that never changes when re-running `make`. -pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { - if (!step.inputs.populated()) try step.addWatchInput(lazy_path); -} - -pub fn clearWatchInputs(step: *Step) void { - const gpa = step.owner.allocator; - step.inputs.clear(gpa); -} - -/// Places a *file* dependency on the path. -pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!void { - switch (lazy_file) { - .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), - .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), - .cwd_relative => |path_string| { - try addWatchInputFromPath(step, .{ - .root_dir = .{ - .path = null, - .handle = Io.Dir.cwd(), - }, - .sub_path = std.fs.path.dirname(path_string) orelse "", - }, std.fs.path.basename(path_string)); - }, - // Nothing to watch because this dependency edge is modeled instead via `dependants`. - .generated => {}, - } -} - -/// Any changes inside the directory will trigger invalidation. -/// -/// See also `addDirectoryWatchInputFromPath` which takes a `Build.Cache.Path` instead. -/// -/// Paths derived from this directory should also be manually added via -/// `addDirectoryWatchInputFromPath` if and only if this function returns -/// `true`. -pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool { - switch (lazy_directory) { - .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), - .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), - .cwd_relative => |path_string| { - try addDirectoryWatchInputFromPath(step, .{ - .root_dir = .{ - .path = null, - .handle = Io.Dir.cwd(), - }, - .sub_path = path_string, - }); - }, - // Nothing to watch because this dependency edge is modeled instead via `dependants`. - .generated => return false, - } - return true; -} - -/// Any changes inside the directory will trigger invalidation. -/// -/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. -/// -/// This function should only be called when it has been verified that the -/// dependency on `path` is not already accounted for by a `Step` dependency. -/// In other words, before calling this function, first check that the -/// `Build.LazyPath` which this `path` is derived from is not `generated`. -pub fn addDirectoryWatchInputFromPath(step: *Step, path: Build.Cache.Path) !void { - return addWatchInputFromPath(step, path, "."); -} - -fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { - return addWatchInputFromPath(step, .{ - .root_dir = builder.build_root, - .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, std.fs.path.basename(sub_path)); -} - -fn addDirectoryWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { - return addDirectoryWatchInputFromPath(step, .{ - .root_dir = builder.build_root, - .sub_path = sub_path, - }); -} - -fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const u8) !void { - const gpa = step.owner.allocator; - const gop = try step.inputs.table.getOrPut(gpa, path); - if (!gop.found_existing) gop.value_ptr.* = .empty; - try gop.value_ptr.append(gpa, basename); -} - -/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. -pub fn reset(step: *Step, gpa: Allocator) void { - assert(step.state == .precheck_done); - - if (step.result_failed_command) |cmd| gpa.free(cmd); - - step.result_error_msgs.clearRetainingCapacity(); - step.result_stderr = ""; - step.result_cached = false; - step.result_duration_ns = null; - step.result_peak_rss = 0; - step.result_failed_command = null; - step.test_results = .{}; - step.clearWatchInputs(); - - step.result_error_bundle.deinit(gpa); - step.result_error_bundle = std.zig.ErrorBundle.empty; -} - -/// Implementation detail of file watching. Prepares the step for being re-evaluated. -/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. -pub fn invalidateResult(step: *Step, gpa: Allocator) bool { - if (step.state == .precheck_done) return false; - assert(step.pending_deps == 0); - step.state = .precheck_done; - step.reset(gpa); - for (step.dependants.items) |dependant| { - _ = dependant.invalidateResult(gpa); - dependant.pending_deps += 1; - } - return true; -} - test { _ = CheckFile; _ = Fail; diff --git a/lib/std/Build/Step/CheckFile.zig b/lib/std/Build/Step/CheckFile.zig index 1c3813ca824bbea35e6e15b19567dd1b47c212cc..4cb968ff96ef2343153d1f522406ac35db088d7d 100644 --- a/lib/std/Build/Step/CheckFile.zig +++ b/lib/std/Build/Step/CheckFile.zig @@ -16,7 +16,7 @@ expected_exact: ?[]const u8, source: std.Build.LazyPath, max_bytes: usize = 20 * 1024 * 1024, -pub const base_id: Step.Id = .check_file; +pub const base_tag: Step.Tag = .check_file; pub const Options = struct { expected_matches: []const []const u8 = &.{}, @@ -31,7 +31,7 @@ pub fn create( const check_file = owner.allocator.create(CheckFile) catch @panic("OOM"); check_file.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "CheckFile", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 5ce7bcb9d6b49d4baa6676bff610c46ff1975704..2897fddb747f199a72381070dfbfd28787e41734 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -20,7 +20,7 @@ const InstallDir = std.Build.InstallDir; const GeneratedFile = std.Build.GeneratedFile; const Path = std.Build.Cache.Path; -pub const base_id: Step.Id = .compile; +pub const base_tag: Step.Tag = .compile; step: Step, root_module: *Module, @@ -235,10 +235,6 @@ is_linking_libc: bool = false, /// Computed during make(). is_linking_libcpp: bool = false, -/// Populated during the make phase when there is a long-lived compiler process. -/// Managed by the build runner, not user build script. -zig_process: ?*Step.ZigProcess, - /// Enables coverage instrumentation that is only useful if you are using third /// party fuzzers that depend on it. Otherwise, slows down the instrumented /// binary with unnecessary function calls. @@ -418,10 +414,9 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .kind = options.kind, .name = name, .step = .init(.{ - .id = base_id, + .tag = base_tag, .name = step_name, .owner = owner, - .makeFn = make, .max_rss = options.max_rss, }), .version = options.version, @@ -452,8 +447,6 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .use_new_linker = null, - - .zig_process = null, }; if (options.zig_lib_dir) |lp| { @@ -701,122 +694,6 @@ pub fn producesImplib(compile: *Compile) bool { return compile.isDll(); } -const PkgConfigResult = struct { - cflags: []const []const u8, - libs: []const []const u8, -}; - -/// Run pkg-config for the given library name and parse the output, returning the arguments -/// that should be passed to zig to link the given library. -pub fn runPkgConfig(step: *Step, lib_name: []const u8) !PkgConfigResult { - const wl_rpath_prefix = "-Wl,-rpath,"; - - const b = step.owner; - const pkg_name = match: { - // First we have to map the library name to pkg config name. Unfortunately, - // there are several examples where this is not straightforward: - // -lSDL2 -> pkg-config sdl2 - // -lgdk-3 -> pkg-config gdk-3.0 - // -latk-1.0 -> pkg-config atk - // -lpulse -> pkg-config libpulse - const pkgs = try getPkgConfigList(b); - - // Exact match means instant winner. - for (pkgs) |pkg| { - if (mem.eql(u8, pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Next we'll try ignoring case. - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Prefixed "lib" or suffixed ".0". - for (pkgs) |pkg| { - if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { - const prefix = pkg.name[0..pos]; - const suffix = pkg.name[pos + lib_name.len ..]; - if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; - if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; - break :match pkg.name; - } - } - - // Trimming "-1.0". - if (mem.endsWith(u8, lib_name, "-1.0")) { - const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { - break :match pkg.name; - } - } - } - - return error.PackageNotFound; - }; - - var code: u8 = undefined; - const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = if (b.runAllowFail(&[_][]const u8{ - pkg_config_exe, - pkg_name, - "--cflags", - "--libs", - }, &code, .ignore)) |stdout| stdout else |err| switch (err) { - error.ProcessTerminated => return error.PkgConfigCrashed, - error.ExecNotSupported => return error.PkgConfigFailed, - error.ExitCodeFailure => return error.PkgConfigFailed, - error.FileNotFound => return error.PkgConfigNotInstalled, - else => return err, - }; - - var zig_cflags: std.ArrayList([]const u8) = .empty; - defer zig_cflags.deinit(b.allocator); - var zig_libs: std.ArrayList([]const u8) = .empty; - defer zig_libs.deinit(b.allocator); - - var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); - while (arg_it.next()) |arg| { - if (mem.eql(u8, arg, "-I")) { - const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_cflags.appendSlice(b.allocator, &.{ "-I", dir }); - } else if (mem.startsWith(u8, arg, "-I")) { - try zig_cflags.append(b.allocator, arg); - } else if (mem.eql(u8, arg, "-L")) { - const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_libs.appendSlice(b.allocator, &.{ "-L", dir }); - } else if (mem.startsWith(u8, arg, "-L")) { - try zig_libs.append(b.allocator, arg); - } else if (mem.eql(u8, arg, "-l")) { - const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_libs.appendSlice(b.allocator, &.{ "-l", lib }); - } else if (mem.startsWith(u8, arg, "-l")) { - try zig_libs.append(b.allocator, arg); - } else if (mem.eql(u8, arg, "-D")) { - const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_cflags.appendSlice(b.allocator, &.{ "-D", macro }); - } else if (mem.startsWith(u8, arg, "-D")) { - try zig_cflags.append(b.allocator, arg); - } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { - try zig_cflags.appendSlice(b.allocator, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); - } else if (b.debug_pkg_config) { - return step.fail("unknown pkg-config flag '{s}'", .{arg}); - } - } - - try zig_cflags.shrinkToLen(b.allocator); - try zig_libs.shrinkToLen(b.allocator); - - return .{ - .cflags = zig_cflags.toOwnedSliceAssert(), - .libs = zig_libs.toOwnedSliceAssert(), - }; -} - pub fn setVerboseLink(compile: *Compile, value: bool) void { compile.verbose_link = value; } @@ -974,863 +851,6 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking return path; } -fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { - const step = &compile.step; - const b = step.owner; - const arena = b.allocator; - - var zig_args = std.array_list.Managed([]const u8).init(arena); - defer zig_args.deinit(); - - try zig_args.append(b.graph.zig_exe); - - const cmd = switch (compile.kind) { - .lib => "build-lib", - .exe => "build-exe", - .obj => "build-obj", - .@"test" => "test", - .test_obj => "test-obj", - }; - try zig_args.append(cmd); - - if (b.reference_trace) |some| { - try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); - } - try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); - - try addFlag(&zig_args, "llvm", compile.use_llvm); - try addFlag(&zig_args, "lld", compile.use_lld); - try addFlag(&zig_args, "new-linker", compile.use_new_linker); - - if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { - try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); - } - - switch (compile.entry) { - .default => {}, - .disabled => try zig_args.append("-fno-entry"), - .enabled => try zig_args.append("-fentry"), - .symbol_name => |entry_name| { - try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); - }, - } - - { - var symbol_it = compile.force_undefined_symbols.keyIterator(); - while (symbol_it.next()) |symbol_name| { - try zig_args.append("--force_undefined"); - try zig_args.append(symbol_name.*); - } - } - - if (compile.stack_size) |stack_size| { - try zig_args.append("--stack"); - try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); - } - - if (fuzz) { - try zig_args.append("-ffuzz"); - } - - { - // Stores system libraries that have already been seen for at least one - // module, along with any arguments that need to be passed to the - // compiler for each module individually. - var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; - var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; - - var prev_has_cflags = false; - var prev_has_rcflags = false; - var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; - var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; - // Track the number of positional arguments so that a nice error can be - // emitted if there is nothing to link. - var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); - - // Fully recursive iteration including dynamic libraries to detect - // libc and libc++ linkage. - for (compile.getCompileDependencies(true)) |some_compile| { - for (some_compile.root_module.getGraph().modules) |mod| { - if (mod.link_libc == true) compile.is_linking_libc = true; - if (mod.link_libcpp == true) compile.is_linking_libcpp = true; - } - } - - var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); - - // For this loop, don't chase dynamic libraries because their link - // objects are already linked. - for (compile.getCompileDependencies(false)) |dep_compile| { - for (dep_compile.root_module.getGraph().modules) |mod| { - // While walking transitive dependencies, if a given link object is - // already included in a library, it should not redundantly be - // placed on the linker line of the dependee. - const my_responsibility = dep_compile == compile; - const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); - - // Inherit dependencies on darwin frameworks. - if (!already_linked) { - for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { - try frameworks.put(arena, name, info); - } - } - - // Inherit dependencies on system libraries and static libraries. - for (mod.link_objects.items) |link_object| { - switch (link_object) { - .static_path => |static_path| { - if (my_responsibility) { - try zig_args.append(static_path.getPath2(mod.owner, step)); - total_linker_objects += 1; - } - }, - .system_lib => |system_lib| { - const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); - if (system_lib_gop.found_existing) { - try zig_args.appendSlice(system_lib_gop.value_ptr.*); - continue; - } else { - system_lib_gop.value_ptr.* = &.{}; - } - - if (already_linked) - continue; - - if ((system_lib.search_strategy != prev_search_strategy or - system_lib.preferred_link_mode != prev_preferred_link_mode) and - compile.linkage != .static) - { - switch (system_lib.search_strategy) { - .no_fallback => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_dylibs_only"), - .static => try zig_args.append("-search_static_only"), - }, - .paths_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_paths_first"), - .static => try zig_args.append("-search_paths_first_static"), - }, - .mode_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_dylibs_first"), - .static => try zig_args.append("-search_static_first"), - }, - } - prev_search_strategy = system_lib.search_strategy; - prev_preferred_link_mode = system_lib.preferred_link_mode; - } - - const prefix: []const u8 = prefix: { - if (system_lib.needed) break :prefix "-needed-l"; - if (system_lib.weak) break :prefix "-weak-l"; - break :prefix "-l"; - }; - switch (system_lib.use_pkg_config) { - .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), - .yes, .force => { - if (runPkgConfig(&compile.step, system_lib.name)) |result| { - try zig_args.appendSlice(result.cflags); - try zig_args.appendSlice(result.libs); - try seen_system_libs.put(arena, system_lib.name, result.cflags); - } else |err| switch (err) { - error.PkgConfigInvalidOutput, - error.PkgConfigCrashed, - error.PkgConfigFailed, - error.PkgConfigNotInstalled, - error.PackageNotFound, - => switch (system_lib.use_pkg_config) { - .yes => { - // pkg-config failed, so fall back to linking the library - // by name directly. - try zig_args.append(b.fmt("{s}{s}", .{ - prefix, - system_lib.name, - })); - }, - .force => { - panic("pkg-config failed for library {s}", .{system_lib.name}); - }, - .no => unreachable, - }, - - else => |e| return e, - } - }, - } - }, - .other_step => |other| { - switch (other.kind) { - .exe => return step.fail("cannot link with an executable build artifact", .{}), - .@"test" => return step.fail("cannot link with a test", .{}), - .obj, .test_obj => { - const included_in_lib_or_obj = !my_responsibility and - (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); - if (!already_linked and !included_in_lib_or_obj) { - try zig_args.append(other.getEmittedBin().getPath2(b, step)); - total_linker_objects += 1; - } - }, - .lib => l: { - const other_produces_implib = other.producesImplib(); - const other_is_static = other_produces_implib or other.isStaticLibrary(); - - if (compile.isStaticLibrary() and other_is_static) { - // Avoid putting a static library inside a static library. - break :l; - } - - // For DLLs, we must link against the implib. - // For everything else, we directly link - // against the library file. - const full_path_lib = if (other_produces_implib) - try other.getGeneratedFilePath("generated_implib", &compile.step) - else - try other.getGeneratedFilePath("generated_bin", &compile.step); - - try zig_args.append(full_path_lib); - total_linker_objects += 1; - - if (other.linkage == .dynamic and - compile.rootModuleTarget().os.tag != .windows) - { - if (fs.path.dirname(full_path_lib)) |dirname| { - try zig_args.append("-rpath"); - try zig_args.append(dirname); - } - } - }, - } - }, - .assembly_file => |asm_file| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags) { - try zig_args.append("-cflags"); - try zig_args.append("--"); - prev_has_cflags = false; - } - try zig_args.append(asm_file.getPath2(mod.owner, step)); - total_linker_objects += 1; - }, - - .c_source_file => |c_source_file| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags or c_source_file.flags.len != 0) { - try zig_args.append("-cflags"); - for (c_source_file.flags) |arg| { - try zig_args.append(arg); - } - try zig_args.append("--"); - } - prev_has_cflags = (c_source_file.flags.len != 0); - - if (c_source_file.language) |lang| { - try zig_args.append("-x"); - try zig_args.append(lang.internalIdentifier()); - } - - try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); - - if (c_source_file.language != null) { - try zig_args.append("-x"); - try zig_args.append("none"); - } - total_linker_objects += 1; - }, - - .c_source_files => |c_source_files| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags or c_source_files.flags.len != 0) { - try zig_args.append("-cflags"); - for (c_source_files.flags) |arg| { - try zig_args.append(arg); - } - try zig_args.append("--"); - } - prev_has_cflags = (c_source_files.flags.len != 0); - - if (c_source_files.language) |lang| { - try zig_args.append("-x"); - try zig_args.append(lang.internalIdentifier()); - } - - const root_path = c_source_files.root.getPath2(mod.owner, step); - for (c_source_files.files) |file| { - try zig_args.append(b.pathJoin(&.{ root_path, file })); - } - - if (c_source_files.language != null) { - try zig_args.append("-x"); - try zig_args.append("none"); - } - - total_linker_objects += c_source_files.files.len; - }, - - .win32_resource_file => |rc_source_file| l: { - if (!my_responsibility) break :l; - - if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { - if (prev_has_rcflags) { - try zig_args.append("-rcflags"); - try zig_args.append("--"); - prev_has_rcflags = false; - } - } else { - try zig_args.append("-rcflags"); - for (rc_source_file.flags) |arg| { - try zig_args.append(arg); - } - for (rc_source_file.include_paths) |include_path| { - try zig_args.append("/I"); - try zig_args.append(include_path.getPath2(mod.owner, step)); - } - try zig_args.append("--"); - prev_has_rcflags = true; - } - try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); - total_linker_objects += 1; - }, - } - } - - // We need to emit the --mod argument here so that the above link objects - // have the correct parent module, but only if the module is part of - // this compilation. - if (!my_responsibility) continue; - if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { - const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; - try mod.appendZigProcessFlags(&zig_args, step); - - // --dep arguments - try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); - for (mod.import_table.keys(), mod.import_table.values()) |name, import| { - const import_index = cli_named_modules.modules.getIndex(import).?; - const import_cli_name = cli_named_modules.names.keys()[import_index]; - zig_args.appendAssumeCapacity("--dep"); - if (std.mem.eql(u8, import_cli_name, name)) { - zig_args.appendAssumeCapacity(import_cli_name); - } else { - zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); - } - } - - // When the CLI sees a -M argument, it determines whether it - // implies the existence of a Zig compilation unit based on - // whether there is a root source file. If there is no root - // source file, then this is not a zig compilation unit - it is - // perhaps a set of linker objects, or C source files instead. - // Linker objects are added to the CLI globally, while C source - // files must have a module parent. - if (mod.root_source_file) |lp| { - const src = lp.getPath2(mod.owner, step); - try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); - } else if (moduleNeedsCliArg(mod)) { - try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); - } - } - } - } - - if (total_linker_objects == 0) { - return step.fail("the linker needs one or more objects to link", .{}); - } - - for (frameworks.keys(), frameworks.values()) |name, info| { - if (info.needed) { - try zig_args.append("-needed_framework"); - } else if (info.weak) { - try zig_args.append("-weak_framework"); - } else { - try zig_args.append("-framework"); - } - try zig_args.append(name); - } - - if (compile.is_linking_libcpp) { - try zig_args.append("-lc++"); - } - - if (compile.is_linking_libc) { - try zig_args.append("-lc"); - } - } - - if (compile.win32_manifest) |manifest_file| { - try zig_args.append(manifest_file.getPath2(b, step)); - } - - if (compile.win32_module_definition) |module_file| { - try zig_args.append(module_file.getPath2(b, step)); - } - - if (compile.image_base) |image_base| { - try zig_args.append("--image-base"); - try zig_args.append(b.fmt("0x{x}", .{image_base})); - } - - for (compile.filters) |filter| { - try zig_args.append("--test-filter"); - try zig_args.append(filter); - } - - if (compile.test_runner) |test_runner| { - try zig_args.append("--test-runner"); - try zig_args.append(test_runner.path.getPath2(b, step)); - } - - for (b.debug_log_scopes) |log_scope| { - try zig_args.append("--debug-log"); - try zig_args.append(log_scope); - } - - if (b.debug_compile_errors) { - try zig_args.append("--debug-compile-errors"); - } - - if (b.debug_incremental) { - try zig_args.append("--debug-incremental"); - } - - if (b.verbose_air) try zig_args.append("--verbose-air"); - if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); - if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); - if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); - if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); - if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); - if (b.graph.time_report) try zig_args.append("--time-report"); - - if (compile.generated_asm != null) try zig_args.append("-femit-asm"); - if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); - if (compile.generated_docs != null) try zig_args.append("-femit-docs"); - if (compile.generated_implib != null) try zig_args.append("-femit-implib"); - if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); - if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); - if (compile.generated_h != null) try zig_args.append("-femit-h"); - - try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); - - switch (compile.compress_debug_sections) { - .none => {}, - .zlib => try zig_args.append("--compress-debug-sections=zlib"), - .zstd => try zig_args.append("--compress-debug-sections=zstd"), - } - - if (compile.link_eh_frame_hdr) { - try zig_args.append("--eh-frame-hdr"); - } - if (compile.link_emit_relocs) { - try zig_args.append("--emit-relocs"); - } - if (compile.link_function_sections) { - try zig_args.append("-ffunction-sections"); - } - if (compile.link_data_sections) { - try zig_args.append("-fdata-sections"); - } - if (compile.link_gc_sections) |x| { - try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); - } - if (!compile.linker_dynamicbase) { - try zig_args.append("--no-dynamicbase"); - } - if (compile.linker_allow_shlib_undefined) |x| { - try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); - } - if (compile.link_z_notext) { - try zig_args.append("-z"); - try zig_args.append("notext"); - } - if (!compile.link_z_relro) { - try zig_args.append("-z"); - try zig_args.append("norelro"); - } - if (compile.link_z_lazy) { - try zig_args.append("-z"); - try zig_args.append("lazy"); - } - if (compile.link_z_common_page_size) |size| { - try zig_args.append("-z"); - try zig_args.append(b.fmt("common-page-size={d}", .{size})); - } - if (compile.link_z_max_page_size) |size| { - try zig_args.append("-z"); - try zig_args.append(b.fmt("max-page-size={d}", .{size})); - } - if (compile.link_z_defs) { - try zig_args.append("-z"); - try zig_args.append("defs"); - } - - if (compile.libc_file) |libc_file| { - try zig_args.append("--libc"); - try zig_args.append(libc_file.getPath2(b, step)); - } else if (b.libc_file) |libc_file| { - try zig_args.append("--libc"); - try zig_args.append(libc_file); - } - - try zig_args.append("--cache-dir"); - try zig_args.append(b.cache_root.path orelse "."); - - try zig_args.append("--global-cache-dir"); - try zig_args.append(b.graph.global_cache_root.path orelse "."); - - if (b.graph.debug_compiler_runtime_libs) |mode| - try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); - - try zig_args.append("--name"); - try zig_args.append(compile.name); - - if (compile.linkage) |some| switch (some) { - .dynamic => try zig_args.append("-dynamic"), - .static => try zig_args.append("-static"), - }; - if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { - if (compile.version) |version| { - try zig_args.append("--version"); - try zig_args.append(b.fmt("{f}", .{version})); - } - - if (compile.rootModuleTarget().os.tag.isDarwin()) { - const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ - compile.rootModuleTarget().libPrefix(), - compile.name, - compile.rootModuleTarget().dynamicLibSuffix(), - }); - try zig_args.append("-install_name"); - try zig_args.append(install_name); - } - } - - if (compile.entitlements) |entitlements| { - try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); - } - if (compile.pagezero_size) |pagezero_size| { - const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); - try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); - } - if (compile.headerpad_size) |headerpad_size| { - const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); - try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); - } - if (compile.headerpad_max_install_names) { - try zig_args.append("-headerpad_max_install_names"); - } - if (compile.dead_strip_dylibs) { - try zig_args.append("-dead_strip_dylibs"); - } - if (compile.force_load_objc) { - try zig_args.append("-ObjC"); - } - if (compile.discard_local_symbols) { - try zig_args.append("--discard-all"); - } - - try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); - try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); - try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); - if (compile.rdynamic) { - try zig_args.append("-rdynamic"); - } - if (compile.import_memory) { - try zig_args.append("--import-memory"); - } - if (compile.export_memory) { - try zig_args.append("--export-memory"); - } - if (compile.import_symbols) { - try zig_args.append("--import-symbols"); - } - if (compile.import_table) { - try zig_args.append("--import-table"); - } - if (compile.export_table) { - try zig_args.append("--export-table"); - } - if (compile.initial_memory) |initial_memory| { - try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); - } - if (compile.max_memory) |max_memory| { - try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); - } - if (compile.shared_memory) { - try zig_args.append("--shared-memory"); - } - if (compile.global_base) |global_base| { - try zig_args.append(b.fmt("--global-base={d}", .{global_base})); - } - - if (compile.wasi_exec_model) |model| { - try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); - } - if (compile.linker_script) |linker_script| { - try zig_args.append("--script"); - try zig_args.append(linker_script.getPath2(b, step)); - } - - if (compile.version_script) |version_script| { - try zig_args.append("--version-script"); - try zig_args.append(version_script.getPath2(b, step)); - } - if (compile.linker_allow_undefined_version) |x| { - try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); - } - - if (compile.linker_enable_new_dtags) |enabled| { - try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); - } - - if (compile.kind == .@"test") { - if (compile.exec_cmd_args) |exec_cmd_args| { - for (exec_cmd_args) |cmd_arg| { - if (cmd_arg) |arg| { - try zig_args.append("--test-cmd"); - try zig_args.append(arg); - } else { - try zig_args.append("--test-cmd-bin"); - } - } - } - } - - if (b.sysroot) |sysroot| { - try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); - } - - // -I and -L arguments that appear after the last --mod argument apply to all modules. - const cwd: Io.Dir = .cwd(); - const io = b.graph.io; - - for (b.search_prefixes.items) |search_prefix| { - var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { - return step.fail("unable to open prefix directory '{s}': {s}", .{ - search_prefix, @errorName(err), - }); - }; - defer prefix_dir.close(io); - - // Avoid passing -L and -I flags for nonexistent directories. - // This prevents a warning, that should probably be upgraded to an error in Zig's - // CLI parsing code, when the linker sees an -L directory that does not exist. - - if (prefix_dir.access(io, "lib", .{})) |_| { - try zig_args.appendSlice(&.{ - "-L", b.pathJoin(&.{ search_prefix, "lib" }), - }); - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ - search_prefix, @errorName(e), - }), - } - - if (prefix_dir.access(io, "include", .{})) |_| { - try zig_args.appendSlice(&.{ - "-I", b.pathJoin(&.{ search_prefix, "include" }), - }); - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ - search_prefix, @errorName(e), - }), - } - } - - if (compile.rc_includes != .any) { - try zig_args.append("-rcincludes"); - try zig_args.append(@tagName(compile.rc_includes)); - } - - try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); - - if (compile.build_id orelse b.build_id) |build_id| { - try zig_args.append(switch (build_id) { - .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), - .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), - }); - } - - const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| - dir.getPath2(b, step) - else if (b.graph.zig_lib_directory.path) |_| - b.fmt("{f}", .{b.graph.zig_lib_directory}) - else - null; - - if (opt_zig_lib_dir) |zig_lib_dir| { - try zig_args.append("--zig-lib-dir"); - try zig_args.append(zig_lib_dir); - } - - try addFlag(&zig_args, "PIE", compile.pie); - - if (compile.lto) |lto| { - try zig_args.append(switch (lto) { - .full => "-flto=full", - .thin => "-flto=thin", - .none => "-fno-lto", - }); - } - - try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); - - if (compile.subsystem) |subsystem| { - try zig_args.append("--subsystem"); - try zig_args.append(@tagName(subsystem)); - } - - if (compile.mingw_unicode_entry_point) { - try zig_args.append("-municode"); - } - - if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ - "--error-limit", b.fmt("{d}", .{err_limit}), - }); - - try addFlag(&zig_args, "incremental", b.graph.incremental); - - try zig_args.append("--listen=-"); - - // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux - // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and - // pass that to zig, e.g. via 'zig build-lib @args.rsp' - // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html - var args_length: usize = 0; - for (zig_args.items) |arg| { - args_length += arg.len + 1; // +1 to account for null terminator - } - if (args_length >= 30 * 1024) { - try b.cache_root.handle.createDirPath(io, "args"); - - const args_to_escape = zig_args.items[2..]; - var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); - arg_blk: for (args_to_escape) |arg| { - for (arg, 0..) |c, arg_idx| { - if (c == '\\' or c == '"') { - // Slow path for arguments that need to be escaped. We'll need to allocate and copy - var escaped: std.ArrayList(u8) = .empty; - try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1); - try escaped.appendSlice(arena, arg[0..arg_idx]); - for (arg[arg_idx..]) |to_escape| { - if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\'); - try escaped.append(arena, to_escape); - } - escaped_args.appendAssumeCapacity(escaped.items); - continue :arg_blk; - } - } - escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument - } - - // Write the args to zig-cache/args/ to avoid conflicts with - // other zig build commands running in parallel. - const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); - const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); - - var args_hash: [Sha256.digest_length]u8 = undefined; - Sha256.hash(args, &args_hash, .{}); - var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; - _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); - - const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; - if (b.cache_root.handle.access(io, args_file, .{})) |_| { - // The args file is already present from a previous run. - } else |err| switch (err) { - error.FileNotFound => { - var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ - .replace = false, - .make_path = true, - }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ - b.cache_root, args_file, e, - }); - defer af.deinit(io); - - af.file.writeStreamingAll(io, args) catch |e| { - return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ - b.cache_root, args_file, e, - }); - }; - // Note we can't clean up this file, not even after build - // success, because that might interfere with another build - // process that needs the same file. - af.link(io) catch |e| switch (e) { - error.PathAlreadyExists => { - // The args file was created by another concurrent build process. - }, - else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ - b.cache_root, args_file, other_err, - }), - }; - }, - else => |other_err| return other_err, - } - - const resolved_args_file = try mem.concat(arena, u8, &.{ - "@", - try b.cache_root.join(arena, &.{args_file}), - }); - - zig_args.shrinkRetainingCapacity(2); - try zig_args.append(resolved_args_file); - } - - return try zig_args.toOwnedSlice(); -} - -fn make(step: *Step, options: Step.MakeOptions) !void { - const b = step.owner; - const compile: *Compile = @fieldParentPtr("step", step); - - const zig_args = try getZigArgs(compile, false); - - const maybe_output_dir = step.evalZigProcess( - zig_args, - options.progress_node, - (b.graph.incremental == true) and (options.watch or options.web_server != null), - options.web_server, - options.gpa, - ) catch |err| switch (err) { - error.NeedCompileErrorCheck => { - assert(compile.expect_errors != null); - try checkCompileErrors(compile); - return; - }, - else => |e| return e, - }; - - // Update generated files - if (maybe_output_dir) |output_dir| { - if (compile.emit_directory) |lp| { - lp.path = b.fmt("{f}", .{output_dir}); - } - - // zig fmt: off - if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); - if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); - // hack for stage2_x86_64 + coff - if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); - if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); - if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); - if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); - if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); - if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); - if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); - // zig fmt: on - } - - if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and - compile.version != null and compile.generated_bin != null and - std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) - { - try doAtomicSymLinks( - step, - compile.getEmittedBin().getPath2(b, step), - compile.major_only_filename.?, - compile.name_only_filename.?, - ); - } -} fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifact) []const u8 { const arena = c.step.owner.graph.arena; const name = ea.cacheName(arena, .{ @@ -1847,100 +867,6 @@ fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifa return out_dir.joinString(arena, name) catch @panic("OOM"); } -pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { - c.step.result_error_msgs.clearRetainingCapacity(); - c.step.result_stderr = ""; - - c.step.result_error_bundle.deinit(gpa); - c.step.result_error_bundle = std.zig.ErrorBundle.empty; - - if (c.step.result_failed_command) |cmd| { - gpa.free(cmd); - c.step.result_failed_command = null; - } - - const zig_args = try getZigArgs(c, true); - const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); - return maybe_output_bin_path.?; -} - -pub fn doAtomicSymLinks( - step: *Step, - output_path: []const u8, - filename_major_only: []const u8, - filename_name_only: []const u8, -) !void { - const b = step.owner; - const io = b.graph.io; - const out_dir = fs.path.dirname(output_path) orelse "."; - const out_basename = fs.path.basename(output_path); - // sym link for libfoo.so.1 to libfoo.so.1.2.3 - const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); - const cwd: Io.Dir = .cwd(); - cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { - return step.fail("unable to symlink {s} -> {s}: {s}", .{ - major_only_path, out_basename, @errorName(err), - }); - }; - // sym link for libfoo.so to libfoo.so.1 - const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); - cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { - return step.fail("Unable to symlink {s} -> {s}: {s}", .{ - name_only_path, filename_major_only, @errorName(err), - }); - }; -} - -fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { - const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); - var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); - errdefer list.deinit(); - var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); - while (line_it.next()) |line| { - if (mem.trim(u8, line, " \t").len == 0) continue; - var tok_it = mem.tokenizeAny(u8, line, " \t"); - try list.append(PkgConfigPkg{ - .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, - .desc = tok_it.rest(), - }); - } - return list.toOwnedSlice(); -} - -fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { - if (b.pkg_config_pkg_list) |res| { - return res; - } - var code: u8 = undefined; - if (execPkgConfigList(b, &code)) |list| { - b.pkg_config_pkg_list = list; - return list; - } else |err| { - const result = switch (err) { - error.ProcessTerminated => error.PkgConfigCrashed, - error.ExecNotSupported => error.PkgConfigFailed, - error.ExitCodeFailure => error.PkgConfigFailed, - error.FileNotFound => error.PkgConfigNotInstalled, - error.InvalidName => error.PkgConfigNotInstalled, - error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, - else => return err, - }; - b.pkg_config_pkg_list = result; - return result; - } -} - -fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { - const cond = opt orelse return; - try args.ensureUnusedCapacity(1); - if (cond) { - args.appendAssumeCapacity("-f" ++ name); - } else { - args.appendAssumeCapacity("-fno-" ++ name); - } -} - fn checkCompileErrors(compile: *Compile) !void { // Clear this field so that it does not get printed by the build runner. const actual_eb = compile.step.result_error_bundle; diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index 23c977296600c7eef619f49d56a12839159db141..2406250b4764d59fed65b8186e820f32b7078ecb 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -47,7 +47,7 @@ max_bytes: usize, include_path: []const u8, include_guard_override: ?[]const u8, -pub const base_id: Step.Id = .config_header; +pub const base_tag: Step.Tag = .config_header; pub const Options = struct { style: Style = .blank, @@ -88,7 +88,7 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { config_header.* = .{ .step = .init(.{ - .id = base_id, + .tag = base_tag, .name = name, .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Fail.zig b/lib/std/Build/Step/Fail.zig index 9236c2ac7b6176ab9a7fc15f0afee83fa2107c42..50c7ed2789fb2b81434716cd4687c38eb0cb5a3c 100644 --- a/lib/std/Build/Step/Fail.zig +++ b/lib/std/Build/Step/Fail.zig @@ -6,14 +6,14 @@ const Fail = @This(); step: Step, error_msg: []const u8, -pub const base_id: Step.Id = .fail; +pub const base_tag: Step.Tag = .fail; pub fn create(owner: *std.Build, error_msg: []const u8) *Fail { const fail = owner.allocator.create(Fail) catch @panic("OOM"); fail.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "fail", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Fmt.zig b/lib/std/Build/Step/Fmt.zig index 2da07b63bb422176c8674ff6ccf71a0e00749c4e..bca5385a541ca992e4429e2e64b945ddcf8dfc4e 100644 --- a/lib/std/Build/Step/Fmt.zig +++ b/lib/std/Build/Step/Fmt.zig @@ -10,7 +10,7 @@ paths: []const []const u8, exclude_paths: []const []const u8, check: bool, -pub const base_id: Step.Id = .fmt; +pub const base_tag: Step.Tag = .fmt; pub const Options = struct { paths: []const []const u8 = &.{}, @@ -24,7 +24,7 @@ pub fn create(owner: *std.Build, options: Options) *Fmt { const name = if (options.check) "zig fmt --check" else "zig fmt"; fmt.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = name, .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig index aafd18f01c6354d7d87632c529c49a908d4152c0..f4e9b1d8185f665a08e90983514593b43357bdb8 100644 --- a/lib/std/Build/Step/InstallArtifact.zig +++ b/lib/std/Build/Step/InstallArtifact.zig @@ -33,7 +33,7 @@ const DylibSymlinkInfo = struct { name_only_filename: []const u8, }; -pub const base_id: Step.Id = .install_artifact; +pub const base_tag: Step.Tag = .install_artifact; pub const Options = struct { /// Which installation directory to put the main output file into. @@ -69,10 +69,9 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins }; install_artifact.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = owner.fmt("install {s}", .{artifact.name}), .owner = owner, - .makeFn = make, }), .dest_dir = dest_dir, .pdb_dir = switch (options.pdb_dir) { @@ -126,99 +125,3 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins return install_artifact; } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); - const b = step.owner; - const io = b.graph.io; - - var all_cached = true; - - if (install_artifact.dest_dir) |dest_dir| { - const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); - const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); - all_cached = all_cached and p == .fresh; - - if (install_artifact.dylib_symlinks) |dls| { - try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); - } - - install_artifact.artifact.installed_path = full_dest_path; - } - - if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { - const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.implib_dir) |implib_dir| { - const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.pdb_dir) |pdb_dir| { - const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.h_dir) |h_dir| { - if (install_artifact.emitted_h) |emitted_h| { - const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); - const p = try step.installFile(emitted_h, full_h_path); - all_cached = all_cached and p == .fresh; - } - - for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { - .file => |file| { - const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); - const p = try step.installFile(file.source, full_h_path); - all_cached = all_cached and p == .fresh; - }, - .directory => |dir| { - const src_dir_path = dir.source.getPath3(b, step); - const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); - - var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {s}", .{ - src_dir_path, @errorName(err), - }); - }; - defer src_dir.close(io); - - var it = try src_dir.walk(b.allocator); - next_entry: while (try it.next(io)) |entry| { - for (dir.options.exclude_extensions) |ext| { - if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; - } - if (dir.options.include_extensions) |incs| { - for (incs) |inc| { - if (std.mem.endsWith(u8, entry.path, inc)) break; - } else { - continue :next_entry; - } - } - - const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); - switch (entry.kind) { - .directory => { - try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); - const p = try step.installDir(full_dest_path); - all_cached = all_cached and p == .existed; - }, - .file => { - const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); - all_cached = all_cached and p == .fresh; - }, - else => continue, - } - } - }, - }; - } - - step.result_cached = all_cached; -} diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index d03e72ca75f45a314fc911affc150b1488623f85..f755a28662b24f60a08157bef23a2a2b3b40c5e1 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -8,7 +8,7 @@ const InstallDir = @This(); step: Step, options: Options, -pub const base_id: Step.Id = .install_dir; +pub const base_tag: Step.Tag = .install_dir; pub const Options = struct { source_dir: LazyPath, @@ -44,7 +44,7 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir { const install_dir = owner.allocator.create(InstallDir) catch @panic("OOM"); install_dir.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = owner.fmt("install {s}/", .{options.source_dir.getDisplayName()}), .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index 10adb4754db39cf6b8e93267257c07bdfc760815..a73f126d16087893b9787bff6c372ba7f15cdedc 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -5,7 +5,7 @@ const InstallDir = std.Build.InstallDir; const InstallFile = @This(); const assert = std.debug.assert; -pub const base_id: Step.Id = .install_file; +pub const base_tag: Step.Tag = .install_file; step: Step, source: LazyPath, @@ -22,7 +22,7 @@ pub fn create( const install_file = owner.allocator.create(InstallFile) catch @panic("OOM"); install_file.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = owner.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index ea0714adf9b5517dfd206c9001749636d4e9a438..5ab21c3bcc835254226d27e66c5049be1b972ad3 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -10,7 +10,7 @@ const elf = std.elf; const fs = std.fs; const sort = std.sort; -pub const base_id: Step.Id = .objcopy; +pub const base_tag: Step.Tag = .objcopy; pub const RawFormat = enum { bin, @@ -111,7 +111,7 @@ pub fn create( const objcopy = owner.allocator.create(ObjCopy) catch @panic("OOM"); objcopy.* = ObjCopy{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = owner.fmt("objcopy {s}", .{input_file.getDisplayName()}), .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 34073264e888553f1ff59b74959ede7a675b0241..21df380b18ed05dd0ad181116370b4c38b533c84 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -8,7 +8,7 @@ const Step = std.Build.Step; const GeneratedFile = std.Build.GeneratedFile; const LazyPath = std.Build.LazyPath; -pub const base_id: Step.Id = .options; +pub const base_tag: Step.Tag = .options; step: Step, generated_file: GeneratedFile, @@ -21,7 +21,7 @@ pub fn create(owner: *std.Build) *Options { const options = owner.allocator.create(Options) catch @panic("OOM"); options.* = .{ .step = .init(.{ - .id = base_id, + .tag = base_tag, .name = "options", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 221a8b686027300fb0b6f645ca033787f479aa03..b7dd256d8ed34189e9f59fbcf1b41b1bdcb4eb15 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -12,7 +12,7 @@ const EnvMap = std.process.Environ.Map; const assert = std.debug.assert; const Path = std.Build.Cache.Path; -pub const base_id: Step.Id = .run; +pub const base_tag: Step.Tag = .run; step: Step, @@ -88,12 +88,6 @@ dep_output_file: ?*Output, has_side_effects: bool, -/// If this is a Zig unit test binary, this tracks the names of the unit -/// tests that are also fuzz tests. Indexes cannot be used as they may -/// change between reruns. -fuzz_tests: std.ArrayList([]const u8), -cached_test_metadata: ?CachedTestMetadata = null, - /// Populated during the fuzz phase if this run step corresponds to a unit test /// executable that contains fuzz tests. rebuilt_executable: ?Path, @@ -209,10 +203,9 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { const run = owner.allocator.create(Run) catch @panic("OOM"); run.* = .{ .step = .init(.{ - .id = base_id, + .tag = base_tag, .name = name, .owner = owner, - .makeFn = make, }), .argv = .empty, .cwd = null, @@ -229,7 +222,6 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .captured_stderr = null, .dep_output_file = null, .has_side_effects = false, - .fuzz_tests = .empty, .rebuilt_executable = null, .producer = null, }; @@ -702,2107 +694,3 @@ pub fn addFileInput(self: *Run, file_input: std.Build.LazyPath) void { file_input.addStepDependencies(&self.step); self.file_inputs.append(self.step.owner.allocator, file_input.dupe(self.step.owner)) catch @panic("OOM"); } - -/// Returns whether the Run step has side effects *other than* updating the output arguments. -fn hasSideEffects(run: Run) bool { - if (run.has_side_effects) return true; - return switch (run.stdio) { - .infer_from_args => !run.hasAnyOutputArgs(), - .inherit => true, - .check => false, - .zig_test => false, - }; -} - -fn hasAnyOutputArgs(run: Run) bool { - if (run.captured_stdout != null) return true; - if (run.captured_stderr != null) return true; - for (run.argv.items) |arg| switch (arg) { - .output_file, .output_directory => return true, - else => continue, - }; - return false; -} - -fn checksContainStdout(checks: []const StdIo.Check) bool { - for (checks) |check| switch (check) { - .expect_stderr_exact, - .expect_stderr_match, - .expect_term, - => continue, - - .expect_stdout_exact, - .expect_stdout_match, - => return true, - }; - return false; -} - -fn checksContainStderr(checks: []const StdIo.Check) bool { - for (checks) |check| switch (check) { - .expect_stdout_exact, - .expect_stdout_match, - .expect_term, - => continue, - - .expect_stderr_exact, - .expect_stderr_match, - => return true, - }; - return false; -} - -/// If `path` is cwd-relative, make it relative to the cwd of the child instead. -/// -/// Whenever a path is included in the argv of a child, it should be put through this function first -/// to make sure the child doesn't see paths relative to a cwd other than its own. -fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { - const b = run.step.owner; - const graph = b.graph; - const arena = graph.arena; - - const path_str = path.toString(arena) catch @panic("OOM"); - if (Dir.path.isAbsolute(path_str)) { - // Absolute paths don't need changing. - return path_str; - } - const child_cwd_rel: []const u8 = rel: { - const child_lazy_cwd = run.cwd orelse break :rel path_str; - const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); - // Convert it from relative to *our* cwd, to relative to the *child's* cwd. - break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); - }; - // Not every path can be made relative, e.g. if the path and the child cwd are on different - // disk designators on Windows. In that case, `relative` will return an absolute path which we can - // just return. - if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel; - - // We're not done yet. In some cases this path must be prefixed with './': - // * On POSIX, the executable name cannot be a single component like 'foo' - // * Some executables might treat a leading '-' like a flag, which we must avoid - // There's no harm in it, so just *always* apply this prefix. - return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); -} - -const IndexedOutput = struct { - index: usize, - tag: @typeInfo(Arg).@"union".tag_type.?, - output: *Output, -}; -fn make(step: *Step, options: Step.MakeOptions) !void { - const b = step.owner; - const io = b.graph.io; - const arena = b.allocator; - const run: *Run = @fieldParentPtr("step", step); - const has_side_effects = run.hasSideEffects(); - - var argv_list = std.array_list.Managed([]const u8).init(arena); - var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); - - var man = b.graph.cache.obtain(); - defer man.deinit(); - - if (run.environ_map) |environ_map| { - for (environ_map.keys(), environ_map.values()) |key, value| { - man.hash.addBytes(key); - man.hash.addBytes(value); - } - } - - man.hash.add(run.color); - man.hash.add(run.disable_zig_progress); - - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(bytes); - man.hash.addBytes(bytes); - }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); - man.hash.addBytes(file.prefix); - _ = try man.addFilePath(file_path, null); - }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); - try argv_list.append(resolved_arg); - man.hash.addBytes(resolved_arg); - }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); - - var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; - - const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { - return step.fail( - "unable to open input file '{f}': {t}", - .{ file_path, err }, - ); - }; - defer file.close(io); - - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); - _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return step.fail( - "failed to read from '{f}': {t}", - .{ file_path, file_reader.err.? }, - ), - error.WriteFailed => return error.OutOfMemory, - }; - - try argv_list.append(result.written()); - man.hash.addBytes(file_plp.prefix); - _ = try man.addFilePath(file_path, null); - }, - .artifact => |pa| { - const artifact = pa.artifact; - - if (artifact.rootModuleTarget().os.tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - run.addPathForDynLibs(artifact); - } - const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; - - try argv_list.append(b.fmt("{s}{s}", .{ - pa.prefix, - run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), - })); - - _ = try man.addFile(file_path, null); - }, - .output_file, .output_directory => |output| { - man.hash.addBytes(output.prefix); - man.hash.addBytes(output.basename); - // Add a placeholder into the argument list because we need the - // manifest hash to be updated with all arguments before the - // object directory is computed. - try output_placeholders.append(.{ - .index = argv_list.items.len, - .tag = arg, - .output = output, - }); - _ = try argv_list.addOne(); - }, - } - } - - switch (run.stdin) { - .bytes => |bytes| { - man.hash.addBytes(bytes); - }, - .lazy_path => |lazy_path| { - const file_path = lazy_path.getPath2(b, step); - _ = try man.addFile(file_path, null); - }, - .none => {}, - } - - if (run.captured_stdout) |captured| { - man.hash.addBytes(captured.output.basename); - man.hash.add(captured.trim_whitespace); - } - - if (run.captured_stderr) |captured| { - man.hash.addBytes(captured.output.basename); - man.hash.add(captured.trim_whitespace); - } - - hashStdIo(&man.hash, run.stdio); - - for (run.file_inputs.items) |lazy_path| { - _ = try man.addFile(lazy_path.getPath2(b, step), null); - } - - if (run.cwd) |cwd| { - const cwd_path = cwd.getPath3(b, step); - _ = man.hash.addBytes(try cwd_path.toString(arena)); - } - - if (!has_side_effects and try step.cacheHitAndWatch(&man)) { - // cache hit, skip running command - const digest = man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); - - step.result_cached = true; - return; - } - - const dep_output_file = run.dep_output_file orelse { - // We already know the final output paths, use them directly. - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); - - const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; - for (output_placeholders.items) |placeholder| { - const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, output_sub_dir_path, @errorName(err), - }); - }; - const arg_output_path = run.convertPathArg(.{ - .root_dir = .cwd(), - .sub_path = placeholder.output.generated_file.getPath(), - }); - argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) - arg_output_path - else - b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); - } - - try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); - if (!has_side_effects) try step.writeManifestAndWatch(&man); - return; - }; - - // We do not know the final output paths yet, use temp paths to run the command. - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - - for (output_placeholders.items) |placeholder| { - const output_components = .{ tmp_dir_path, placeholder.output.basename }; - const output_sub_path = b.pathJoin(&output_components); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, output_sub_dir_path, @errorName(err), - }); - }; - const raw_output_path: Build.Cache.Path = .{ - .root_dir = b.cache_root, - .sub_path = b.pathJoin(&output_components), - }; - placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); - argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ - placeholder.output.prefix, - run.convertPathArg(raw_output_path), - }); - } - - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); - - const dep_file_dir = Dir.cwd(); - const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); - if (has_side_effects) - try man.addDepFile(dep_file_dir, dep_file_basename) - else - try man.addDepFilePost(dep_file_dir, dep_file_basename); - - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - const any_output = output_placeholders.items.len > 0 or - run.captured_stdout != null or run.captured_stderr != null; - - // Rename into place - if (any_output) { - const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; - - b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { - Dir.RenameError.DirNotEmpty => { - b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { - return step.fail("unable to remove dir '{f}'{s}: {t}", .{ - b.cache_root, tmp_dir_path, del_err, - }); - }; - b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { - return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, - }); - }; - }, - else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, - }), - }; - } - - if (!has_side_effects) try step.writeManifestAndWatch(&man); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); -} - -pub fn rerunInFuzzMode( - run: *Run, - fuzz: *std.Build.Fuzz, - prog_node: std.Progress.Node, -) !void { - const step = &run.step; - const b = step.owner; - const io = b.graph.io; - const arena = b.allocator; - var argv_list: std.ArrayList([]const u8) = .empty; - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(arena, bytes); - }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); - }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); - }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); - - var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; - - const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); - defer file.close(io); - - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); - _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - error.WriteFailed => return error.OutOfMemory, - }; - - try argv_list.append(arena, result.written()); - }, - .artifact => |pa| { - const artifact = pa.artifact; - const file_path: []const u8 = p: { - if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); - break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; - }; - try argv_list.append(arena, b.fmt("{s}{s}", .{ - pa.prefix, - run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), - })); - }, - .output_file, .output_directory => unreachable, - } - } - - if (run.step.result_failed_command) |cmd| { - fuzz.gpa.free(cmd); - run.step.result_failed_command = null; - } - - const has_side_effects = false; - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ - .progress_node = prog_node, - .watch = undefined, // not used by `runCommand` - .web_server = null, // only needed for time reports - .unit_test_timeout_ns = null, // don't time out fuzz tests for now - .gpa = fuzz.gpa, - }, .{ - .fuzz = fuzz, - }); -} - -fn populateGeneratedPaths( - arena: std.mem.Allocator, - output_placeholders: []const IndexedOutput, - captured_stdout: ?*CapturedStdIo, - captured_stderr: ?*CapturedStdIo, - cache_root: Build.Cache.Directory, - digest: *const Build.Cache.HexDigest, -) !void { - for (output_placeholders) |placeholder| { - placeholder.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, placeholder.output.basename, - }); - } - - if (captured_stdout) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); - } - - if (captured_stderr) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); - } -} - -fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void { - if (term) |t| switch (t) { - .exited => |code| try w.print("exited with code {d}", .{code}), - .signal => |sig| try w.print("terminated with signal {t}", .{sig}), - .stopped => |sig| try w.print("stopped with signal {t}", .{sig}), - .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}), - } else { - try w.writeAll("exited with any code"); - } -} -fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) { - return .{ .data = term }; -} - -fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { - return if (expected) |e| switch (e) { - .exited => |expected_code| switch (actual) { - .exited => |actual_code| expected_code == actual_code, - else => false, - }, - .signal => |expected_sig| switch (actual) { - .signal => |actual_sig| expected_sig == actual_sig, - else => false, - }, - .stopped => |expected_sig| switch (actual) { - .stopped => |actual_sig| expected_sig == actual_sig, - else => false, - }, - .unknown => |expected_code| switch (actual) { - .unknown => |actual_code| expected_code == actual_code, - else => false, - }, - } else switch (actual) { - .exited => true, - else => false, - }; -} - -const FuzzContext = struct { - fuzz: *std.Build.Fuzz, -}; - -fn runCommand( - run: *Run, - argv: []const []const u8, - has_side_effects: bool, - output_dir_path: []const u8, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !void { - const step = &run.step; - const b = step.owner; - const arena = b.allocator; - const gpa = options.gpa; - const io = b.graph.io; - - const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; - - try step.handleChildProcUnsupported(); - try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); - - const allow_skip = switch (run.stdio) { - .check, .zig_test => run.skip_foreign_checks, - else => false, - }; - - var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); - defer interp_argv.deinit(); - - var environ_map: EnvMap = env: { - const orig = run.environ_map orelse &b.graph.environ_map; - break :env try orig.clone(gpa); - }; - defer environ_map.deinit(); - - const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { - // InvalidExe: cpu arch mismatch - // FileNotFound: can happen with a wrong dynamic linker path - if (err == error.InvalidExe or err == error.FileNotFound) interpret: { - // TODO: learn the target from the binary directly rather than from - // relying on it being a Compile step. This will make this logic - // work even for the edge case that the binary was produced by a - // third party. - const exe = switch (run.argv.items[0]) { - .artifact => |exe| exe.artifact, - else => break :interpret, - }; - switch (exe.kind) { - .exe, .@"test" => {}, - else => break :interpret, - } - - const root_target = exe.rootModuleTarget(); - const need_cross_libc = exe.is_linking_libc and - (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); - const other_target = exe.root_module.resolved_target.?.result; - switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ - .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, - .link_libc = exe.is_linking_libc, - })) { - .native, .rosetta => { - if (allow_skip) return error.MakeSkipped; - break :interpret; - }, - .wine => |bin_name| { - if (b.enable_wine) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); - - // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but - // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. - if (environ_map.get("WINEDEBUG") == null) { - try environ_map.put("WINEDEBUG", "-all"); - } - } else { - return failForeign(run, "-fwine", argv[0], exe); - } - }, - .qemu => |bin_name| { - if (b.enable_qemu) { - try interp_argv.append(bin_name); - - if (need_cross_libc) { - if (b.libc_runtimes_dir) |dir| { - try interp_argv.append("-L"); - try interp_argv.append(b.pathJoin(&.{ - dir, - try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( - b.allocator, - root_target.cpu.arch, - root_target.os.tag, - root_target.abi, - ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( - b.allocator, - root_target.cpu.arch, - root_target.abi, - ) else unreachable, - })); - } else return failForeign(run, "--libc-runtimes", argv[0], exe); - } - - try interp_argv.appendSlice(argv); - } else return failForeign(run, "-fqemu", argv[0], exe); - }, - .darling => |bin_name| { - if (b.enable_darling) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); - } else { - return failForeign(run, "-fdarling", argv[0], exe); - } - }, - .wasmtime => |bin_name| { - if (b.enable_wasmtime) { - try interp_argv.append(bin_name); - try interp_argv.append("--dir=."); - // Wasmtime doeesn't inherit environment variables from the parent process - // by default. '-S inherit-env' was added in Wasmtime version 20. - try interp_argv.append("-Sinherit-env"); - try interp_argv.append(argv[0]); - try interp_argv.appendSlice(argv[1..]); - } else { - return failForeign(run, "-fwasmtime", argv[0], exe); - } - }, - .bad_dl => |foreign_dl| { - if (allow_skip) return error.MakeSkipped; - - const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; - - return step.fail( - \\the host system is unable to execute binaries from the target - \\ because the host dynamic linker is '{s}', - \\ while the target dynamic linker is '{s}'. - \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step - , .{ host_dl, foreign_dl }); - }, - .bad_os_or_cpu => { - if (allow_skip) return error.MakeSkipped; - - const host_name = try b.graph.host.result.zigTriple(b.allocator); - const foreign_name = try root_target.zigTriple(b.allocator); - - return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ - host_name, foreign_name, - }); - }, - } - - if (root_target.os.tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - run.addPathForDynLibs(exe); - } - - gpa.free(step.result_failed_command.?); - step.result_failed_command = null; - try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); - - break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { - if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; - if (e == error.MakeFailed) return error.MakeFailed; // error already reported - return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); - }; - } - if (err == error.MakeFailed) return error.MakeFailed; // error already reported - - return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); - }; - - const generic_result = opt_generic_result orelse { - assert(run.stdio == .zig_test); - // Specific errors have already been reported, and test results are populated. All we need - // to do is report step failure if any test failed. - if (!step.test_results.isSuccess()) return error.MakeFailed; - return; - }; - - assert(fuzz_context == null); - assert(run.stdio != .zig_test); - - // Capture stdout and stderr to GeneratedFile objects. - const Stream = struct { - captured: ?*CapturedStdIo, - bytes: ?[]const u8, - }; - for ([_]Stream{ - .{ - .captured = run.captured_stdout, - .bytes = generic_result.stdout, - }, - .{ - .captured = run.captured_stderr, - .bytes = generic_result.stderr, - }, - }) |stream| { - if (stream.captured) |captured| { - const output_components = .{ output_dir_path, captured.output.basename }; - const output_path = try b.cache_root.join(arena, &output_components); - captured.output.generated_file.path = output_path; - - const sub_path = b.pathJoin(&output_components); - const sub_path_dirname = Dir.path.dirname(sub_path).?; - b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, sub_path_dirname, @errorName(err), - }); - }; - const data = switch (captured.trim_whitespace) { - .none => stream.bytes.?, - .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), - .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), - .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), - }; - b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { - return step.fail("unable to write file '{f}{s}': {s}", .{ - b.cache_root, sub_path, @errorName(err), - }); - }; - } - } - - switch (run.stdio) { - .zig_test => unreachable, - .check => |checks| for (checks.items) |check| switch (check) { - .expect_stderr_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { - return step.fail( - \\========= expected this stderr: ========= - \\{s} - \\========= but found: ==================== - \\{s} - , .{ - expected_bytes, - generic_result.stderr.?, - }); - } - }, - .expect_stderr_match => |match| { - if (mem.find(u8, generic_result.stderr.?, match) == null) { - return step.fail( - \\========= expected to find in stderr: ========= - \\{s} - \\========= but stderr does not contain it: ===== - \\{s} - , .{ - match, - generic_result.stderr.?, - }); - } - }, - .expect_stdout_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { - return step.fail( - \\========= expected this stdout: ========= - \\{s} - \\========= but found: ==================== - \\{s} - , .{ - expected_bytes, - generic_result.stdout.?, - }); - } - }, - .expect_stdout_match => |match| { - if (mem.find(u8, generic_result.stdout.?, match) == null) { - return step.fail( - \\========= expected to find in stdout: ========= - \\{s} - \\========= but stdout does not contain it: ===== - \\{s} - , .{ - match, - generic_result.stdout.?, - }); - } - }, - .expect_term => |expected_term| { - if (!termMatches(expected_term, generic_result.term)) { - return step.fail("process {f} (expected {f})", .{ - fmtTerm(generic_result.term), - fmtTerm(expected_term), - }); - } - }, - }, - else => { - // On failure, report captured stderr like normal standard error output. - const bad_exit = switch (generic_result.term) { - .exited => |code| code != 0, - .signal, .stopped, .unknown => true, - }; - if (bad_exit) { - if (generic_result.stderr) |bytes| { - run.step.result_stderr = bytes; - } - } - - try step.handleChildProcessTerm(generic_result.term); - }, - } -} - -const EvalGenericResult = struct { - term: process.Child.Term, - stdout: ?[]const u8, - stderr: ?[]const u8, -}; - -fn spawnChildAndCollect( - run: *Run, - argv: []const []const u8, - environ_map: *EnvMap, - has_side_effects: bool, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !?EvalGenericResult { - const b = run.step.owner; - const graph = b.graph; - const io = graph.io; - - if (fuzz_context != null) { - assert(!has_side_effects); - assert(run.stdio == .zig_test); - } - - const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; - - // If an error occurs, it's caused by this command: - assert(run.step.result_failed_command == null); - run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ - .child = environ_map, - .parent = &graph.environ_map, - }, argv); - - var spawn_options: process.SpawnOptions = .{ - .argv = argv, - .cwd = child_cwd, - .environ_map = environ_map, - .request_resource_usage_statistics = true, - .stdin = if (run.stdin != .none) s: { - assert(run.stdio != .inherit); - break :s .pipe; - } else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .ignore, - .inherit => .inherit, - .check => .ignore, - .zig_test => .pipe, - }, - .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .ignore, - .inherit => .inherit, - .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, - .zig_test => .pipe, - }, - .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .pipe, - .inherit => .inherit, - .check => .pipe, - .zig_test => .pipe, - }, - }; - - if (run.stdio == .zig_test) { - const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| e, - }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); - try result; - return null; - } else { - const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; - if (!run.disable_zig_progress and !inherit) { - spawn_options.progress_node = options.progress_node; - } - const terminal_mode: Io.Terminal.Mode = if (inherit) m: { - const stderr = try io.lockStderr(&.{}, graph.stderr_mode); - break :m stderr.terminal_mode; - } else .no_color; - defer if (inherit) io.unlockStderr(); - try setColorEnvironmentVariables(run, environ_map, terminal_mode); - - const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalGeneric(run, spawn_options) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| e, - }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); - return try result; - } -} - -fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { - color: switch (run.color) { - .manual => {}, - .enable => { - try environ_map.put("CLICOLOR_FORCE", "1"); - _ = environ_map.swapRemove("NO_COLOR"); - }, - .disable => { - try environ_map.put("NO_COLOR", "1"); - _ = environ_map.swapRemove("CLICOLOR_FORCE"); - }, - .inherit => switch (terminal_mode) { - .no_color, .windows_api => continue :color .disable, - .escape_codes => continue :color .enable, - }, - .auto => { - const capture_stderr = run.captured_stderr != null or switch (run.stdio) { - .check => |checks| checksContainStderr(checks.items), - .infer_from_args, .inherit, .zig_test => false, - }; - if (capture_stderr) { - continue :color .disable; - } else { - continue :color .inherit; - } - }, - } -} - -const StdioPollEnum = enum { stdout, stderr }; - -fn evalZigTest( - run: *Run, - spawn_options: process.SpawnOptions, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !void { - if (fuzz_context != null) { - try evalFuzzTest(run, spawn_options, options, fuzz_context.?); - return; - } - - const step_owner = run.step.owner; - const gpa = step_owner.allocator; - const arena = step_owner.allocator; - const io = step_owner.graph.io; - - // We will update this every time a child runs. - run.step.result_peak_rss = 0; - - var test_results: Step.TestResults = .{ - .test_count = 0, - .skip_count = 0, - .fail_count = 0, - .crash_count = 0, - .timeout_count = 0, - .leak_count = 0, - .log_err_count = 0, - }; - var test_metadata: ?TestMetadata = null; - - while (true) { - var child = try process.spawn(io, spawn_options); - var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; - var multi_reader: Io.File.MultiReader = undefined; - multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); - var child_killed = false; - defer if (!child_killed) { - child.kill(io); - multi_reader.deinit(); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - }; - - switch (try waitZigTest( - run, - &child, - options, - &multi_reader, - &test_metadata, - &test_results, - )) { - .write_failed => |err| { - // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured - // all available stderr to make our error output as useful as possible. - const stderr_fr = multi_reader.fileReader(1); - while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) { - error.ReadFailed => return stderr_fr.err.?, - error.EndOfStream => {}, - } - run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); - - // Clean up everything and wait for the child to exit. - child.stdin.?.close(io); - child.stdin = null; - multi_reader.deinit(); - child_killed = true; - const term = try child.wait(io); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - - // The individual unit test results are irrelevant: the test runner itself broke! - // Fail immediately without populating `s.test_results`. - return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); - }, - .no_poll => |no_poll| { - // This might be a success (we requested exit and the child dutifully closed stdout) or - // a crash of some kind. Either way, the child will terminate by itself -- wait for it. - const stderr_reader = multi_reader.reader(1); - const stderr_owned = try arena.dupe(u8, stderr_reader.buffered()); - - // Clean up everything and wait for the child to exit. - child.stdin.?.close(io); - child.stdin = null; - multi_reader.deinit(); - child_killed = true; - const term = try child.wait(io); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - - if (no_poll.active_test_index) |test_index| { - // A test was running, so this is definitely a crash. Report it against that - // test, and continue to the next test. - test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; - test_results.crash_count += 1; - try run.step.addError("'{s}' {f}{s}{s}", .{ - test_metadata.?.testName(test_index), - fmtTerm(term), - if (stderr_owned.len != 0) " with stderr:\n" else "", - std.mem.trim(u8, stderr_owned, "\n"), - }); - continue; - } - - // Report an error if the child terminated uncleanly or if we were still trying to run more tests. - run.step.result_stderr = stderr_owned; - const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); - if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { - // The individual unit test results are irrelevant: the test runner itself broke! - // Fail immediately without populating `s.test_results`. - return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); - } - - // We're done with all of the tests! Commit the test results and return. - run.step.test_results = test_results; - if (test_metadata) |tm| { - run.cached_test_metadata = tm.toCachedTestMetadata(); - if (options.web_server) |ws| { - if (run.step.owner.graph.time_report) { - ws.updateTimeReportRunTest( - run, - &run.cached_test_metadata.?, - tm.ns_per_test, - ); - } - } - } - return; - }, - .timeout => |timeout| { - const stderr_reader = multi_reader.reader(1); - const stderr = stderr_reader.buffered(); - stderr_reader.tossBuffered(); - if (timeout.active_test_index) |test_index| { - // A test was running. Report the timeout against that test, and continue on to - // the next test. - test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; - test_results.timeout_count += 1; - try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ - test_metadata.?.testName(test_index), - Io.Duration{ .nanoseconds = timeout.ns_elapsed }, - if (stderr.len != 0) " with stderr:\n" else "", - std.mem.trim(u8, stderr, "\n"), - }); - continue; - } - // Just log an error and let the child be killed. - run.step.result_stderr = try arena.dupe(u8, stderr); - // The individual unit test results in `results` are irrelevant: the test runner - // is broken! Fail immediately without populating `s.test_results`. - return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); - }, - } - comptime unreachable; - } -} - -/// Reads stdout of a Zig test process until a termination condition is reached: -/// * A write fails, indicating the child unexpectedly closed stdin -/// * A test (or a response from the test runner) times out -/// * The wait fails, indicating the child closed stdout and stderr -fn waitZigTest( - run: *Run, - child: *process.Child, - options: Step.MakeOptions, - multi_reader: *Io.File.MultiReader, - opt_metadata: *?TestMetadata, - results: *Step.TestResults, -) !union(enum) { - write_failed: anyerror, - no_poll: struct { - active_test_index: ?u32, - ns_elapsed: u64, - }, - timeout: struct { - active_test_index: ?u32, - ns_elapsed: u64, - }, -} { - const gpa = run.step.owner.allocator; - const arena = run.step.owner.allocator; - const io = run.step.owner.graph.io; - - var sub_prog_node: ?std.Progress.Node = null; - defer if (sub_prog_node) |n| n.end(); - - if (opt_metadata.*) |*md| { - // Previous unit test process died or was killed; we're continuing where it left off - requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; - } else { - // Running unit tests normally - run.fuzz_tests.clearRetainingCapacity(); - sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err }; - } - - var active_test_index: ?u32 = null; - - var last_update: Io.Clock.Timestamp = .now(io, .awake); - - // This timeout is used when we're waiting on the test runner itself rather than a user-specified - // test. For instance, if the test runner leaves this much time between us requesting a test to - // start and it acknowledging the test starting, we terminate the child and raise an error. This - // *should* never happen, but could in theory be caused by some very unlucky IB in a test. - const response_timeout: Io.Clock.Duration = t: { - const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); - break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; - }; - const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ - .clock = .awake, - .raw = .fromNanoseconds(ns), - } else null; - - const stdout = multi_reader.reader(0); - const stderr = multi_reader.reader(1); - const Header = std.zig.Server.Message.Header; - - while (true) { - const timeout: Io.Timeout = t: { - const opt_duration = if (active_test_index == null) response_timeout else test_timeout; - const duration = opt_duration orelse break :t .none; - break :t .{ .deadline = last_update.addDuration(duration) }; - }; - - // This block is exited when `stdout` contains enough bytes for a `Header`. - header_ready: { - if (stdout.buffered().len >= @sizeOf(Header)) { - // We already have one, no need to poll! - break :header_ready; - } - - multi_reader.fill(64, timeout) catch |err| switch (err) { - error.Timeout => return .{ .timeout = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - error.EndOfStream => return .{ .no_poll = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - else => |e| return e, - }; - - continue; - } - // There is definitely a header available now -- read it. - const header = stdout.takeStruct(Header, .little) catch unreachable; - - while (stdout.buffered().len < header.bytes_len) { - multi_reader.fill(64, timeout) catch |err| switch (err) { - error.Timeout => return .{ .timeout = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - error.EndOfStream => return .{ .no_poll = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - else => |e| return e, - }; - } - - const body = stdout.take(header.bytes_len) catch unreachable; - var body_r: std.Io.Reader = .fixed(body); - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - }, - .test_metadata => { - // `metadata` would only be populated if we'd already seen a `test_metadata`, but we - // only request it once (and importantly, we don't re-request it if we kill and - // restart the test runner). - assert(opt_metadata.* == null); - - const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable; - results.test_count = tm_hdr.tests_len; - - const names = try arena.alloc(u32, results.test_count); - for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; - - const expected_panic_msgs = try arena.alloc(u32, results.test_count); - for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; - - const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; - - options.progress_node.setEstimatedTotalItems(names.len); - opt_metadata.* = .{ - .string_bytes = try arena.dupe(u8, string_bytes), - .ns_per_test = try arena.alloc(u64, results.test_count), - .names = names, - .expected_panic_msgs = expected_panic_msgs, - .next_index = 0, - .prog_node = options.progress_node, - }; - @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); - - active_test_index = null; - last_update = .now(io, .awake); - - requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err }; - }, - .test_started => { - active_test_index = opt_metadata.*.?.next_index - 1; - last_update = .now(io, .awake); - }, - .test_results => { - const md = &opt_metadata.*.?; - - const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable; - assert(tr_hdr.index == active_test_index); - - switch (tr_hdr.flags.status) { - .pass => {}, - .skip => results.skip_count +|= 1, - .fail => results.fail_count +|= 1, - } - const leak_count = tr_hdr.flags.leak_count; - const log_err_count = tr_hdr.flags.log_err_count; - results.leak_count +|= leak_count; - results.log_err_count +|= log_err_count; - - if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index)); - - if (tr_hdr.flags.status == .fail) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - if (stderr_bytes.len == 0) { - try run.step.addError("'{s}' failed without output", .{name}); - } else { - try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); - } - } else if (leak_count > 0) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); - } else if (log_err_count > 0) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); - } - - active_test_index = null; - - const now: Io.Clock.Timestamp = .now(io, .awake); - md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds); - last_update = now; - - requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; - }, - else => {}, // ignore other messages - } - } -} - -const FuzzTestRunner = struct { - run: *Run, - ctx: FuzzContext, - coverage_id: ?u64, - - instances: []Instance, - /// The indexes of this are layed out such that it is effectively an array - /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr. - batch: Io.Batch, - /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter. - pending_broadcasts: std.ArrayList(u8), - broadcast: std.ArrayList(u8), - broadcast_undelivered: u32, - - const Instance = struct { - child: process.Child, - message: std.ArrayListAligned(u8, .@"4"), - broadcast_written: usize, - stderr: std.ArrayList(u8), - stdin_vec: [1][]u8, - stdout_vec: [1][]u8, - stderr_vec: [1][]u8, - progress_node: std.Progress.Node, - - fn messageHeader(instance: *Instance) InHeader { - assert(instance.message.items.len >= @sizeOf(InHeader)); - const header_ptr: *InHeader = @ptrCast(instance.message.items); - var header = header_ptr.*; - if (std.builtin.Endian.native != .little) { - std.mem.byteSwapAllFields(InHeader, &header); - } - return header; - } - }; - - const PendingBroadcastFooter = struct { - from_id: u32, - body_len: u32, - }; - - const InHeader = std.zig.Server.Message.Header; - const OutHeader = std.zig.Client.Message.Header; - - const stdin_i = 0; - const stdout_i = 1; - const stderr_i = 2; - - fn init( - run: *Run, - ctx: FuzzContext, - progress_node: std.Progress.Node, - spawn_options: process.SpawnOptions, - ) !FuzzTestRunner { - const step_owner = run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - - const n_instances = switch (ctx.fuzz.mode) { - .forever => step_owner.graph.max_jobs orelse @min( - std.Thread.getCpuCount() catch 1, - (std.math.maxInt(u32) - 2) / 3, - ), - .limit => 1, - }; - const instances = try gpa.alloc(Instance, n_instances); - errdefer gpa.free(instances); - const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3); - errdefer gpa.free(batch_storage); - - @memset(instances, .{ - .child = undefined, - .message = .empty, - .broadcast_written = undefined, - .stderr = .empty, - .stdin_vec = undefined, - .stdout_vec = undefined, - .stderr_vec = undefined, - .progress_node = undefined, - }); - for (0.., instances) |id, *instance| { - errdefer for (instances[0..id]) |*spawned| { - spawned.child.kill(io); - spawned.progress_node.end(); - }; - instance.child = try process.spawn(io, spawn_options); - instance.progress_node = progress_node.start("starting fuzzer", 0); - } - - return .{ - .run = run, - .ctx = ctx, - .coverage_id = null, - - .instances = instances, - .batch = .init(batch_storage), - .pending_broadcasts = .empty, - .broadcast = .empty, - .broadcast_undelivered = 0, - }; - } - - fn deinit(f: *FuzzTestRunner) void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - - f.batch.cancel(io); - gpa.free(f.batch.storage); - var total_rss: usize = 0; - for (f.instances) |*instance| { - instance.child.kill(io); - instance.message.deinit(gpa); - instance.stderr.deinit(gpa); - instance.progress_node.end(); - total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; - } - f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); - gpa.free(f.instances); - } - - fn startInstances(f: *FuzzTestRunner) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - - for (0.., f.instances) |id, *instance| { - const id32: u32 = @intCast(id); - (switch (f.ctx.fuzz.mode) { - .forever => sendRunFuzzTestMessage( - io, - instance.child.stdin.?, - f.run.fuzz_tests.items, - .forever, - id32, - ), - .limit => |limit| sendRunFuzzTestMessage( - io, - instance.child.stdin.?, - f.run.fuzz_tests.items, - .iterations, - limit.amount, - ), - }) catch |write_err| { - // The runner unexpectedly closed stdin, which means it crashed during initialization. - // Clean up everything and wait for the child to exit. - instance.child.stdin.?.close(io); - instance.child.stdin = null; - const term = try instance.child.wait(io); - return f.run.step.fail( - "unable to write stdin ({t}); test process unexpectedly {f}", - .{ write_err, fmtTerm(term) }, - ); - }; - - try f.addStdoutRead(id32, @sizeOf(InHeader)); - try f.addStderrRead(id32); - } - } - - fn listen(f: *FuzzTestRunner) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - - while (true) { - try f.batch.awaitConcurrent(io, .none); - while (f.batch.next()) |completion| { - const id = completion.index / 3; - const result = completion.result; - switch (completion.index % 3) { - 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) { - // Avoid calling `instanceEos` until EndOfStream is seen with stderr so - // that all stderr is collected. - error.BrokenPipe => continue, - else => |write_e| return write_e, - }), - 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) { - // Avoid calling `instanceEos` until EndOfStream is seen with stderr so - // that all stderr is collected. - error.EndOfStream => continue, - else => |read_e| return read_e, - }), - 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { - error.EndOfStream => return f.instanceEos(id), - else => |read_e| return read_e, - }), - else => unreachable, - } - } - } - } - - fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - const instance = &f.instances[id]; - - instance.message.items.len += n; - const total_read = instance.message.items.len; - if (total_read < @sizeOf(InHeader)) { - try f.addStdoutRead(id, @sizeOf(InHeader)); - return; - } - - const header = instance.messageHeader(); - const body = instance.message.items[@sizeOf(InHeader)..]; - if (body.len != header.bytes_len) { - try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len); - return; - } - - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - }, - .coverage_id => { - var body_r: Io.Reader = .fixed(body); - f.coverage_id = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable; - - const fuzz = f.ctx.fuzz; - fuzz.queue_mutex.lockUncancelable(io); - defer fuzz.queue_mutex.unlock(io); - try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ - .id = f.coverage_id.?, - .cumulative = .{ - .runs = cumulative_runs, - .unique = cumulative_unique, - .coverage = cumulative_coverage, - }, - .run = f.run, - } }); - fuzz.queue_cond.signal(io); - }, - .fuzz_start_addr => { - var body_r: Io.Reader = .fixed(body); - const fuzz = f.ctx.fuzz; - const addr = body_r.takeInt(u64, .little) catch unreachable; - - fuzz.queue_mutex.lockUncancelable(io); - defer fuzz.queue_mutex.unlock(io); - try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ - .addr = addr, - .coverage_id = f.coverage_id.?, - } }); - fuzz.queue_cond.signal(io); - }, - .fuzz_test_change => { - const test_i = std.mem.readInt(u32, body[0..4], .little); - instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); - }, - .broadcast_fuzz_input => { - if (f.instances.len == 1) { - // No other processes to broadcast to. - } else if (f.broadcast_undelivered == 0) { - try f.instanceBroadcast(id, body); - } else { - const footer: PendingBroadcastFooter = .{ - .from_id = id, - .body_len = @intCast(body.len), - }; - // There is another broadcast in progress so add this one to the queue. - const size = @sizeOf(PendingBroadcastFooter) + body.len; - try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); - f.pending_broadcasts.appendSliceAssumeCapacity(body); - f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); - } - }, - else => {}, // ignore other messages - } - - instance.message.clearRetainingCapacity(); - try f.addStdoutRead(id, @sizeOf(InHeader)); - } - - fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void { - const instance = &f.instances[id]; - instance.stderr.items.len += n; - try f.addStderrRead(id); - } - - fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void { - const instance = &f.instances[id]; - - instance.broadcast_written += n; - if (instance.broadcast_written == f.broadcast.items.len) { - f.broadcast_undelivered -= 1; - if (f.broadcast_undelivered == 0) { - try f.broadcastComplete(); - } - } else { - f.addStdinWrite(id); - } - } - - fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const instance = &f.instances[id]; - - try instance.message.ensureTotalCapacity(gpa, end); - const start = instance.message.items.len; - instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]}; - f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{ - .file = instance.child.stdout.?, - .data = &instance.stdout_vec, - } }); - } - - fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const instance = &f.instances[id]; - - try instance.stderr.ensureUnusedCapacity(gpa, 1); - instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()}; - f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{ - .file = instance.child.stderr.?, - .data = &instance.stderr_vec, - } }); - } - - fn addStdinWrite(f: *FuzzTestRunner, id: u32) void { - const instance = &f.instances[id]; - - assert(f.broadcast.items.len != instance.broadcast_written); - instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]}; - f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{ - .file = instance.child.stdin.?, - .data = &instance.stdin_vec, - } }); - } - - fn instanceEos(f: *FuzzTestRunner, id: u32) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - const instance = &f.instances[id]; - - instance.child.stdin.?.close(io); - instance.child.stdin = null; - const term = try instance.child.wait(io); - if (!termMatches(.{ .exited = 0 }, term)) { - f.run.step.result_stderr = try f.mergedStderr(); - try f.saveCrash(id, term); - return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); - } - } - - fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { - const step = &f.run.step; - const b = step.owner; - const io = b.graph.io; - - if (f.coverage_id == null) return; - - // Search for the input file corresponding to the instance - const InputHeader = Build.abi.fuzz.MmapInputHeader; - var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; - var in_r: Io.File.Reader = undefined; - var in_f: Io.File = undefined; - var in_name_buf: [12]u8 = undefined; - var in_name: []const u8 = undefined; - var i: u32 = 0; - const header: InputHeader = while (true) : ({ - if (i == std.math.maxInt(u32)) return; - i += 1; - }) { - const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; - in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; - in_f = b.cache_root.handle.openFile(io, in_name, .{ - .lock = .exclusive, - .lock_nonblocking = true, - }) catch |e| switch (e) { - error.FileNotFound => return, - error.WouldBlock => continue, // Can not be from - // the crashed instance since it is still locked. - else => return step.fail("failed to open file '{f}{s}': {t}", .{ - b.cache_root, in_name, e, - }), - }; - - in_r = in_f.readerStreaming(io, &in_r_buf); - const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { - in_f.close(io); - switch (e) { - error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, - }), - error.EndOfStream => continue, - } - }; - - if (header.pc_digest == f.coverage_id.? and - header.instance_id == id and - header.test_i < f.run.fuzz_tests.items.len) - { - break header; - } - - in_f.close(io); - }; - defer in_f.close(io); - - // Save it to a seperate file - const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; - const out = b.cache_root.handle.createFile(io, crash_name, .{ - .lock = .exclusive, // Multiple run steps could have found a crash at the same time - }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ - b.cache_root, crash_name, e, - }); - defer out.close(io); - - var out_w_buf: [512]u8 = undefined; - var out_w = out.writerStreaming(io, &out_w_buf); - _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { - error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, - }), - error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ - b.cache_root, crash_name, out_w.err.?, - }), - }; - - return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ - f.run.fuzz_tests.items[header.test_i], - fmtTerm(term), - b.cache_root, - crash_name, - }); - } - - fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void { - assert(f.instances.len > 1); - assert(f.broadcast_undelivered == 0); // no other broadcast is progress - assert(f.broadcast.items.len == 0); - assert(from_id < f.instances.len); - - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - - var out_header: OutHeader = .{ - .tag = .new_fuzz_input, - .bytes_len = @intCast(bytes.len), - }; - if (std.builtin.Endian.native != .little) { - std.mem.byteSwapAllFields(OutHeader, &out_header); - } - try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len); - f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header)); - f.broadcast.appendSliceAssumeCapacity(bytes); - - f.broadcast_undelivered = @intCast(f.instances.len - 1); - for (0.., f.instances) |to_id, *instance| { - if (to_id == from_id) continue; - instance.broadcast_written = 0; - f.addStdinWrite(@intCast(to_id)); - } - } - - fn broadcastComplete(f: *FuzzTestRunner) !void { - assert(f.instances.len > 1); - assert(f.broadcast_undelivered == 0); - f.broadcast.clearRetainingCapacity(); - - const pending = &f.pending_broadcasts; - if (pending.items.len != 0) { - // Another broadcast is pending; copy it over to `broadcast` - - const footer_len = @sizeOf(PendingBroadcastFooter); - const footer_bytes = pending.items[pending.items.len - footer_len ..]; - const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes); - pending.items.len -= footer_len; - - const body = pending.items[pending.items.len - footer.body_len ..]; - try f.instanceBroadcast(footer.from_id, body); - pending.items.len -= body.len; - } - } - - fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { - const step_owner = f.run.step.owner; - const arena = step_owner.allocator; - - // Collect any available stderr - while (f.batch.next()) |completion| { - if (completion.index % 3 != 2) continue; - const len = completion.result.file_read_streaming catch continue; - f.instances[completion.index / 3].stderr.items.len += len; - } - - var stderr_len: usize = 0; - for (f.instances) |*instance| stderr_len += instance.stderr.items.len; - const stderr = try arena.alloc(u8, stderr_len); - - stderr_len = 0; - for (f.instances) |*instance| { - @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items); - stderr_len += instance.stderr.items.len; - } - return stderr; - } -}; - -fn evalFuzzTest( - run: *Run, - spawn_options: process.SpawnOptions, - options: Step.MakeOptions, - fuzz_context: FuzzContext, -) !void { - var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); - defer f.deinit(); - try f.startInstances(); - try f.listen(); -} - -const TestMetadata = struct { - names: []const u32, - ns_per_test: []u64, - expected_panic_msgs: []const u32, - string_bytes: []const u8, - next_index: u32, - prog_node: std.Progress.Node, - - fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { - return .{ - .names = tm.names, - .string_bytes = tm.string_bytes, - }; - } - - fn testName(tm: TestMetadata, index: u32) []const u8 { - return tm.toCachedTestMetadata().testName(index); - } -}; - -pub const CachedTestMetadata = struct { - names: []const u32, - string_bytes: []const u8, - - pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { - return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); - } -}; - -fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { - while (metadata.next_index < metadata.names.len) { - const i = metadata.next_index; - metadata.next_index += 1; - - if (metadata.expected_panic_msgs[i] != 0) continue; - - const name = metadata.testName(i); - if (sub_prog_node.*) |n| n.end(); - sub_prog_node.* = metadata.prog_node.start(name, 0); - - try sendRunTestMessage(io, in, .run_test, i); - return; - } else { - metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done - try sendMessage(io, in, .exit); - } -} - -fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 0, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 4, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u32, index, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -fn sendRunFuzzTestMessage( - io: Io, - file: Io.File, - test_names: []const []const u8, - kind: std.Build.abi.fuzz.LimitKind, - amount_or_instance: u64, -) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = .start_fuzzing, - .bytes_len = 1 + 8 + 4 + count: { - var c: u32 = @intCast(test_names.len * 4); - for (test_names) |name| { - c += @intCast(name.len); - } - break :count c; - }, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - for (test_names) |test_name| { - w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeAll(test_name) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - } -} - -fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { - const b = run.step.owner; - const io = b.graph.io; - const arena = b.allocator; - const gpa = b.allocator; - - var child = try process.spawn(io, spawn_options); - defer child.kill(io); - - switch (run.stdin) { - .bytes => |bytes| { - child.stdin.?.writeStreamingAll(io, bytes) catch |err| { - return run.step.fail("unable to write stdin: {t}", .{err}); - }; - child.stdin.?.close(io); - child.stdin = null; - }, - .lazy_path => |lazy_path| { - const path = lazy_path.getPath3(b, &run.step); - const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { - return run.step.fail("unable to open stdin file: {t}", .{err}); - }; - defer file.close(io); - // TODO https://github.com/ziglang/zig/issues/23955 - var read_buffer: [1024]u8 = undefined; - var file_reader = file.reader(io, &read_buffer); - var write_buffer: [1024]u8 = undefined; - var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); - _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { - error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ - path, file_reader.err.?, - }), - error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ - stdin_writer.err.?, - }), - }; - stdin_writer.interface.flush() catch |err| switch (err) { - error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ - stdin_writer.err.?, - }), - }; - child.stdin.?.close(io); - child.stdin = null; - }, - .none => {}, - } - - var stdout_bytes: ?[]const u8 = null; - var stderr_bytes: ?[]const u8 = null; - - if (child.stdout) |stdout| { - if (child.stderr) |stderr| { - var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; - var multi_reader: Io.File.MultiReader = undefined; - multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); - defer multi_reader.deinit(); - - const stdout_reader = multi_reader.reader(0); - const stderr_reader = multi_reader.reader(1); - - while (multi_reader.fill(64, .none)) |_| { - if (run.stdio_limit.toInt()) |limit| { - if (stdout_reader.buffered().len > limit) - return error.StdoutStreamTooLong; - if (stderr_reader.buffered().len > limit) - return error.StderrStreamTooLong; - } - } else |err| switch (err) { - error.Timeout => unreachable, - error.EndOfStream => {}, - else => |e| return e, - } - - try multi_reader.checkAnyError(); - - // TODO: this string can leak since alloc below can return error. - stdout_bytes = try multi_reader.toOwnedSlice(0); - // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. - stderr_bytes = try multi_reader.toOwnedSlice(1); - } else { - var stdout_reader = stdout.readerStreaming(io, &.{}); - stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => return stdout_reader.err.?, - error.StreamTooLong => return error.StdoutStreamTooLong, - }; - } - } else if (child.stderr) |stderr| { - var stderr_reader = stderr.readerStreaming(io, &.{}); - stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => return stderr_reader.err.?, - error.StreamTooLong => return error.StderrStreamTooLong, - }; - } - - if (stderr_bytes) |bytes| if (bytes.len > 0) { - // Treat stderr as an error message. - const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { - .check => |checks| !checksContainStderr(checks.items), - else => true, - }; - if (stderr_is_diagnostic) { - run.step.result_stderr = bytes; - } - }; - - run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; - - return .{ - .term = try child.wait(io), - .stdout = stdout_bytes, - .stderr = stderr_bytes, - }; -} - -fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { - const b = run.step.owner; - const compiles = artifact.getCompileDependencies(true); - for (compiles) |compile| { - if (compile.root_module.resolved_target.?.result.os.tag == .windows and - compile.isDynamicLibrary()) - { - addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); - } - } -} - -fn failForeign( - run: *Run, - suggested_flag: []const u8, - argv0: []const u8, - exe: *Step.Compile, -) error{ MakeFailed, MakeSkipped, OutOfMemory } { - switch (run.stdio) { - .check, .zig_test => { - if (run.skip_foreign_checks) - return error.MakeSkipped; - - const b = run.step.owner; - const host_name = try b.graph.host.result.zigTriple(b.allocator); - const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); - - return run.step.fail( - \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) - \\ consider using {s} or enabling skip_foreign_checks in the Run step - , .{ argv0, foreign_name, host_name, suggested_flag }); - }, - else => { - return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); - }, - } -} - -fn hashStdIo(hh: *std.Build.Cache.HashHelper, stdio: StdIo) void { - switch (stdio) { - .infer_from_args, .inherit, .zig_test => {}, - .check => |checks| for (checks.items) |check| { - hh.add(@as(std.meta.Tag(StdIo.Check), check)); - switch (check) { - .expect_stderr_exact, - .expect_stderr_match, - .expect_stdout_exact, - .expect_stdout_match, - => |s| hh.addBytes(s), - - .expect_term => |term| { - hh.add(@as(std.meta.Tag(process.Child.Term), term)); - switch (term) { - inline .exited, .signal, .stopped => |x| hh.add(x), - .unknown => |x| hh.add(x), - } - }, - } - }, - } -} diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index fd14090812ab8730822724792fe2233a420ecf5e..90d9e28155fabf1a853808d19bd3b81097fa745e 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -6,7 +6,7 @@ const mem = std.mem; const TranslateC = @This(); -pub const base_id: Step.Id = .translate_c; +pub const base_tag: Step.Tag = .translate_c; step: Step, source: std.Build.LazyPath, @@ -31,7 +31,7 @@ pub fn create(owner: *std.Build, options: Options) *TranslateC { const source = options.root_source_file.dupe(owner); translate_c.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "translate-c", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig index 0cc3b787c3f8416c284b0977ea9e73b9bf405fd7..b41cca59d9d82a6c959929101a4ebf203cc2cc1e 100644 --- a/lib/std/Build/Step/UpdateSourceFiles.zig +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -14,7 +14,7 @@ const ArrayList = std.ArrayList; step: Step, output_source_files: std.ArrayList(OutputSourceFile), -pub const base_id: Step.Id = .update_source_files; +pub const base_tag: Step.Tag = .update_source_files; pub const OutputSourceFile = struct { contents: Contents, @@ -30,7 +30,7 @@ pub fn create(owner: *std.Build) *UpdateSourceFiles { const usf = owner.allocator.create(UpdateSourceFiles) catch @panic("OOM"); usf.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "UpdateSourceFiles", .owner = owner, .makeFn = make, diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 3613fa3fef8fa5ac1a5587a4c4cdcca96661b7b6..06f030efbc8003fb31334141be311e2487bb0c52 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -18,7 +18,7 @@ directories: std.ArrayList(Directory), generated_directory: std.Build.GeneratedFile, mode: Mode = .whole_cached, -pub const base_id: Step.Id = .write_file; +pub const base_tag: Step.Tag = .write_file; pub const Mode = union(enum) { /// Default mode. Integrates with the cache system. The directory should be @@ -89,10 +89,9 @@ pub fn create(owner: *std.Build) *WriteFile { const write_file = owner.allocator.create(WriteFile) catch @panic("OOM"); write_file.* = .{ .step = Step.init(.{ - .id = base_id, + .tag = base_tag, .name = "WriteFile", .owner = owner, - .makeFn = make, }), .files = .empty, .directories = .empty, @@ -191,209 +190,3 @@ fn maybeUpdateName(write_file: *WriteFile) void { } } } - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const graph = b.graph; - const io = graph.io; - const arena = b.allocator; - const gpa = graph.cache.gpa; - const write_file: *WriteFile = @fieldParentPtr("step", step); - - const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); - var open_dirs_count: usize = 0; - defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); - - switch (write_file.mode) { - .whole_cached => { - step.clearWatchInputs(); - - // The cache is used here not really as a way to speed things up - because writing - // the data to a file would probably be very fast - but as a way to find a canonical - // location to put build artifacts. - - // If, for example, a hard-coded path was used as the location to put WriteFile - // files, then two WriteFiles executing in parallel might clobber each other. - - var man = b.graph.cache.obtain(); - defer man.deinit(); - - for (write_file.files.items) |file| { - man.hash.addBytes(file.sub_path); - - switch (file.contents) { - .bytes => |bytes| { - man.hash.addBytes(bytes); - }, - .copy => |lazy_path| { - const path = lazy_path.getPath3(b, step); - _ = try man.addFilePath(path, null); - try step.addWatchInput(lazy_path); - }, - } - } - - for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { - man.hash.addBytes(dir.sub_path); - for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); - if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); - - const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); - const src_dir_path = dir.source.getPath3(b, step); - - var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {s}", .{ - src_dir_path, @errorName(err), - }); - }; - open_dir_cache_elem.* = src_dir; - open_dirs_count += 1; - - var it = try src_dir.walk(gpa); - defer it.deinit(); - while (try it.next(io)) |entry| { - if (!dir.options.pathIncluded(entry.path)) continue; - - switch (entry.kind) { - .directory => { - if (need_derived_inputs) { - const entry_path = try src_dir_path.join(arena, entry.path); - try step.addDirectoryWatchInputFromPath(entry_path); - } - }, - .file => { - const entry_path = try src_dir_path.join(arena, entry.path); - _ = try man.addFilePath(entry_path, null); - }, - else => continue, - } - } - } - - if (try step.cacheHit(&man)) { - const digest = man.final(); - write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); - assert(step.result_cached); - return; - } - - const digest = man.final(); - const cache_path = "o" ++ Dir.path.sep_str ++ digest; - - write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); - - try operate(write_file, open_dir_cache, .{ - .root_dir = b.cache_root, - .sub_path = cache_path, - }); - - try step.writeManifest(&man); - }, - .tmp => { - step.result_cached = false; - - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - - write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); - - try operate(write_file, open_dir_cache, .{ - .root_dir = b.cache_root, - .sub_path = tmp_dir_sub_path, - }); - }, - .mutate => |lp| { - step.result_cached = false; - const root_path = try lp.getPath4(b, step); - write_file.generated_directory.path = try root_path.toString(arena); - try operate(write_file, open_dir_cache, root_path); - }, - } -} - -fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { - const step = &write_file.step; - const b = step.owner; - const io = b.graph.io; - const gpa = b.graph.cache.gpa; - const arena = b.allocator; - - var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| - return step.fail("unable to make path {f}: {t}", .{ root_path, err }); - defer cache_dir.close(io); - - for (write_file.files.items) |file| { - if (Dir.path.dirname(file.sub_path)) |dirname| { - cache_dir.createDirPath(io, dirname) catch |err| { - return step.fail("unable to make path '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, dirname, err, - }); - }; - } - switch (file.contents) { - .bytes => |bytes| { - cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, file.sub_path, err, - }); - }; - }, - .copy => |file_source| { - const source_path = file_source.getPath2(b, step); - const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { - return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ - source_path, root_path, Dir.path.sep, file.sub_path, err, - }); - }; - // At this point we already will mark the step as a cache miss. - // But this is kind of a partial cache hit since individual - // file copies may be avoided. Oh well, this information is - // discarded. - _ = prev_status; - }, - } - } - - for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { - const src_dir_path = dir.source.getPath3(b, step); - const dest_dirname = dir.sub_path; - - if (dest_dirname.len != 0) { - cache_dir.createDirPath(io, dest_dirname) catch |err| { - return step.fail("unable to make path '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, dest_dirname, err, - }); - }; - } - - var it = try already_open_dir.walk(gpa); - defer it.deinit(); - while (try it.next(io)) |entry| { - if (!dir.options.pathIncluded(entry.path)) continue; - - const src_entry_path = try src_dir_path.join(arena, entry.path); - const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); - switch (entry.kind) { - .directory => try cache_dir.createDirPath(io, dest_path), - .file => { - const prev_status = Io.Dir.updateFile( - src_entry_path.root_dir.handle, - io, - src_entry_path.sub_path, - cache_dir, - dest_path, - .{}, - ) catch |err| { - return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ - src_entry_path, root_path, Dir.path.sep, dest_path, err, - }); - }; - _ = prev_status; - }, - else => continue, - } - } - } -} diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 9c2c956582d70a6582a268641702f3b41f7c08c7..75ef9c9b63195335b78c5cfa04d4d75742532905 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -11,8 +11,6 @@ const Writer = std.Io.Writer; const tokenizer = @import("zig/tokenizer.zig"); -/// The serialized output of configure phase ingested by make phase. -pub const Configuration = @import("zig/Configuration.zig"); pub const ErrorBundle = @import("zig/ErrorBundle.zig"); pub const Server = @import("zig/Server.zig"); pub const Client = @import("zig/Client.zig"); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig index 86bf82fa3d0c73117dc85755b36a240e76fbc8c1..589828295d70f3a2daffd4307f40636daf4a234a 100644 --- a/lib/std/zig/Configuration.zig +++ b/lib/std/zig/Configuration.zig @@ -3,22 +3,240 @@ const Configuration = @This(); const std = @import("../std.zig"); const Io = std.Io; const Allocator = std.mem.Allocator; +const assert = std.debug.assert; string_bytes: []u8, steps: []Step, path_deps_base: []Path.Base, path_deps_sub: []String, unlazy_deps: []String, +extra: []u32, +/// The field order here matches `Configuration` which documents the order in +/// the serialized format. pub const Header = extern struct { string_bytes_len: u32, steps_len: u32, path_deps_len: u32, unlazy_deps_len: u32, + extra_len: u32, + + /// Index into `steps`. + default_step: u32, +}; + +pub const Wip = struct { + gpa: Allocator, + string_table: StringTable = .empty, + deps_table: DepsTable = .empty, + + string_bytes: std.ArrayList(u8) = .empty, + unlazy_deps: std.ArrayList(String) = .empty, + steps: std.ArrayList(Step) = .empty, + path_deps: std.MultiArrayList(Path) = .empty, + extra: std.ArrayList(u32) = .empty, + + const DepsTable = std.HashMapUnmanaged(Deps, void, DepsTableContext, std.hash_map.default_max_load_percentage); + + const DepsTableContext = struct { + extra: []const u32, + + pub fn eql(ctx: @This(), a: Deps, b: Deps) bool { + const len_a = ctx.extra[@intFromEnum(a)]; + const len_b = ctx.extra[@intFromEnum(b)]; + const slice_a = ctx.extra[@intFromEnum(a) + 1 ..][0..len_a]; + const slice_b = ctx.extra[@intFromEnum(b) + 1 ..][0..len_b]; + return std.mem.eql(u32, slice_a, slice_b); + } + + pub fn hash(ctx: @This(), key: Deps) u64 { + const len = ctx.extra[@intFromEnum(key)]; + const slice = ctx.extra[@intFromEnum(key) + 1 ..][0..len]; + return std.hash_map.hashString(@ptrCast(slice)); + } + }; + + const StringTable = std.HashMapUnmanaged(String, void, StringTableContext, std.hash_map.default_max_load_percentage); + const StringTableContext = struct { + bytes: []const u8, + + pub fn eql(_: @This(), a: String, b: String) bool { + return a == b; + } + + pub fn hash(ctx: @This(), key: String) u64 { + return std.hash_map.hashString(std.mem.sliceTo(ctx.bytes[@intFromEnum(key)..], 0)); + } + }; + + const StringTableIndexAdapter = struct { + bytes: []const u8, + + pub fn eql(ctx: @This(), a: []const u8, b: String) bool { + return std.mem.eql(u8, a, std.mem.sliceTo(ctx.bytes[@intFromEnum(b)..], 0)); + } + + pub fn hash(_: @This(), adapted_key: []const u8) u64 { + assert(std.mem.indexOfScalar(u8, adapted_key, 0) == null); + return std.hash_map.hashString(adapted_key); + } + }; + + pub fn init(gpa: Allocator) Wip { + return .{ .gpa = gpa }; + } + + pub fn deinit(wip: *Wip) void { + const gpa = wip.gpa; + wip.string_bytes.deinit(gpa); + wip.unlazy_deps.deinit(gpa); + wip.steps.deinit(gpa); + wip.path_deps.deinit(gpa); + wip.extra.deinit(gpa); + wip.* = undefined; + } + + pub const Static = struct { + default_step: u32, + }; + + pub fn write(wip: *Wip, w: *Io.Writer, static: Static) Io.Writer.Error!void { + const header: Header = .{ + .string_bytes_len = @intCast(wip.string_bytes.items.len), + .steps_len = @intCast(wip.steps.items.len), + .path_deps_len = @intCast(wip.path_deps.len), + .unlazy_deps_len = @intCast(wip.unlazy_deps.items.len), + .extra_len = @intCast(wip.extra.items.len), + + .default_step = static.default_step, + }; + var buffers = [_][]const u8{ + @ptrCast(&header), + wip.string_bytes.items, + @ptrCast(wip.steps.items), + @ptrCast(wip.path_deps.items(.base)), + @ptrCast(wip.path_deps.items(.sub)), + @ptrCast(wip.unlazy_deps.items), + @ptrCast(wip.extra.items), + }; + try w.writeVecAll(&buffers); + } + + pub fn addString(wip: *Wip, bytes: []const u8) Allocator.Error!String { + const gpa = wip.gpa; + assert(std.mem.indexOfScalar(u8, bytes, 0) == null); + const gop = try wip.string_table.getOrPutContextAdapted( + gpa, + @as([]const u8, bytes), + @as(StringTableIndexAdapter, .{ .bytes = wip.string_bytes.items }), + @as(StringTableContext, .{ .bytes = wip.string_bytes.items }), + ); + if (gop.found_existing) return gop.key_ptr.*; + + try wip.string_bytes.ensureUnusedCapacity(gpa, bytes.len + 1); + const new_off: String = @enumFromInt(wip.string_bytes.items.len); + + wip.string_bytes.appendSliceAssumeCapacity(bytes); + wip.string_bytes.appendAssumeCapacity(0); + + gop.key_ptr.* = new_off; + + return new_off; + } + + pub fn prepareDeps(wip: *Wip, n: usize) Allocator.Error![]u32 { + const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1); + slice[0] = @intCast(n); + return slice[1..]; + } + + pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps { + const gpa = wip.gpa; + const gop = try wip.deps_table.getOrPutContext(gpa, deps, @as(DepsTableContext, .{ + .extra = wip.extra.items, + })); + if (gop.found_existing) { + wip.extra.items.len = @intFromEnum(deps); + return gop.key_ptr.*; + } else { + return deps; + } + } + + pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { + const gpa = wip.gpa; + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + try wip.extra.ensureUnusedCapacity(gpa, fields.len); + return addExtraAssumeCapacity(wip, extra); + } + + pub fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 { + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + const result: u32 = @intCast(wip.extra.items.len); + wip.extra.items.len += fields.len; + setExtra(wip, result, extra); + return result; + } + + fn setExtra(wip: *Wip, index: usize, extra: anytype) void { + const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; + var i = index; + inline for (fields) |field| { + wip.extra.items[i] = switch (field.type) { + u32 => @field(extra, field.name), + String, Deps => @intFromEnum(@field(extra, field.name)), + else => @compileError("bad field type"), + }; + i += 1; + } + } }; pub const Step = extern struct { name: String, + flags: Flags, + deps: Deps, + /// Points into `extra` for step-specific data. + extra_index: u32, + + pub const Flags = packed struct(u32) { + tag: Tag, + _: u24 = 0, + }; + + pub const Index = enum(u32) { + _, + }; + + pub const Tag = enum(u8) { + top_level, + compile, + install_artifact, + install_file, + install_dir, + remove_dir, + fail, + fmt, + translate_c, + write_file, + update_source_files, + run, + check_file, + check_object, + config_header, + objcopy, + options, + }; + + pub const TopLevel = struct { + description: String, + }; +}; + +/// Points into `extra`, where the first element is number of deps, +/// following elements is `Step.Index` per dep. +pub const Deps = enum(u32) { + _, }; pub const Path = extern struct { @@ -27,8 +245,8 @@ pub const Path = extern struct { pub const Base = enum(u8) { cwd, - global_cache, local_cache, + global_cache, build_root, }; @@ -40,6 +258,7 @@ pub const Path = extern struct { } }; +/// Points into `string_bytes`, null-terminated. pub const String = enum(u32) { _, @@ -49,24 +268,29 @@ pub const String = enum(u32) { } }; -pub const LoadError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; +pub const LoadFileError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; -pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration { +pub fn loadFile(arena: Allocator, io: Io, file: Io.File) LoadFileError!Configuration { var buffer: [2000]u8 = undefined; var fr = file.reader(io, &buffer); - const header = fr.interface.takeStruct(Header, .little) catch |err| switch (err) { + return load(arena, &fr.interface) catch |err| switch (err) { error.ReadFailed => return fr.err.?, else => |e| return e, }; +} +pub const LoadError = Io.Reader.Error || Allocator.Error; + +pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { + const header = try reader.takeStruct(Header, .little); var result: Configuration = .{ .string_bytes = try arena.alloc(u8, header.string_bytes_len), .steps = try arena.alloc(Step, header.steps_len), .path_deps_sub = try arena.alloc(String, header.path_deps_len), .path_deps_base = try arena.alloc(Path.Base, header.path_deps_len), .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), + .extra = try arena.alloc(u32, header.extra_len), }; - var vecs = [_][]u8{ result.string_bytes, @ptrCast(result.steps), @@ -74,10 +298,6 @@ pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration { @ptrCast(result.path_deps_sub), @ptrCast(result.unlazy_deps), }; - fr.interface.readVecAll(&vecs) catch |err| switch (err) { - error.ReadFailed => return fr.err.?, - else => |e| return e, - }; - + try reader.readVecAll(&vecs); return result; } diff --git a/src/main.zig b/src/main.zig index 7648cadeeaa7a19581beac9e2f6175ddec486157..5f2776cf38a7b7a5c3572115f7ead563dc0733b9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -298,7 +298,11 @@ fn mainArgs( return process.exit(try llvmArMain(arena, args)); } else if (mem.eql(u8, cmd, "build")) { dev.check(.build_command); - return cmdBuild(gpa, arena, io, cmd_args, environ_map); + var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ + .child_allocator = arena, + .io = io, + }; + return cmdBuild(gpa, thread_safe_arena.allocator(), io, cmd_args, environ_map); } else if (mem.eql(u8, cmd, "clang") or mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) { @@ -4941,6 +4945,7 @@ test sanitizeExampleName { fn cmdBuild( gpa: Allocator, + /// Needs a thread-safe arena. arena: Allocator, io: Io, args: []const []const u8, @@ -4973,28 +4978,34 @@ fn cmdBuild( var debug_target: ?[]const u8 = null; var debug_libc_paths_file: ?[]const u8 = null; - const argv_index_exe = configure_argv.items.len; - _ = try configure_argv.addOne(arena); - const self_exe_path = try process.executablePathAlloc(io, arena); - try configure_argv.append(arena, self_exe_path); + const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}); + try configure_argv.ensureUnusedCapacity(arena, 16); + + const argv_index_exe = configure_argv.items.len; + _ = configure_argv.addOneAssumeCapacity(); + + configure_argv.appendAssumeCapacity("--zig"); + configure_argv.appendAssumeCapacity(self_exe_path); + + configure_argv.appendAssumeCapacity("--zig-lib-dir"); const argv_index_zig_lib_dir = configure_argv.items.len; - _ = try configure_argv.addOne(arena); + _ = configure_argv.addOneAssumeCapacity(); + configure_argv.appendAssumeCapacity("--build-root"); const argv_index_build_file = configure_argv.items.len; - _ = try configure_argv.addOne(arena); + _ = configure_argv.addOneAssumeCapacity(); + configure_argv.appendAssumeCapacity("--local-cache"); const argv_index_cache_dir = configure_argv.items.len; - _ = try configure_argv.addOne(arena); + _ = configure_argv.addOneAssumeCapacity(); + configure_argv.appendAssumeCapacity("--global-cache"); const argv_index_global_cache_dir = configure_argv.items.len; - _ = try configure_argv.addOne(arena); + _ = configure_argv.addOneAssumeCapacity(); - try configure_argv.appendSlice(arena, &.{ - "--seed", - try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}), - }); + configure_argv.appendSliceAssumeCapacity(&.{ "--seed", default_seed }); const argv_index_seed = configure_argv.items.len - 1; const argv_index_configuration_file = make_argv.items.len; @@ -5192,14 +5203,6 @@ fn cmdBuild( ); try setThreadLimit(arena, thread_limit); - // Kick off an optimized compilation of the make runner. - var make_runner_task = io.async(compileMakeRunner, .{ io, .{ - .dirs = &dirs, - .optimize = .ReleaseSafe, - .parent_prog_node = root_prog_node, - } }); - defer if (make_runner_task.cancel(io)) |mr| mr.deinit(io) else |_| {}; - // Cache lookup for configure options. If we get a match, we can skip // execution of the configure script. If not, we get the file path to pass // to the configure process. @@ -5255,6 +5258,19 @@ fn cmdBuild( break :lci lci; }; + // Kick off an optimized compilation of the make runner. + var make_runner_task = io.async(compileMakeRunner, .{ gpa, arena, io, .{ + .dirs = &dirs, + .environ_map = environ_map, + .parent_prog_node = root_prog_node, + .resolved_target = resolved_target, + .libc_installation = libc_installation, + .thread_limit = thread_limit, + .self_exe_path = self_exe_path, + .color = color, + } }); + defer _ = make_runner_task.cancel(io) catch {}; + configure_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; configure_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; configure_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; @@ -5305,7 +5321,7 @@ fn cmdBuild( .root_src_path = fs.path.basename(runner), } else .{ .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), - .root_src_path = "build_runner.zig", + .root_src_path = "configure_runner.zig", }; const config = try Compilation.Config.resolve(.{ @@ -5533,7 +5549,7 @@ fn cmdBuild( const comp = Compilation.create(gpa, arena, io, &create_diag, .{ .libc_installation = libc_installation, .dirs = dirs, - .root_name = "build", + .root_name = "configure", .config = config, .root_mod = root_mod, .main_mod = build_mod, @@ -5554,7 +5570,7 @@ fn cmdBuild( .environ_map = environ_map, }) catch |err| switch (err) { error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - else => fatal("failed to create compilation: {t}", .{err}), + else => |e| fatal("failed to create compilation: {t}", .{e}), }; defer comp.destroy(); @@ -5625,7 +5641,7 @@ fn cmdBuild( // add them to `config_man` before obtaining the final digest. // * If it contains a set of lazy packages that need to be // fetched, we need to fetch those now and re-run configure. - var configuration = std.zig.Configuration.load(arena, io, config_tmp_file) catch |err| + var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); if (configuration.unlazy_deps.len != 0) { @@ -5661,7 +5677,7 @@ fn cmdBuild( } for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { - const conf_path: std.zig.Configuration.Path = .{ .base = base, .sub = sub }; + const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); } @@ -5707,7 +5723,6 @@ fn cmdBuild( const make_runner = make_runner_task.await(io) catch |err| fatal("failed to compile maker: {t}", .{err}); - defer make_runner.deinit(io); make_argv.items[0] = try make_runner.exe_path.toString(arena); make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); @@ -5748,22 +5763,86 @@ const MakeRunner = struct { exe_path: Path, const Options = struct { + environ_map: *const process.Environ.Map, dirs: *Compilation.Directories, - optimize: std.builtin.OptimizeMode, parent_prog_node: std.Progress.Node, + resolved_target: Package.Module.ResolvedTarget, + libc_installation: ?*const LibCInstallation, + self_exe_path: []const u8, + thread_limit: usize, + color: Color, }; - - fn deinit(mr: MakeRunner, io: Io) void { - _ = mr; - _ = io; - @panic("TODO"); - } }; -fn compileMakeRunner(io: Io, options: MakeRunner.Options) !MakeRunner { - _ = io; - _ = options; - @panic("TODO"); +fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner { + const compile_prog_node = options.parent_prog_node.start("Compile Maker", 0); + defer compile_prog_node.end(); + + const optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(options.environ_map)) + .Debug + else + .ReleaseSafe; + const strip = optimize_mode != .Debug; + + const main_mod_paths: Package.Module.CreateOptions.Paths = .{ + .root = try .fromRoot(arena, options.dirs.*, .zig_lib, "compiler"), + .root_src_path = "maker.zig", + }; + + const config = try Compilation.Config.resolve(.{ + .output_mode = .Exe, + .root_strip = strip, + .root_optimize_mode = optimize_mode, + .resolved_target = options.resolved_target, + .have_zcu = true, + .emit_bin = true, + .is_test = false, + }); + + const root_mod = try Package.Module.create(arena, .{ + .paths = main_mod_paths, + .fully_qualified_name = "root", + .cc_argv = &.{}, + .inherited = .{ + .resolved_target = options.resolved_target, + .optimize_mode = optimize_mode, + .strip = strip, + }, + .global = config, + .parent = null, + }); + + var create_diag: Compilation.CreateDiagnostic = undefined; + const comp = Compilation.create(gpa, arena, io, &create_diag, .{ + .dirs = options.dirs.*, + .root_name = "maker", + .config = config, + .root_mod = root_mod, + .main_mod = root_mod, + .emit_bin = .yes_cache, + .self_exe_path = options.self_exe_path, + .thread_limit = options.thread_limit, + .cache_mode = .whole, + .environ_map = options.environ_map, + }) catch |err| switch (err) { + error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), + error.Canceled => |e| return e, + else => |e| fatal("failed to create compilation: {t}", .{e}), + }; + defer comp.destroy(); + + try updateModule(comp, options.color, compile_prog_node); + + const exe_path: Path = .{ + .root_dir = options.dirs.global_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ + &Cache.binToHex(comp.digest.?), comp.emit_bin.?, + }), + }; + + return .{ + .exe_path = exe_path, + }; } const Fork = struct { @@ -5972,7 +6051,7 @@ fn jitCmdInner( .environ_map = environ_map, }) catch |err| switch (err) { error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - else => fatal("failed to create compilation: {s}", .{@errorName(err)}), + else => fatal("failed to create compilation: {t}", .{err}), }; defer comp.destroy(); -- 2.54.0