diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig new file mode 100644 index 0000000000000000000000000000000000000000..15c2799185bb5610206e2dfda5ab61ee215898da --- /dev/null +++ b/lib/compiler/Maker.zig @@ -0,0 +1,1848 @@ +const Maker = @This(); +const builtin = @import("builtin"); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; +const File = std.Io.File; +const Io = std.Io; +const Path = std.Build.Cache.Path; +const Writer = std.Io.Writer; +const assert = std.debug.assert; +const fatal = std.process.fatal; +const fmt = std.fmt; +const log = std.log; +const mem = std.mem; +const process = std.process; + +const Fuzz = @import("Maker/Fuzz.zig"); +const Graph = @import("Maker/Graph.zig"); +const Step = @import("Maker/Step.zig"); +const Watch = @import("Maker/Watch.zig"); +const WebServer = @import("Maker/WebServer.zig"); + +pub const std_options: std.Options = .{ + .side_channels_mitigations = .none, + .http_disable_tls = true, +}; + +gpa: Allocator, +graph: *Graph, +install_paths: InstallPaths, +scanned_config: *const ScannedConfig, +steps: []Step, + +available_rss: usize, +max_rss_is_default: bool, +max_rss_mutex: Io.Mutex, +skip_oom_steps: bool, +unit_test_timeout_ns: ?u64, +watch: bool, +web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, +/// Allocated into `gpa`. +memory_blocked_steps: std.ArrayList(Configuration.Step.Index), +/// Allocated into `gpa`. +step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), + +error_style: ErrorStyle, +multiline_errors: MultilineErrors, +summary: Summary, + +pub fn main(init: process.Init.Minimal) !void { + // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not + // always the case. So, we do need a true gpa for some things. + var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); + defer _ = safe_gpa_state.deinit(); + const gpa = safe_gpa_state.allocator(); + + var threaded: std.Io.Threaded = .init(gpa, .{ + .environ = init.environ, + .argv0 = .init(init.args), + }); + defer threaded.deinit(); + const io = threaded.io(); + + // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. + var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + const args = try init.args.toSlice(arena); + + // skip my own exe name + var arg_idx: usize = 1; + + const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); + const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); + const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); + const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); + const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); + const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); + + const cwd: Io.Dir = .cwd(); + + const zig_lib_directory: Cache.Directory = .{ + .path = zig_lib_dir, + .handle = try cwd.openDir(io, zig_lib_dir, .{}), + }; + + const build_root_directory: Cache.Directory = .{ + .path = build_root, + .handle = try cwd.openDir(io, build_root, .{}), + }; + + const local_cache_directory: Cache.Directory = .{ + .path = local_cache_root, + .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), + }; + + const global_cache_directory: Cache.Directory = .{ + .path = global_cache_root, + .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), + }; + + var graph: Graph = .{ + .io = io, + .arena = arena, + .cache = .{ + .io = io, + .gpa = gpa, + .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), + .cwd = try process.currentPathAlloc(io, arena), + }, + .zig_exe = zig_exe, + .environ_map = try init.environ.createMap(arena), + .global_cache_root = global_cache_directory, + .zig_lib_directory = zig_lib_directory, + }; + + graph.cache.addPrefix(.{ .path = null, .handle = cwd }); + graph.cache.addPrefix(build_root_directory); + graph.cache.addPrefix(local_cache_directory); + graph.cache.addPrefix(global_cache_directory); + graph.cache.hash.addBytes(builtin.zig_version_string); + + var step_names: std.ArrayList([]const u8) = .empty; + var debug_log_scopes: std.ArrayList([]const u8) = .empty; + var help_menu = false; + var steps_menu = false; + var print_configuration = false; + var override_install_prefix: ?[]const u8 = null; + var override_lib_dir: ?[]const u8 = null; + var override_bin_dir: ?[]const u8 = null; + var override_include_dir: ?[]const u8 = null; + var error_style: ErrorStyle = .verbose; + var multiline_errors: MultilineErrors = .indent; + var summary: ?Summary = null; + var max_rss: u64 = 0; + var skip_oom_steps = false; + var test_timeout_ns: ?u64 = null; + var color: Color = .auto; + var watch = false; + var fuzz: ?Fuzz.Mode = null; + var debounce_interval_ms: u16 = 50; + var webui_listen: ?Io.net.IpAddress = null; + var verbose = false; + var sysroot: ?[]const u8 = null; + var search_prefixes: std.ArrayList([]const u8) = .empty; + var libc_file: ?[]const u8 = null; + var debug_pkg_config: bool = false; + // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, + // this will be the directory $glibc-build-dir/install/glibcs + // Given the example of the aarch64 target, this is the directory + // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. + // Also works for dynamic musl. + var libc_runtimes_dir: ?[]const u8 = null; + var enable_wine = false; + var enable_qemu = false; + var enable_wasmtime = false; + var enable_darling = false; + var enable_rosetta = false; + var reference_trace: ?u32 = null; + var run_args: ?[]const []const u8 = null; + + if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { + if (std.meta.stringToEnum(ErrorStyle, str)) |style| { + error_style = style; + } + } + + if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { + if (std.meta.stringToEnum(MultilineErrors, str)) |style| { + multiline_errors = style; + } + } + + while (nextArg(args, &arg_idx)) |arg| { + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + help_menu = true; + } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { + steps_menu = true; + } else if (mem.eql(u8, arg, "--print-configuration")) { + print_configuration = true; + } else if (mem.eql(u8, arg, "--verbose")) { + verbose = true; + } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { + override_install_prefix = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { + override_lib_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { + override_bin_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-include-dir")) { + override_include_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--sysroot")) { + sysroot = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--maxrss")) { + const max_rss_text = nextArgOrFatal(args, &arg_idx); + max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| + fatal("invalid byte size: '{s}': {t}", .{ max_rss_text, err }); + } else if (mem.eql(u8, arg, "--skip-oom-steps")) { + skip_oom_steps = true; + } else if (mem.eql(u8, arg, "--test-timeout")) { + const units: []const struct { []const u8, u64 } = &.{ + .{ "ns", 1 }, + .{ "nanosecond", 1 }, + .{ "us", std.time.ns_per_us }, + .{ "microsecond", std.time.ns_per_us }, + .{ "ms", std.time.ns_per_ms }, + .{ "millisecond", std.time.ns_per_ms }, + .{ "s", std.time.ns_per_s }, + .{ "second", std.time.ns_per_s }, + .{ "m", std.time.ns_per_min }, + .{ "minute", std.time.ns_per_min }, + .{ "h", std.time.ns_per_hour }, + .{ "hour", std.time.ns_per_hour }, + }; + const timeout_str = nextArgOrFatal(args, &arg_idx); + const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( + "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)", + .{timeout_str}, + ); + const num_str = timeout_str[0 .. num_end_idx + 1]; + const unit_str = timeout_str[num_end_idx + 1 ..]; + const unit_factor: f64 = for (units) |unit_and_factor| { + if (std.mem.eql(u8, unit_str, unit_and_factor[0])) { + break @floatFromInt(unit_and_factor[1]); + } + } else fatal( + "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)", + .{ timeout_str, unit_str }, + ); + const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal( + "invalid timeout '{s}': invalid number '{s}' ({t})", + .{ timeout_str, num_str, err }, + ); + test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); + } else if (mem.eql(u8, arg, "--search-prefix")) { + try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); + } else if (mem.eql(u8, arg, "--libc")) { + libc_file = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--color")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); + color = std.meta.stringToEnum(Color, next_arg) orelse { + fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ + arg, next_arg, + }); + }; + } else if (mem.eql(u8, arg, "--error-style")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected style after '{s}'", .{arg}); + error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { + fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + }; + } else if (mem.eql(u8, arg, "--multiline-errors")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected style after '{s}'", .{arg}); + multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { + fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + }; + } else if (mem.eql(u8, arg, "--summary")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg}); + summary = std.meta.stringToEnum(Summary, next_arg) orelse { + fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{ + arg, next_arg, + }); + }; + } else if (mem.eql(u8, arg, "--seed")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u32 after '{s}'", .{arg}); + graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { + fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {t}", .{ next_arg, err }); + }; + } else if (mem.eql(u8, arg, "--debounce")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u16 after '{s}'", .{arg}); + debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { + fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{ + next_arg, err, + }); + }; + } else if (mem.eql(u8, arg, "--webui")) { + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; + } else if (mem.startsWith(u8, arg, "--webui=")) { + const addr_str = arg["--webui=".len..]; + if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); + webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { + fatal("invalid web UI address '{s}': {t}", .{ addr_str, err }); + }; + } else if (mem.eql(u8, arg, "--debug-log")) { + const next_arg = nextArgOrFatal(args, &arg_idx); + try debug_log_scopes.append(arena, next_arg); + } else if (mem.eql(u8, arg, "--debug-pkg-config")) { + debug_pkg_config = true; + } else if (mem.eql(u8, arg, "--debug-rt")) { + graph.debug_compiler_runtime_libs = .Debug; + } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { + graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse + fatal("unrecognized optimization mode: {s}", .{rest}); + } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { + // --glibc-runtimes was the old name of the flag; kept for compatibility for now. + libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--watch")) { + watch = true; + } else if (mem.eql(u8, arg, "--time-report")) { + graph.time_report = true; + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; + } else if (mem.eql(u8, arg, "--fuzz")) { + fuzz = .{ .forever = undefined }; + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; + } else if (mem.startsWith(u8, arg, "--fuzz=")) { + const value = arg["--fuzz=".len..]; + if (value.len == 0) fatal("missing argument to --fuzz", .{}); + + const unit: u8 = value[value.len - 1]; + const digits = switch (unit) { + '0'...'9' => value, + 'K', 'M', 'G' => value[0 .. value.len - 1], + else => fatal( + "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", + .{}, + ), + }; + + const amount = std.fmt.parseInt(u64, digits, 10) catch { + fatal( + "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", + .{}, + ); + }; + + const normalized_amount = std.math.mul(u64, amount, switch (unit) { + else => unreachable, + '0'...'9' => 1, + 'K' => 1000, + 'M' => 1_000_000, + 'G' => 1_000_000_000, + }) catch fatal("fuzzing limit amount overflows u64", .{}); + + fuzz = .{ + .limit = .{ + .amount = normalized_amount, + }, + }; + } else if (mem.eql(u8, arg, "-fincremental")) { + graph.incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + graph.incremental = false; + } else if (mem.eql(u8, arg, "-fwine")) { + enable_wine = true; + } else if (mem.eql(u8, arg, "-fno-wine")) { + enable_wine = false; + } else if (mem.eql(u8, arg, "-fqemu")) { + enable_qemu = true; + } else if (mem.eql(u8, arg, "-fno-qemu")) { + enable_qemu = false; + } else if (mem.eql(u8, arg, "-fwasmtime")) { + enable_wasmtime = true; + } else if (mem.eql(u8, arg, "-fno-wasmtime")) { + enable_wasmtime = false; + } else if (mem.eql(u8, arg, "-frosetta")) { + enable_rosetta = true; + } else if (mem.eql(u8, arg, "-fno-rosetta")) { + enable_rosetta = false; + } else if (mem.eql(u8, arg, "-fdarling")) { + enable_darling = true; + } else if (mem.eql(u8, arg, "-fno-darling")) { + enable_darling = false; + } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { + graph.allow_so_scripts = true; + } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { + graph.allow_so_scripts = false; + } else if (mem.eql(u8, arg, "-freference-trace")) { + reference_trace = 256; + } else if (mem.startsWith(u8, arg, "-freference-trace=")) { + const num = arg["-freference-trace=".len..]; + reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + std.debug.print("unable to parse reference_trace count '{s}': {t}", .{ num, err }); + process.exit(1); + }; + } else if (mem.eql(u8, arg, "-fno-reference-trace")) { + reference_trace = null; + } else if (mem.cutPrefix(u8, arg, "-j")) |text| { + const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| + fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); + if (n < 1) fatal("number of jobs must be at least 1", .{}); + threaded.setAsyncLimit(.limited(n)); + graph.max_jobs = n; + } else if (mem.eql(u8, arg, "--")) { + run_args = argsRest(args, arg_idx); + break; + } else { + fatalWithHint("unrecognized argument: '{s}'", .{arg}); + } + } else { + try step_names.append(arena, arg); + } + } + + const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); + const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); + + graph.stderr_mode = switch (color) { + .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), + .on => .escape_codes, + .off => .no_color, + }; + + const scanned_config: ScannedConfig = sc: { + const configuration = c: { + var file = cwd.openFile(io, configure_path, .{}) catch |err| + fatal("failed to open configuration file {s}: {t}", .{ configure_path, err }); + defer file.close(io); + break :c Configuration.loadFile(arena, io, file) catch |err| + fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); + }; + var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; + for (configuration.steps, 0..) |*conf_step, step_index_usize| { + const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); + const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]); + if (flags.tag == .top_level) { + const name = step_index.ptr(&configuration).name.slice(&configuration); + try top_level_steps.put(arena, name, step_index); + } + } + break :sc .{ + .configuration = configuration, + .top_level_steps = top_level_steps, + }; + }; + + if (help_menu) { + var w = initStdoutWriter(io); + scanned_config.printUsage(&graph, w) catch |err| switch (err) { + error.WriteFailed => return stdout_writer_allocation.err.?, + else => |e| return e, + }; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } else if (steps_menu) { + var w = initStdoutWriter(io); + scanned_config.printSteps(&graph, w) catch |err| switch (err) { + error.WriteFailed => return stdout_writer_allocation.err.?, + else => |e| return e, + }; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } else if (print_configuration) { + var w = initStdoutWriter(io); + scanned_config.print(w) catch return stdout_writer_allocation.err.?; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } + + if (webui_listen != null) { + if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); + if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); + } + + const main_progress_node = std.Progress.start(io, .{ + .disable_printing = (color == .off), + }); + defer main_progress_node.end(); + + const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ + .root_dir = .cwd(), + .sub_path = try Io.Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), + } else if (override_install_prefix) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else .{ + .root_dir = build_root_directory, + .sub_path = "zig-out", + }; + + const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "lib"); + + const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "bin"); + + const install_include_path: Path = if (override_include_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "include"); + + var maker: Maker = .{ + .gpa = gpa, + .graph = &graph, + .scanned_config = &scanned_config, + .install_paths = .{ + .prefix = install_prefix_path, + .lib = install_lib_path, + .bin = install_bin_path, + .include = install_include_path, + }, + .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), + + .available_rss = max_rss, + .max_rss_is_default = false, + .max_rss_mutex = .init, + .skip_oom_steps = skip_oom_steps, + .unit_test_timeout_ns = test_timeout_ns, + + .watch = watch, + .web_server = undefined, // set after `prepare` + .memory_blocked_steps = .empty, + .step_stack = .empty, + + .error_style = error_style, + .multiline_errors = multiline_errors, + .summary = summary orelse if (watch or webui_listen != null) .line else .failures, + }; + defer { + maker.memory_blocked_steps.deinit(gpa); + maker.step_stack.deinit(gpa); + } + + if (maker.available_rss == 0) { + maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); + maker.max_rss_is_default = true; + } + + maker.prepare(step_names.items) catch |err| switch (err) { + error.DependencyLoopDetected, error.InsufficientMemory => { + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(1); + }, + else => |e| return e, + }; + + var w: Watch = w: { + if (!watch) break :w undefined; + if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); + break :w try .init(graph.cache.cwd, &scanned_config.configuration, maker.steps); + }; + + const now = Io.Clock.Timestamp.now(io, .awake); + + maker.web_server = if (webui_listen) |listen_address| ws: { + if (builtin.single_threaded) unreachable; // `fatal` above + break :ws .init(.{ + .gpa = gpa, + .graph = &graph, + .all_steps = maker.step_stack.keys(), + .root_prog_node = main_progress_node, + .watch = watch, + .listen_address = listen_address, + .base_timestamp = now, + .configuration = &scanned_config.configuration, + }); + } else null; + + if (maker.web_server) |*ws| { + ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); + } + + rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H"); + }) { + if (maker.web_server) |*ws| ws.startBuild(); + + try maker.makeStepNames(step_names.items, main_progress_node, fuzz); + + if (maker.web_server) |*web_server| { + if (fuzz) |mode| if (mode != .forever) fatal( + "error: limited fuzzing is not implemented yet for --webui", + .{}, + ); + + web_server.finishBuild(.{ .fuzz = fuzz != null }); + } + + if (maker.web_server) |*ws| { + const c = &scanned_config.configuration; + assert(!watch); // fatal error after CLI parsing + while (true) switch (try ws.wait()) { + .rebuild => { + for (maker.step_stack.keys()) |step_index| { + const step = maker.stepByIndex(step_index); + step.state = .precheck_done; + const deps = step_index.ptr(c).deps.slice(c); + step.pending_deps = @intCast(deps.len); + step.reset(gpa); + } + continue :rebuild; + }, + }; + } + + // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. + if (!Watch.have_impl) unreachable; + + try w.update(gpa, maker.step_stack.keys()); + + // Wait until a file system notification arrives. Read all such events + // until the buffer is empty. Then wait for a debounce interval, resetting + // if any more events come in. After the debounce interval has passed, + // trigger a rebuild on all steps with modified inputs, as well as their + // recursive dependants. + var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; + const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ + w.dir_count, countSubProcesses(maker.steps, maker.step_stack.keys()), + }) catch &caption_buf; + var debouncing_node = main_progress_node.start(caption, 0); + var in_debounce = false; + while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { + .timeout => { + assert(in_debounce); + debouncing_node.end(); + markFailedStepsDirty(gpa, maker.steps, maker.step_stack.keys()); + continue :rebuild; + }, + .dirty => if (!in_debounce) { + in_debounce = true; + debouncing_node.end(); + debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); + }, + .clean => {}, + }; + } +} + +fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void { + for (all_steps) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + switch (step.state) { + .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), + else => continue, + } + } + // Now that all dirty steps have been found, the remaining steps that + // succeeded from last run shall be marked "cached". + for (all_steps) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + switch (step.state) { + .success => step.result_cached = true, + else => continue, + } + } +} + +fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize { + var count: usize = 0; + for (all_steps) |step_index| { + const s = &make_steps[@intFromEnum(step_index)]; + count += @intFromBool(s.getZigProcess() != null); + } + return count; +} + +const InstallPaths = struct { + prefix: Path, + lib: Path, + bin: Path, + include: Path, +}; + +fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step { + return &maker.steps[@intFromEnum(i)]; +} + +fn prepare(maker: *Maker, step_names: []const []const u8) !void { + const gpa = maker.gpa; + const graph = maker.graph; + const arena = graph.arena; + const seed: u32 = graph.random_seed; + const step_stack = &maker.step_stack; + const c = &maker.scanned_config.configuration; + + @memset(maker.steps, .{}); + + if (step_names.len == 0) { + try step_stack.put(gpa, c.default_step, {}); + } else { + try step_stack.ensureUnusedCapacity(gpa, step_names.len); + for (0..step_names.len) |i| { + const step_name = step_names[step_names.len - i - 1]; + const s = maker.scanned_config.top_level_steps.get(step_name) orelse { + log.info("to list available steps: zig build -l", .{}); + fatal("no such step: {s}", .{step_name}); + }; + step_stack.putAssumeCapacity(s, {}); + } + } + + const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys()); + + var rng = std.Random.DefaultPrng.init(seed); + const rand = rng.random(); + rand.shuffle(Configuration.Step.Index, starting_steps); + + for (starting_steps) |s| { + try constructGraphAndCheckForDependencyLoop(gpa, c, maker.steps, s, &maker.step_stack, rand); + } + + { + // Check that we have enough memory to complete the build. + var any_problems = false; + var max_needed: usize = 0; + for (step_stack.keys()) |step_index| { + const make_step = maker.stepByIndex(step_index); + const conf_step = step_index.ptr(c); + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss == 0) continue; + max_needed = @max(max_needed, max_rss); + if (max_rss > maker.available_rss) { + if (maker.skip_oom_steps) { + make_step.state = .skipped_oom; + for (make_step.dependants.items) |dependant| { + maker.stepByIndex(dependant).pending_deps -= 1; + } + } else { + log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ + conf_step.owner.depPrefixSlice(c), + conf_step.name.slice(c), + max_rss, + maker.available_rss, + }); + any_problems = true; + } + } + } + if (any_problems) { + if (maker.max_rss_is_default) { + std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ + max_needed, + }); + } + return error.InsufficientMemory; + } + } +} + +fn makeStepNames( + maker: *Maker, + step_names: []const []const u8, + parent_prog_node: std.Progress.Node, + fuzz: ?Fuzz.Mode, +) !void { + const graph = maker.graph; + const gpa = maker.gpa; + const io = graph.io; + const step_stack = &maker.step_stack; + const top_level_steps = &maker.scanned_config.top_level_steps; + const c = &maker.scanned_config.configuration; + + { + // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, + // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking + // a step is initial when it actually became ready due to an earlier initial step. + var initial_set: std.ArrayList(Configuration.Step.Index) = .empty; + defer initial_set.deinit(gpa); + try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); + for (step_stack.keys()) |step_index| { + const s = maker.stepByIndex(step_index); + if (s.state == .precheck_done and s.pending_deps == 0) { + initial_set.appendAssumeCapacity(step_index); + } + } + + const step_prog = parent_prog_node.start("steps", step_stack.count()); + defer step_prog.end(); + + var group: Io.Group = .init; + defer group.cancel(io); + // Start working on all of the initial steps... + for (initial_set.items) |step_index| try stepReady(maker, &group, step_index, step_prog); + // ...and `makeStep` will trigger every other step when their last dependency finishes. + try group.await(io); + } + + assert(maker.memory_blocked_steps.items.len == 0); + + var test_pass_count: usize = 0; + var test_skip_count: usize = 0; + var test_fail_count: usize = 0; + var test_crash_count: usize = 0; + var test_timeout_count: usize = 0; + + var test_count: usize = 0; + + var success_count: usize = 0; + var skipped_count: usize = 0; + var failure_count: usize = 0; + var pending_count: usize = 0; + var total_compile_errors: usize = 0; + + var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); + defer cleanup_task.await(io); + + for (step_stack.keys()) |step_index| { + const make_step = maker.stepByIndex(step_index); + test_pass_count += make_step.test_results.passCount(); + test_skip_count += make_step.test_results.skip_count; + test_fail_count += make_step.test_results.fail_count; + test_crash_count += make_step.test_results.crash_count; + test_timeout_count += make_step.test_results.timeout_count; + + test_count += make_step.test_results.test_count; + + switch (make_step.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + .dependency_failure => pending_count += 1, + .success => success_count += 1, + .skipped, .skipped_oom => skipped_count += 1, + .failure => { + failure_count += 1; + const compile_errors_len = make_step.result_error_bundle.errorMessageCount(); + if (compile_errors_len > 0) { + total_compile_errors += compile_errors_len; + } + }, + } + } + + if (fuzz) |mode| blk: { + switch (builtin.os.tag) { + // Current implementation depends on two things that need to be ported to Windows: + // * Memory-mapping to share data between the fuzzer and build runner. + // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving + // many addresses to source locations). + .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), + else => {}, + } + if (@bitSizeOf(usize) != 64) { + // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, + // being compatible with file system's u64 return value. This is not the case + // on 32-bit platforms. + // Affects or affected by issues #5185, #22523, and #22464. + fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); + } + + switch (mode) { + .forever => break :blk, + .limit => {}, + } + + assert(mode == .limit); + var f = Fuzz.init( + gpa, + io, + step_stack.keys(), + parent_prog_node, + mode, + ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); + defer f.deinit(); + + f.start(); + try f.waitAndPrintReport(); + } + + // Every test has a state + assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); + + if (failure_count == 0) { + std.Progress.setStatus(.success); + } else { + std.Progress.setStatus(.failure); + } + + summary: { + switch (maker.summary) { + .all, .new, .line => {}, + .failures => if (failure_count == 0) break :summary, + .none => break :summary, + } + + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + const t = stderr.terminal(); + const w = &stderr.file_writer.interface; + + const total_count = success_count + failure_count + pending_count + skipped_count; + t.setColor(.cyan) catch {}; + t.setColor(.bold) catch {}; + w.writeAll("Build Summary: ") catch {}; + t.setColor(.reset) catch {}; + w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; + { + t.setColor(.dim) catch {}; + var first = true; + if (skipped_count > 0) { + w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; + first = false; + } + if (failure_count > 0) { + w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; + first = false; + } + if (!first) w.writeByte(')') catch {}; + t.setColor(.reset) catch {}; + } + + if (test_count > 0) { + w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; + t.setColor(.dim) catch {}; + var first = true; + if (test_skip_count > 0) { + w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; + first = false; + } + if (test_fail_count > 0) { + w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; + first = false; + } + if (test_crash_count > 0) { + w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; + first = false; + } + if (test_timeout_count > 0) { + w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; + first = false; + } + if (!first) w.writeByte(')') catch {}; + t.setColor(.reset) catch {}; + } + + w.writeAll("\n") catch {}; + + if (maker.summary == .line) break :summary; + + // Print a fancy tree with build results. + var step_stack_copy = try step_stack.clone(gpa); + defer step_stack_copy.deinit(gpa); + + var print_node: PrintNode = .{ .parent = null }; + if (step_names.len == 0) { + print_node.last = true; + printTreeStep(maker, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, + }; + } else { + const last_index = if (maker.summary == .all) top_level_steps.count() else blk: { + var i: usize = step_names.len; + while (i > 0) { + i -= 1; + const step_index = top_level_steps.get(step_names[i]).?; + const step = maker.stepByIndex(step_index); + const found = switch (maker.summary) { + .all, .line, .none => unreachable, + .failures => step.state != .success, + .new => !step.result_cached, + }; + if (found) break :blk i; + } + break :blk top_level_steps.count(); + }; + for (step_names, 0..) |step_name, i| { + const step_index = top_level_steps.get(step_name).?; + print_node.last = i + 1 == last_index; + printTreeStep(maker, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, + }; + } + } + w.writeByte('\n') catch {}; + } + + if (maker.watch or maker.web_server != null) return; + + // Perhaps in the future there could be an Advanced Options flag such as + // --debug-build-runner-leaks which would make this code return instead of + // calling exit. + + const code: u8 = code: { + if (failure_count == 0) break :code 0; // success + if (maker.error_style.verboseContext()) break :code 1; // failure; print build command + break :code 2; // failure; do not print build command + }; + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(code); +} + +fn stepReady( + maker: *Maker, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, +) Io.Cancelable!void { + const graph = maker.graph; + const io = graph.io; + const c = &maker.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + if (max_rss != 0) { + try maker.max_rss_mutex.lock(io); + defer maker.max_rss_mutex.unlock(io); + if (maker.available_rss < max_rss) { + // Running this step right now could possibly exceed the allotted RSS. + maker.memory_blocked_steps.append(maker.gpa, step_index) catch + @panic("TODO eliminate memory allocation here"); + return; + } + maker.available_rss -= max_rss; + } + group.async(io, makeStep, .{ maker, group, step_index, root_prog_node }); +} + +/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready +/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must +/// have already subtracted this value from `maker.available_rss`. This function will release the RSS +/// claim (i.e. add `s.max_rss` back into `maker.available_rss`) and queue any viable memory-blocked +/// steps after "make" completes for `s`. +fn makeStep( + maker: *Maker, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, +) Io.Cancelable!void { + const graph = maker.graph; + const io = graph.io; + const gpa = maker.gpa; + const c = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const step_name = conf_step.name.slice(c); + const deps = conf_step.deps.slice(c); + const make_step = maker.stepByIndex(step_index); + + { + const step_prog_node = root_prog_node.start(step_name, 0); + defer step_prog_node.end(); + + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip); + + const new_state: Step.State = for (deps) |dep_index| { + const dep_make_step = maker.stepByIndex(dep_index); + switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .failure, + .dependency_failure, + .skipped_oom, + => break .dependency_failure, + + .success, .skipped => {}, + } + } else if (make_step.make(.{ + .progress_node = step_prog_node, + .watch = maker.watch, + .web_server = if (maker.web_server) |*ws| ws else null, + .unit_test_timeout_ns = maker.unit_test_timeout_ns, + .gpa = gpa, + })) state: { + break :state .success; + } else |err| switch (err) { + error.MakeFailed => .failure, + error.MakeSkipped => .skipped, + }; + + @atomicStore(Step.State, &make_step.state, new_state, .monotonic); + + switch (new_state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .failure, + .dependency_failure, + .skipped_oom, + => { + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure); + std.Progress.setStatus(.failure_working); + }, + + .success, + .skipped, + => { + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success); + }, + } + } + + // No matter the result, we want to display error/warning messages. + if (make_step.result_error_bundle.errorMessageCount() > 0 or + make_step.result_error_msgs.items.len > 0 or + make_step.result_stderr.len > 0) + { + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + printErrorMessages(gpa, c, maker.steps, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) { + error.Canceled => |e| return e, + error.WriteFailed => switch (stderr.file_writer.err.?) { + error.Canceled => |e| return e, + else => {}, + }, + else => {}, + }; + } + + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss != 0) { + var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty; + defer dispatch_set.deinit(gpa); + + // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` + // as a staging buffer to avoid recursing into `makeStep` while `maker.max_rss_mutex` is held. + { + try maker.max_rss_mutex.lock(io); + defer maker.max_rss_mutex.unlock(io); + maker.available_rss += max_rss; + dispatch_set.ensureUnusedCapacity(gpa, maker.memory_blocked_steps.items.len) catch + @panic("TODO eliminate memory allocation here"); + while (maker.memory_blocked_steps.getLast()) |candidate_index| { + const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes(); + if (maker.available_rss < candidate_max_rss) break; + assert(maker.memory_blocked_steps.pop() == candidate_index); + dispatch_set.appendAssumeCapacity(candidate_index); + } + } + for (dispatch_set.items) |candidate| { + group.async(io, makeStep, .{ maker, group, candidate, root_prog_node }); + } + } + + for (make_step.dependants.items) |dependant_index| { + const dependant = maker.stepByIndex(dependant_index); + // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. + if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { + try stepReady(maker, group, dependant_index, root_prog_node); + } + } +} + +fn printTreeStep( + maker: *const Maker, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + parent_node: *PrintNode, + step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), +) !void { + const writer = stderr.writer; + const first = step_stack.swapRemove(step_index); + const summary = maker.summary; + const c = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const make_step = maker.stepByIndex(step_index); + const skip = switch (summary) { + .none, .line => unreachable, + .all => false, + .new => make_step.result_cached, + .failures => make_step.state == .success, + }; + if (skip) return; + try printPrefix(parent_node, stderr); + + if (parent_node.parent != null) { + if (parent_node.last) { + try printChildNodePrefix(stderr); + } else { + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ + else => "+- ", + }); + } + } + + if (!first) try stderr.setColor(.dim); + + // dep_prefix omitted here because it is redundant with the tree. + try writer.writeAll(conf_step.name.slice(c)); + + const deps = conf_step.deps.slice(c); + + if (first) { + try printStepStatus(maker, step_index, stderr); + + const last_index = if (summary == .all) deps.len -| 1 else blk: { + var i: usize = deps.len; + while (i > 0) { + i -= 1; + + const dep_index = deps[i]; + const dep = maker.stepByIndex(dep_index); + const found = switch (summary) { + .all, .line, .none => unreachable, + .failures => dep.state != .success, + .new => !dep.result_cached, + }; + if (found) break :blk i; + } + break :blk deps.len -| 1; + }; + for (deps, 0..) |dep, i| { + var print_node: PrintNode = .{ + .parent = parent_node, + .last = i == last_index, + }; + try printTreeStep(maker, dep, stderr, &print_node, step_stack); + } + } else { + if (deps.len == 0) { + try writer.writeAll(" (reused)\n"); + } else { + try writer.print(" (+{d} more reused dependencies)\n", .{deps.len}); + } + try stderr.setColor(.reset); + } +} + +fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { + const s = maker.stepByIndex(step_index); + const writer = stderr.writer; + switch (s.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .dependency_failure => { + try stderr.setColor(.dim); + try writer.writeAll(" transitive failure\n"); + try stderr.setColor(.reset); + }, + + .success => { + try stderr.setColor(.green); + if (s.result_cached) { + try writer.writeAll(" cached"); + } else if (s.test_results.test_count > 0) { + const pass_count = s.test_results.passCount(); + assert(s.test_results.test_count == pass_count + s.test_results.skip_count); + try writer.print(" {d} pass", .{pass_count}); + if (s.test_results.skip_count > 0) { + try stderr.setColor(.reset); + try writer.writeAll(", "); + try stderr.setColor(.yellow); + try writer.print("{d} skip", .{s.test_results.skip_count}); + } + try stderr.setColor(.reset); + try writer.print(" ({d} total)", .{s.test_results.test_count}); + } else { + try writer.writeAll(" success"); + } + try stderr.setColor(.reset); + if (s.result_duration_ns) |ns| { + try stderr.setColor(.dim); + if (ns >= std.time.ns_per_min) { + try writer.print(" {d}m", .{ns / std.time.ns_per_min}); + } else if (ns >= std.time.ns_per_s) { + try writer.print(" {d}s", .{ns / std.time.ns_per_s}); + } else if (ns >= std.time.ns_per_ms) { + try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); + } else if (ns >= std.time.ns_per_us) { + try writer.print(" {d}us", .{ns / std.time.ns_per_us}); + } else { + try writer.print(" {d}ns", .{ns}); + } + try stderr.setColor(.reset); + } + if (s.result_peak_rss != 0) { + const rss = s.result_peak_rss; + try stderr.setColor(.dim); + if (rss >= 1000_000_000) { + try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); + } else if (rss >= 1000_000) { + try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); + } else if (rss >= 1000) { + try writer.print(" MaxRSS:{d}K", .{rss / 1000}); + } else { + try writer.print(" MaxRSS:{d}B", .{rss}); + } + try stderr.setColor(.reset); + } + try writer.writeAll("\n"); + }, + .skipped => { + try stderr.setColor(.yellow); + try writer.writeAll(" skipped\n"); + try stderr.setColor(.reset); + }, + .skipped_oom => { + const c = &maker.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + try stderr.setColor(.yellow); + try writer.writeAll(" skipped (not enough memory)"); + try stderr.setColor(.dim); + try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ + max_rss, maker.available_rss, + }); + try stderr.setColor(.reset); + }, + .failure => { + try printStepFailure(maker.steps, step_index, stderr, false); + try stderr.setColor(.reset); + }, + } +} + +fn printStepFailure( + make_steps: []Step, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + dim: bool, +) !void { + const w = stderr.writer; + const s = &make_steps[@intFromEnum(step_index)]; + if (s.result_error_bundle.errorMessageCount() > 0) { + try stderr.setColor(.red); + try w.print(" {d} errors\n", .{ + s.result_error_bundle.errorMessageCount(), + }); + } else if (!s.test_results.isSuccess()) { + // These first values include all of the test "statuses". Every test is either passsed, + // skipped, failed, crashed, or timed out. + try stderr.setColor(.green); + try w.print(" {d} pass", .{s.test_results.passCount()}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + if (s.test_results.skip_count > 0) { + try w.writeAll(", "); + try stderr.setColor(.yellow); + try w.print("{d} skip", .{s.test_results.skip_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + if (s.test_results.fail_count > 0) { + try w.writeAll(", "); + try stderr.setColor(.red); + try w.print("{d} fail", .{s.test_results.fail_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + if (s.test_results.crash_count > 0) { + try w.writeAll(", "); + try stderr.setColor(.red); + try w.print("{d} crash", .{s.test_results.crash_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + if (s.test_results.timeout_count > 0) { + try w.writeAll(", "); + try stderr.setColor(.red); + try w.print("{d} timeout", .{s.test_results.timeout_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + try w.print(" ({d} total)", .{s.test_results.test_count}); + + // Memory leaks are intentionally written after the total, because is isn't a test *status*, + // but just a flag that any tests -- even passed ones -- can have. We also use a different + // separator, so it looks like: + // 2 pass, 1 skip, 2 fail (5 total); 2 leaks + if (s.test_results.leak_count > 0) { + try w.writeAll("; "); + try stderr.setColor(.red); + try w.print("{d} leaks", .{s.test_results.leak_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + + // It's usually not helpful to know how many error logs there were because they tend to + // just come with other errors (e.g. crashes and leaks print stack traces, and clean + // failures print error traces). So only mention them if they're the only thing causing + // the failure. + const show_err_logs: bool = show: { + var alt_results = s.test_results; + alt_results.log_err_count = 0; + break :show alt_results.isSuccess(); + }; + if (show_err_logs) { + try w.writeAll("; "); + try stderr.setColor(.red); + try w.print("{d} error logs", .{s.test_results.log_err_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + + try w.writeAll("\n"); + } else if (s.result_error_msgs.items.len > 0) { + try stderr.setColor(.red); + try w.writeAll(" failure\n"); + } else { + assert(s.result_stderr.len > 0); + try stderr.setColor(.red); + try w.writeAll(" w\n"); + } +} + +const PrintNode = struct { + parent: ?*PrintNode, + last: bool = false, +}; + +fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { + const parent = node.parent orelse return; + const writer = stderr.writer; + if (parent.parent == null) return; + try printPrefix(parent, stderr); + if (parent.last) { + try writer.writeAll(" "); + } else { + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ + else => "| ", + }); + } +} + +fn printChildNodePrefix(stderr: Io.Terminal) !void { + try stderr.writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ + else => "+- ", + }); +} + +/// Traverse the dependency graph depth-first and make it undirected by having +/// steps know their dependants (they only know dependencies at start). +/// Along the way, check that there is no dependency loop, and record the steps +/// in traversal order in `step_stack`. +/// Each step has its dependencies traversed in random order, this accomplishes +/// two things: +/// - `step_stack` will be in randomized-depth-first order, so the build runner +/// spawns initial steps in a random order +/// - each step's `dependants` list is also filled in a random order, so that +/// when it finishes executing in `makeStep`, it spawns next steps to run in +/// random order +fn constructGraphAndCheckForDependencyLoop( + gpa: Allocator, + c: *const Configuration, + steps: []Step, + step_index: Configuration.Step.Index, + step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), + rand: std.Random, +) error{ DependencyLoopDetected, OutOfMemory }!void { + const make_step: *Step = &steps[@intFromEnum(step_index)]; + switch (make_step.state) { + .precheck_started => { + log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)}); + return error.DependencyLoopDetected; + }, + .precheck_unstarted => { + make_step.state = .precheck_started; + + const step = step_index.ptr(c); + const dependencies = step.deps.slice(c); + try step_stack.ensureUnusedCapacity(gpa, dependencies.len); + + // We dupe to avoid shuffling the steps in the summary, it depends + // on dependencies' order. + const deps = try gpa.dupe(Configuration.Step.Index, dependencies); + defer gpa.free(deps); + + rand.shuffle(Configuration.Step.Index, deps); + + for (deps) |dep| { + const dep_step: *Step = &steps[@intFromEnum(dep)]; + try step_stack.put(gpa, dep, {}); + try dep_step.dependants.append(gpa, step_index); + constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) { + error.DependencyLoopDetected => { + log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); + return err; + }, + else => return err, + }; + } + + make_step.state = .precheck_done; + make_step.pending_deps = @intCast(dependencies.len); + }, + .precheck_done => {}, + + // These don't happen until we actually run the step graph. + .dependency_failure => unreachable, + .success => unreachable, + .failure => unreachable, + .skipped => unreachable, + .skipped_oom => unreachable, + } +} + +pub fn printErrorMessages( + gpa: Allocator, + c: *const Configuration, + make_steps: []Step, + failing_step_index: Configuration.Step.Index, + options: std.zig.ErrorBundle.RenderOptions, + stderr: Io.Terminal, + error_style: ErrorStyle, + multiline_errors: MultilineErrors, +) !void { + const writer = stderr.writer; + if (error_style.verboseContext()) { + // Provide context for where these error messages are coming from by + // printing the corresponding Step subtree. + var step_stack: std.ArrayList(Configuration.Step.Index) = .empty; + defer step_stack.deinit(gpa); + try step_stack.append(gpa, failing_step_index); + while (true) { + const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])]; + if (last_step.dependants.items.len == 0) break; + try step_stack.append(gpa, last_step.dependants.items[0]); + } + + // Now, `step_stack` has the subtree that we want to print, in reverse order. + try stderr.setColor(.dim); + var indent: usize = 0; + while (step_stack.pop()) |step_index| : (indent += 1) { + if (indent > 0) { + try writer.splatByteAll(' ', (indent - 1) * 3); + try printChildNodePrefix(stderr); + } + + try writer.writeAll(step_index.ptr(c).name.slice(c)); + + if (step_index == failing_step_index) { + try printStepFailure(make_steps, step_index, stderr, true); + } else { + try writer.writeAll("\n"); + } + } + try stderr.setColor(.reset); + } else { + // Just print the failing step itself. + try stderr.setColor(.dim); + try writer.writeAll(failing_step_index.ptr(c).name.slice(c)); + try printStepFailure(make_steps, failing_step_index, stderr, true); + try stderr.setColor(.reset); + } + + const failing_step = &make_steps[@intFromEnum(failing_step_index)]; + + if (failing_step.result_stderr.len > 0) { + try writer.writeAll(failing_step.result_stderr); + if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { + try writer.writeAll("\n"); + } + } + + try failing_step.result_error_bundle.renderToTerminal(options, stderr); + + for (failing_step.result_error_msgs.items) |msg| { + try stderr.setColor(.red); + try writer.writeAll("error:"); + try stderr.setColor(.reset); + if (std.mem.indexOfScalar(u8, msg, '\n') == null) { + try writer.print(" {s}\n", .{msg}); + } else switch (multiline_errors) { + .indent => { + var it = std.mem.splitScalar(u8, msg, '\n'); + try writer.print(" {s}\n", .{it.first()}); + while (it.next()) |line| { + try writer.print(" {s}\n", .{line}); + } + }, + .newline => try writer.print("\n{s}\n", .{msg}), + .none => try writer.print(" {s}\n", .{msg}), + } + } + + if (error_style.verboseContext()) { + if (failing_step.result_failed_command) |cmd_str| { + try stderr.setColor(.red); + try writer.writeAll("failed command: "); + try stderr.setColor(.reset); + try writer.writeAll(cmd_str); + try writer.writeByte('\n'); + } + } + + try writer.writeByte('\n'); +} + +fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { + if (idx.* >= args.len) return null; + defer idx.* += 1; + return args[idx.*]; +} + +fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { + return nextArg(args, idx) orelse { + fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); + }; +} + +fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { + const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); + if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); + const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); + return arg; +} + +fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { + if (idx >= args.len) return null; + return args[idx..]; +} + +const Color = std.zig.Color; +const ErrorStyle = enum { + verbose, + minimal, + verbose_clear, + minimal_clear, + fn verboseContext(s: ErrorStyle) bool { + return switch (s) { + .verbose, .verbose_clear => true, + .minimal, .minimal_clear => false, + }; + } + fn clearOnUpdate(s: ErrorStyle) bool { + return switch (s) { + .verbose, .minimal => false, + .verbose_clear, .minimal_clear => true, + }; + } +}; +const MultilineErrors = enum { indent, newline, none }; +const Summary = enum { all, new, failures, line, none }; + +fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { + log.info("to access the help menu: zig build -h", .{}); + fatal(f, args); +} + +fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { + for (steps) |step_index| { + if (true) @panic("TODO"); + const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; + if (wf.mode != .tmp) continue; + const path = wf.generated_directory.path orelse continue; + Io.Dir.cwd().deleteTree(io, path) catch |err| { + log.warn("failed to delete {s}: {t}", .{ path, err }); + }; + } +} + +var stdio_buffer_allocation: [256]u8 = undefined; +var stdout_writer_allocation: Io.File.Writer = undefined; + +fn initStdoutWriter(io: Io) *Writer { + stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); + return &stdout_writer_allocation.interface; +} + +const ScannedConfig = struct { + configuration: Configuration, + top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), + + fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { + const c = &sc.configuration; + var serializer: std.zon.Serializer = .{ .writer = w }; + var s = try serializer.beginStruct(.{}); + + try s.field("default_step", @intFromEnum(c.default_step), .{}); + { + var ss = try s.beginStructField("top_level_steps", .{}); + for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| { + try ss.field(name, @intFromEnum(step), .{}); + } + try ss.end(); + } + + try s.end(); + } + + fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + const c = &sc.configuration; + for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| { + const step = step_index.ptr(c); + const decorated_name = if (step_index == c.default_step) + try fmt.allocPrint(arena, "{s} (default)", .{name}) + else + name; + const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); + const description = top_level.description.slice(c); + try w.print(" {s:<28} {s}\n", .{ decorated_name, description }); + } + } + + fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + + try w.print( + \\Usage: {s} build [steps] [options] + \\ + \\Steps: + \\ + , .{graph.zig_exe}); + try printSteps(sc, graph, w); + try w.writeAll( + \\ + \\Project-Specific Options: + \\ + ); + + const available_options = sc.configuration.available_options; + if (available_options.len == 0) { + try w.print(" (none)\n", .{}); + } else { + for (available_options) |option| { + const name = option.name.slice(&sc.configuration); + const description = option.description.slice(&sc.configuration); + const help = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type }); + try w.print("{s:<30} {s}\n", .{ help, description }); + if (option.enum_options.slice(&sc.configuration)) |enum_options| { + const padding: [33]u8 = @splat(' '); + try w.writeAll(padding ++ "Supported Values:\n"); + for (enum_options) |enum_option_index| { + const enum_option = enum_option_index.slice(&sc.configuration); + try w.print(padding ++ " {s}\n", .{enum_option}); + } + } + } + } + + try w.writeAll( + \\ + \\System Integration Options: + \\ --search-prefix [path] Add a path to look for binaries, libraries, headers + \\ --sysroot [path] Set the system root directory (usually /) + \\ --libc [file] Provide a file which specifies libc paths + \\ + \\ --system [pkgdir] Disable package fetching; enable all integrations + \\ -fsys=[name] Enable a system integration + \\ -fno-sys=[name] Disable a system integration + \\ + \\ -fdarling, -fno-darling Integration with system-installed Darling to + \\ execute macOS programs on Linux hosts + \\ (default: no) + \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute + \\ foreign-architecture programs on Linux hosts + \\ (default: no) + \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc + \\ (e.g. glibc or musl) built for multiple foreign + \\ architectures, allowing execution of non-native + \\ programs that link with libc. + \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on + \\ ARM64 macOS hosts. (default: no) + \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to + \\ execute WASI binaries. (default: no) + \\ -fwine, -fno-wine Integration with system-installed Wine to execute + \\ Windows programs on Linux hosts. (default: no) + \\ + \\ Available System Integrations: Enabled: + \\ + ); + if (sc.configuration.system_integrations.len == 0) { + try w.writeAll(" (none) -\n"); + } else { + for (sc.configuration.system_integrations) |system_integration| { + const name = system_integration.name.slice(&sc.configuration); + const status = switch (system_integration.status) { + .disabled => "no", + .enabled => "yes", + }; + try w.print(" {s:<43} {s}\n", .{ name, status }); + } + } + + try w.writeAll( + \\ + \\General Options: + \\ -h, --help Print this help to stdout and exit + \\ -l, --list-steps Print available steps to stdout and exit + \\ + \\ -p, --prefix [path] Where to install files (default: zig-out) + \\ --prefix-lib-dir [path] Where to install libraries + \\ --prefix-exe-dir [path] Where to install executables + \\ --prefix-include-dir [path] Where to install C header files + \\ --release[=mode] Request release mode, optionally specifying a + \\ preferred optimization mode: fast, safe, small + \\ + \\ --verbose Print commands before executing them + \\ --color [auto|off|on] Enable or disable colored error messages + \\ --error-style [style] Control how build errors are printed + \\ verbose (Default) Report errors with full context + \\ minimal Report errors after summary, excluding context like command lines + \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update + \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update + \\ --multiline-errors [style] Control how multi-line error messages are printed + \\ indent (Default) Indent non-initial lines to align with initial line + \\ newline Include a leading newline so that the error message is on its own lines + \\ none Print as usual so the first line is misaligned + \\ --summary [mode] Control the printing of the build summary + \\ all Print the build summary in its entirety + \\ new Omit cached steps + \\ failures (Default if short-lived) Only print failed steps + \\ line (Default if long-lived) Only print the single-line summary + \\ none Do not print the build summary + \\ -j Limit concurrent jobs (default is to use all CPU cores) + \\ --maxrss Limit memory usage (default is to use available memory) + \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss + \\ --test-timeout Limit execution time of unit tests, terminating if exceeded. + \\ The timeout must include a unit: ns, us, ms, s, m, h + \\ --watch Continuously rebuild when source files are modified + \\ --debounce Delay before rebuilding after changed file detected + \\ --webui[=ip] Enable the web interface on the given IP address + \\ --fuzz[=limit] Continuously search for unit test failures with an optional + \\ limit to the max number of iterations. The argument supports + \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies + \\ '--webui' when no limit is specified. + \\ --time-report Force full rebuild and provide detailed information on + \\ compilation time of Zig source code (implies '--webui') + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation + \\ + \\Package Management Options: + \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit + \\ needed (Default) Lazy dependencies are fetched as needed + \\ all Lazy dependencies are always fetched + \\ --fork=[path] Override one or more projects from dependency tree + \\ + \\Advanced Options: + \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error + \\ -fno-reference-trace Disable reference trace + \\ -fallow-so-scripts Allows .so files to be GNU ld scripts + \\ -fno-allow-so-scripts (default) .so files must be ELF files + \\ --build-file [file] Override path to build.zig + \\ --cache-dir [path] Override path to local Zig cache directory + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --zig-lib-dir [arg] Override path to Zig lib directory + \\ --build-runner [file] Override path to build runner + \\ --seed [integer] For shuffling dependency traversal order (default: random) + \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries + \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) + \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) + \\ md5 16-byte cryptographic hash (ELF) + \\ uuid 16-byte random UUID (ELF, WASM) + \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) + \\ none (default) No build ID + \\ --debug-log [scope] Enable debugging the compiler + \\ --debug-pkg-config Fail if unknown pkg-config flags encountered + \\ --debug-rt Debug compiler runtime libraries + \\ --verbose-link Enable compiler debug output for linking + \\ --verbose-air Enable compiler debug output for Zig AIR + \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR + \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC + \\ --verbose-cimport Enable compiler debug output for C imports + \\ --verbose-cc Enable compiler debug output for C compilation + \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features + \\ + ); + } +}; diff --git a/lib/compiler/Maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig new file mode 100644 index 0000000000000000000000000000000000000000..d3066d24afc31632e211e2883843adb497c83de7 --- /dev/null +++ b/lib/compiler/Maker/Fuzz.zig @@ -0,0 +1,606 @@ +const Fuzz = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Build = std.Build; +const Cache = std.Build.Cache; +const Coverage = std.debug.Coverage; +const Configuration = std.Build.Configuration; +const Io = std.Io; +const abi = std.Build.abi.fuzz; +const assert = std.debug.assert; +const fatal = std.process.fatal; +const log = std.log; + +const Maker = @import("../Maker.zig"); +const WebServer = @import("WebServer.zig"); + +gpa: Allocator, +io: Io, +mode: Mode, + +/// Allocated into `gpa`. +run_steps: []const Configuration.Step.Index, + +group: Io.Group, +root_prog_node: std.Progress.Node, +prog_node: std.Progress.Node, + +/// Protects `coverage_files`. +coverage_mutex: Io.Mutex, +coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap), + +queue_mutex: Io.Mutex, +queue_cond: Io.Condition, +msg_queue: std.ArrayList(Msg), + +pub const Mode = union(enum) { + forever: struct { ws: *WebServer }, + limit: Limited, + + pub const Limited = struct { + amount: u64, + }; +}; + +const Msg = union(enum) { + coverage: struct { + id: u64, + cumulative: struct { + runs: u64, + unique: u64, + coverage: u64, + }, + run: Configuration.Step.Index, + }, + entry_point: struct { + coverage_id: u64, + addr: u64, + }, +}; + +const CoverageMap = struct { + mapped_memory: []align(std.heap.page_size_min) const u8, + coverage: Coverage, + source_locations: []Coverage.SourceLocation, + /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. + entry_points: std.ArrayList(u32), + start_timestamp: i64, + start_n_runs: u64, + + fn deinit(cm: *CoverageMap, gpa: Allocator) void { + std.posix.munmap(cm.mapped_memory); + cm.coverage.deinit(gpa); + cm.* = undefined; + } +}; + +pub fn init( + gpa: Allocator, + io: Io, + all_steps: []const Configuration.Step.Index, + root_prog_node: std.Progress.Node, + mode: Mode, +) error{ OutOfMemory, Canceled }!Fuzz { + const run_steps: []const Configuration.Step.Index = steps: { + var steps: std.ArrayList(Configuration.Step.Index) = .empty; + defer steps.deinit(gpa); + const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0); + defer rebuild_node.end(); + var rebuild_group: Io.Group = .init; + defer rebuild_group.cancel(io); + + for (all_steps) |step| { + if (true) @panic("TODO"); + const run = step.cast(std.Build.Step.Run) orelse continue; + if (run.producer == null) continue; + if (run.fuzz_tests.items.len == 0) continue; + try steps.append(gpa, run); + rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node }); + } + + if (steps.items.len == 0) fatal("no fuzz tests found", .{}); + rebuild_node.setEstimatedTotalItems(steps.items.len); + const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items); + try rebuild_group.await(io); + break :steps run_steps; + }; + errdefer gpa.free(run_steps); + + for (run_steps) |run_step_index| { + if (true) @panic("TODO"); + assert(run_step_index.fuzz_tests.items.len > 0); + if (run_step_index.rebuilt_executable == null) + fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); + } + + return .{ + .gpa = gpa, + .io = io, + .mode = mode, + .run_steps = run_steps, + .group = .init, + .root_prog_node = root_prog_node, + .prog_node = .none, + .coverage_files = .empty, + .coverage_mutex = .init, + .queue_mutex = .init, + .queue_cond = .init, + .msg_queue = .empty, + }; +} + +pub fn start(fuzz: *Fuzz) void { + const io = fuzz.io; + fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0); + + if (fuzz.mode == .forever) { + // For polling messages and sending updates to subscribers. + fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err| + fatal("unable to spawn coverage task: {t}", .{err}); + } + + if (true) @panic("TODO"); + + for (fuzz.run_steps) |run| { + assert(run.rebuilt_executable != null); + fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run }); + } +} + +pub fn deinit(fuzz: *Fuzz) void { + const io = fuzz.io; + fuzz.group.cancel(io); + fuzz.prog_node.end(); + fuzz.gpa.free(fuzz.run_steps); +} + +fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void { + rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { + const compile = run.producer.?; + log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); + }; +} + +fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { + const graph = run.step.owner.graph; + const io = graph.io; + const compile = run.producer.?; + const prog_node = parent_prog_node.start(compile.step.name, 0); + defer prog_node.end(); + + const result = compile.rebuildInFuzzMode(gpa, prog_node); + + const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; + const show_error_msgs = compile.step.result_error_msgs.items.len > 0; + const show_stderr = compile.step.result_stderr.len > 0; + + if (show_error_msgs or show_compile_errors or show_stderr) { + var buf: [256]u8 = undefined; + const stderr = try io.lockStderr(&buf, graph.stderr_mode); + defer io.unlockStderr(); + Maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + } + + const rebuilt_bin_path = result catch |err| switch (err) { + error.MakeFailed => return, + else => |other| return other, + }; + run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); +} + +fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { + const owner = run.step.owner; + const gpa = owner.allocator; + const graph = owner.graph; + const io = graph.io; + + run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) { + error.MakeFailed => { + var buf: [256]u8 = undefined; + const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) { + error.Canceled => return, + }; + defer io.unlockStderr(); + Maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + return; + }, + else => { + log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err }); + return; + }, + }; +} + +pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { + if (true) @panic("TODO"); + assert(fuzz.mode == .forever); + + var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); + var dedup_table: DedupTable = .empty; + defer dedup_table.deinit(fuzz.gpa); + + for (fuzz.run_steps) |run_step| { + const compile_inputs = run_step.producer.?.step.inputs.table; + for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { + try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len); + for (file_list.items) |sub_path| { + if (!std.mem.endsWith(u8, sub_path, ".zig")) continue; + const joined_path = try dir_path.join(arena, sub_path); + dedup_table.putAssumeCapacity(joined_path, {}); + } + } + } + + const deduped_paths = dedup_table.keys(); + const SortContext = struct { + pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { + _ = this; + return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { + .lt => true, + .gt => false, + .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), + }; + } + }; + std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); + return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths); +} + +pub const Previous = struct { + unique_runs: usize, + entry_points: usize, + sent_source_index: bool, + pub const init: Previous = .{ + .unique_runs = 0, + .entry_points = 0, + .sent_source_index = false, + }; +}; +pub fn sendUpdate( + fuzz: *Fuzz, + socket: *std.http.Server.WebSocket, + prev: *Previous, +) !void { + const io = fuzz.io; + + try fuzz.coverage_mutex.lock(io); + defer fuzz.coverage_mutex.unlock(io); + + const coverage_maps = fuzz.coverage_files.values(); + if (coverage_maps.len == 0) return; + // TODO: handle multiple fuzz steps in the WebSocket packets + const coverage_map = &coverage_maps[0]; + const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the + // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the + // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass + // this data straight to the socket with sendfile... + const seen_pcs = cov_header.seenBits(); + const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); + const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); + { + if (!prev.sent_source_index) { + prev.sent_source_index = true; + // We need to send initial context. + const header: abi.SourceIndexHeader = .{ + .directories_len = @intCast(coverage_map.coverage.directories.entries.len), + .files_len = @intCast(coverage_map.coverage.files.entries.len), + .source_locations_len = @intCast(coverage_map.source_locations.len), + .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), + .start_timestamp = coverage_map.start_timestamp, + .start_n_runs = coverage_map.start_n_runs, + }; + var iovecs: [5][]const u8 = .{ + @ptrCast(&header), + @ptrCast(coverage_map.coverage.directories.keys()), + @ptrCast(coverage_map.coverage.files.keys()), + @ptrCast(coverage_map.source_locations), + coverage_map.coverage.string_bytes.items, + }; + try socket.writeMessageVec(&iovecs, .binary); + } + + const header: abi.CoverageUpdateHeader = .{ + .n_runs = n_runs, + .unique_runs = unique_runs, + }; + var iovecs: [2][]const u8 = .{ + @ptrCast(&header), + @ptrCast(seen_pcs), + }; + try socket.writeMessageVec(&iovecs, .binary); + + prev.unique_runs = unique_runs; + } + + if (prev.entry_points != coverage_map.entry_points.items.len) { + const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len)); + var iovecs: [2][]const u8 = .{ + @ptrCast(&header), + @ptrCast(coverage_map.entry_points.items), + }; + try socket.writeMessageVec(&iovecs, .binary); + + prev.entry_points = coverage_map.entry_points.items.len; + } +} + +fn coverageRun(fuzz: *Fuzz) void { + coverageRunCancelable(fuzz) catch |err| switch (err) { + error.Canceled => return, + }; +} + +fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { + const io = fuzz.io; + + try fuzz.queue_mutex.lock(io); + defer fuzz.queue_mutex.unlock(io); + + while (true) { + try fuzz.queue_cond.wait(io, &fuzz.queue_mutex); + for (fuzz.msg_queue.items) |msg| switch (msg) { + .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) { + error.AlreadyReported => continue, + error.Canceled => return, + else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), + }, + .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) { + error.AlreadyReported => continue, + error.Canceled => return, + else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), + }, + }; + fuzz.msg_queue.clearRetainingCapacity(); + } +} +fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { + if (true) @panic("TODO"); + assert(fuzz.mode == .forever); + const ws = fuzz.mode.forever.ws; + const gpa = fuzz.gpa; + const io = fuzz.io; + + try fuzz.coverage_mutex.lock(io); + defer fuzz.coverage_mutex.unlock(io); + + const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id); + if (gop.found_existing) { + // We are fuzzing the same executable with multiple threads. + // Perhaps the same unit test; perhaps a different one. In any + // case, since the coverage file is the same, we only have to + // notice changes to that one file in order to learn coverage for + // this particular executable. + return; + } + errdefer _ = fuzz.coverage_files.pop(); + + gop.value_ptr.* = .{ + .coverage = std.debug.Coverage.init, + .mapped_memory = undefined, // populated below + .source_locations = undefined, // populated below + .entry_points = .empty, + .start_timestamp = ws.now(), + .start_n_runs = undefined, // populated below + }; + errdefer gop.value_ptr.coverage.deinit(gpa); + + const rebuilt_exe_path = run_step_index.rebuilt_executable.?; + const target = run_step_index.producer.?.rootModuleTarget(); + var debug_info = std.debug.Info.load( + gpa, + io, + rebuilt_exe_path, + &gop.value_ptr.coverage, + target.ofmt, + target.cpu.arch, + ) catch |err| { + log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ + run_step_index.step.name, rebuilt_exe_path, err, + }); + return error.AlreadyReported; + }; + defer debug_info.deinit(gpa); + + const coverage_file_path: Build.Cache.Path = .{ + .root_dir = run_step_index.step.owner.cache_root, + .sub_path = "v/" ++ std.fmt.hex(coverage_id), + }; + var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { + log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ + run_step_index.step.name, coverage_file_path, err, + }); + return error.AlreadyReported; + }; + defer coverage_file.close(io); + + const file_size = coverage_file.length(io) catch |err| { + log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err }); + return error.AlreadyReported; + }; + + const mapped_memory = std.posix.mmap( + null, + file_size, + .{ .READ = true }, + .{ .TYPE = .SHARED }, + coverage_file.handle, + 0, + ) catch |err| { + log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err }); + return error.AlreadyReported; + }; + gop.value_ptr.mapped_memory = mapped_memory; + + const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + const pcs = header.pcAddrs(); + const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len); + errdefer gpa.free(source_locations); + + // Unfortunately the PCs array that LLVM gives us from the 8-bit PC + // counters feature is not sorted. + var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .empty; + defer sorted_pcs.deinit(gpa); + try sorted_pcs.resize(gpa, pcs.len); + @memcpy(sorted_pcs.items(.pc), pcs); + for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i); + sorted_pcs.sortUnstable(struct { + addrs: []const u64, + + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.addrs[a_index] < ctx.addrs[b_index]; + } + }{ .addrs = sorted_pcs.items(.pc) }); + + debug_info.resolveAddresses(gpa, io, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| { + log.err("failed to resolve addresses to source locations: {t}", .{err}); + return error.AlreadyReported; + }; + + for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl; + gop.value_ptr.source_locations = source_locations; + gop.value_ptr.start_n_runs = header.n_runs; + + ws.notifyUpdate(); +} + +fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void { + const io = fuzz.io; + + try fuzz.coverage_mutex.lock(io); + defer fuzz.coverage_mutex.unlock(io); + + const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?; + const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + const pcs = header.pcAddrs(); + + // Since this pcs list is unsorted, we must linear scan for the best index. + const index = i: { + var best: usize = 0; + for (pcs[1..], 1..) |elem_addr, i| { + if (elem_addr == addr) break :i i; + if (elem_addr > addr) continue; + if (elem_addr > pcs[best]) best = i; + } + break :i best; + }; + if (index >= pcs.len) { + log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{ + addr, pcs[0], pcs[pcs.len - 1], + }); + return error.AlreadyReported; + } + if (false) { + const sl = coverage_map.source_locations[index]; + const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename); + if (pcs.len == 1) { + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{ + addr, file_name, sl.line, sl.column, + }); + } else if (index == 0) { + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{ + addr, file_name, sl.line, sl.column, pcs[index + 1], + }); + } else if (index == pcs.len - 1) { + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{ + addr, file_name, sl.line, sl.column, index, pcs[index - 1], + }); + } else { + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{ + addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1], + }); + } + } + try coverage_map.entry_points.append(fuzz.gpa, @intCast(index)); +} + +pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { + if (true) @panic("TODO"); + assert(fuzz.mode == .limit); + const io = fuzz.io; + + try fuzz.group.await(io); + fuzz.group = .init; + + std.debug.print("======= FUZZING REPORT =======\n", .{}); + for (fuzz.msg_queue.items) |msg| { + if (msg != .coverage) continue; + + const cov = msg.coverage; + const coverage_file_path: std.Build.Cache.Path = .{ + .root_dir = cov.run.step.owner.cache_root, + .sub_path = "v/" ++ std.fmt.hex(cov.id), + }; + var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { + fatal("step '{s}': failed to load coverage file '{f}': {t}", .{ + cov.run.step.name, coverage_file_path, err, + }); + }; + defer coverage_file.close(io); + + const fuzz_abi = std.Build.abi.fuzz; + var rbuf: [0x1000]u8 = undefined; + var r = coverage_file.reader(io, &rbuf); + + var header: fuzz_abi.SeenPcsHeader = undefined; + r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { + fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ + cov.run.step.name, coverage_file_path, err, + }); + }; + + if (header.pcs_len == 0) { + fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{ + cov.run.step.name, coverage_file_path, + }); + } + + var seen_count: usize = 0; + const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len); + for (0..chunk_count) |_| { + const seen = r.interface.takeInt(usize, .little) catch |err| { + fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ + cov.run.step.name, coverage_file_path, err, + }); + }; + seen_count += @popCount(seen); + } + + const seen_f: f64 = @floatFromInt(seen_count); + const total_f: f64 = @floatFromInt(header.pcs_len); + const ratio = seen_f / total_f; + std.debug.print( + \\Step: {s} + \\Fuzz test: "{s}" ({x}) + \\Runs: {} -> {} + \\Unique runs: {} -> {} + \\Coverage: {}/{} -> {}/{} ({:.02}%) + \\ + , .{ + cov.run.step.name, + cov.run.fuzz_tests.items[0], + cov.id, + cov.cumulative.runs, + header.n_runs, + cov.cumulative.unique, + header.unique_runs, + cov.cumulative.coverage, + header.pcs_len, + seen_count, + header.pcs_len, + ratio * 100, + }); + + std.debug.print("------------------------------\n", .{}); + } + std.debug.print( + \\Values are accumulated across multiple runs when preserving the cache. + \\============================== + \\ + , .{}); +} diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig new file mode 100644 index 0000000000000000000000000000000000000000..ba9cad0ee17b713cd1731aff04e22a132d94afdd --- /dev/null +++ b/lib/compiler/Maker/Graph.zig @@ -0,0 +1,25 @@ +//! Shared maker state among all steps. +const Graph = @This(); + +const std = @import("std"); +const Io = std.Io; +const Allocator = std.mem.Allocator; +const Configuration = std.Build.Configuration; + +io: Io, +/// Process lifetime. +arena: Allocator, +cache: std.Build.Cache, +zig_exe: []const u8, +environ_map: std.process.Environ.Map, +global_cache_root: std.Build.Cache.Directory, +zig_lib_directory: std.Build.Cache.Directory, + +debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, +incremental: ?bool = null, +random_seed: u32 = 0, +allow_so_scripts: ?bool = null, +time_report: bool = false, +/// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also +/// respects the '--color' flag. +stderr_mode: ?Io.Terminal.Mode = null, diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig new file mode 100644 index 0000000000000000000000000000000000000000..845bc1e1f8ecb4191d945a5c3f86b390adc7290e --- /dev/null +++ b/lib/compiler/Maker/Step.zig @@ -0,0 +1,840 @@ +//! The state that maker needs in order to process a step. +const Step = @This(); + +const builtin = @import("builtin"); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const Io = std.Io; +const LazyPath = std.Build.Configuration.LazyPath; +const Package = std.Build.Configuration.Package; +const Path = std.Build.Cache.Path; +const Configuration = std.Build.Configuration; +const assert = std.debug.assert; + +const WebServer = @import("WebServer.zig"); + +pub const Compile = void; // @import("Step/Compile.zig"); +pub const Run = void; // @import("Step/Run.zig"); + +/// Avoid false sharing. +_: void align(std.atomic.cache_line) = {}, + +state: State = .precheck_unstarted, +dependants: std.ArrayList(Configuration.Step.Index) = .empty, +/// Collects the set of files that retrigger this step to run. +/// +/// This is used by the build system's implementation of `--watch` but it can +/// also be potentially useful for IDEs to know what effects editing a +/// particular file has. +/// +/// Populated within `make`. Implementation may choose to clear and repopulate, +/// retain previous value, or update. +inputs: Inputs = .init, +pending_deps: u32 = undefined, + +result_error_msgs: std.ArrayList([]const u8) = .empty, +result_error_bundle: std.zig.ErrorBundle = .empty, +result_stderr: []const u8 = "", +result_cached: bool = false, +result_duration_ns: ?u64 = null, +/// 0 means unavailable or not reported. +result_peak_rss: usize = 0, +/// If the step is failed and this field is populated, this is the command which failed. +/// This field may be populated even if the step succeeded. +result_failed_command: ?[]const u8 = null, +test_results: TestResults = .{}, + +pub const State = enum { + precheck_unstarted, + precheck_started, + /// This is also used to indicate "dirty" steps that have been modified + /// after a previous build completed, in which case, the step may or may + /// not have been completed before. Either way, one or more of its direct + /// file system inputs have been modified, meaning that the step needs to + /// be re-evaluated. + precheck_done, + dependency_failure, + success, + failure, + /// This state indicates that the step did not complete, however, it also did not fail, + /// and it is safe to continue executing its dependencies. + skipped, + /// This step was skipped because it specified a max_rss that exceeded the runner's maximum. + /// It is not safe to run its dependencies. + skipped_oom, +}; + +pub const Inputs = struct { + table: Table, + + pub const init: Inputs = .{ + .table = .{}, + }; + + pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, Files, Cache.Path.TableAdapter, false); + /// The special file name "." means any changes inside the directory. + pub const Files = std.ArrayList([]const u8); + + pub fn populated(inputs: *Inputs) bool { + return inputs.table.count() != 0; + } + + pub fn clear(inputs: *Inputs, gpa: Allocator) void { + for (inputs.table.values()) |*files| files.deinit(gpa); + inputs.table.clearRetainingCapacity(); + } +}; + +pub const TestResults = struct { + /// The total number of tests in the step. Every test has a "status" from the following: + /// * passed + /// * skipped + /// * failed cleanly + /// * crashed + /// * timed out + test_count: u32 = 0, + + /// The number of tests which were skipped (`error.SkipZigTest`). + skip_count: u32 = 0, + /// The number of tests which failed cleanly. + fail_count: u32 = 0, + /// The number of tests which terminated unexpectedly, i.e. crashed. + crash_count: u32 = 0, + /// The number of tests which timed out. + timeout_count: u32 = 0, + + /// The number of detected memory leaks. The associated test may still have passed; indeed, *all* + /// individual tests may have passed. However, the step as a whole fails if any test has leaks. + leak_count: u32 = 0, + /// The number of detected error logs. The associated test may still have passed; indeed, *all* + /// individual tests may have passed. However, the step as a whole fails if any test logs errors. + log_err_count: u32 = 0, + + pub fn isSuccess(tr: TestResults) bool { + // all steps are success or skip + return tr.fail_count == 0 and + tr.crash_count == 0 and + tr.timeout_count == 0 and + // no (otherwise successful) step leaked memory or logged errors + tr.leak_count == 0 and + tr.log_err_count == 0; + } + + /// Computes the number of tests which passed from the other values. + pub fn passCount(tr: TestResults) u32 { + return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count; + } +}; + +pub const MakeOptions = struct { + progress_node: std.Progress.Node, + watch: bool, + web_server: ?*WebServer, + /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. + unit_test_timeout_ns: ?u64, + /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. + gpa: Allocator, +}; + +pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; + +/// If the Step's `make` function reports `error.MakeFailed`, it indicates they +/// have already reported the error. Otherwise, we add a simple error report +/// here. +pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { + if (true) @panic("TODO Step.make"); + const arena = s.owner.allocator; + const graph = s.owner.graph; + const io = graph.io; + + var start_ts: ?Io.Timestamp = t: { + if (!graph.time_report) break :t null; + if (s.id == .compile) break :t null; + if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; + break :t Io.Clock.awake.now(io); + }; + const make_result = s.makeFn(s, options); + if (start_ts) |*ts| { + const duration = ts.untilNow(io, .awake); + options.web_server.?.updateTimeReportGeneric(s, duration); + } + + make_result catch |err| switch (err) { + error.MakeFailed, error.MakeSkipped => |e| return e, + else => { + s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); + return error.MakeFailed; + }, + }; + + if (!s.test_results.isSuccess()) { + return error.MakeFailed; + } + + if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { + const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ + s.result_peak_rss, s.max_rss, + }) catch @panic("OOM"); + s.result_error_msgs.append(arena, msg) catch @panic("OOM"); + } +} + +/// Implementation detail of file watching. Prepares the step for being re-evaluated. +/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. +pub fn invalidateResult(step: *Step, gpa: Allocator) bool { + if (true) @panic("TODO Step.invalidateResult"); + if (step.state == .precheck_done) return false; + assert(step.pending_deps == 0); + step.state = .precheck_done; + step.reset(gpa); + for (step.dependants.items) |dependant| { + _ = dependant.invalidateResult(gpa); + dependant.pending_deps += 1; + } + return true; +} + +/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. +pub fn reset(step: *Step, gpa: Allocator) void { + assert(step.state == .precheck_done); + + if (step.result_failed_command) |cmd| gpa.free(cmd); + + step.result_error_msgs.clearRetainingCapacity(); + step.result_stderr = ""; + step.result_cached = false; + step.result_duration_ns = null; + step.result_peak_rss = 0; + step.result_failed_command = null; + step.test_results = .{}; + step.clearWatchInputs(); + + step.result_error_bundle.deinit(gpa); + step.result_error_bundle = std.zig.ErrorBundle.empty; +} + +/// Populates `s.result_failed_command`. +pub fn captureChildProcess( + s: *Step, + gpa: Allocator, + progress_node: std.Progress.Node, + argv: []const []const u8, +) !std.process.RunResult { + const graph = s.owner.graph; + const arena = graph.arena; + const io = graph.io; + + // If an error occurs, it's happened in this command: + assert(s.result_failed_command == null); + s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); + + try handleChildProcUnsupported(s); + try handleVerbose(s, .inherit, argv); + + const result = std.process.run(arena, io, .{ + .argv = argv, + .environ_map = &graph.environ_map, + .progress_node = progress_node, + }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); + + if (result.stderr.len > 0) { + try s.result_error_msgs.append(arena, result.stderr); + } + + return result; +} + +pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { + try step.addError(fmt, args); + return error.MakeFailed; +} + +pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { + const arena = step.owner.allocator; + const msg = try std.fmt.allocPrint(arena, fmt, args); + try step.result_error_msgs.append(arena, msg); +} + +pub const ZigProcess = struct { + child: std.process.Child, + multi_reader_buffer: Io.File.MultiReader.Buffer(2), + multi_reader: Io.File.MultiReader, + progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn, + + pub const StreamEnum = enum { stdout, stderr }; + + pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void { + zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null; + } + + pub fn deinit(zp: *ZigProcess, io: Io) void { + zp.child.kill(io); + zp.multi_reader.deinit(); + zp.* = undefined; + } +}; + +/// Assumes that argv contains `--listen=-` and that the process being spawned +/// is the zig compiler - the same version that compiled the build runner. +/// Populates `s.result_failed_command`. +pub fn evalZigProcess( + s: *Step, + argv: []const []const u8, + prog_node: std.Progress.Node, + watch: bool, + web_server: ?*WebServer, + gpa: Allocator, +) !?Cache.Path { + const b = s.owner; + const io = b.graph.io; + + // If an error occurs, it's happened in this command: + assert(s.result_failed_command == null); + s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); + + if (s.getZigProcess()) |zp| update: { + assert(watch); + if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index); + zp.progress_ipc_index = null; + var exited = false; + defer if (exited) { + s.cast(Compile).?.zig_process = null; + zp.deinit(io); + gpa.destroy(zp); + } else zp.saveState(prog_node); + const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { + error.BrokenPipe, error.EndOfStream => |reason| { + std.log.info("{s} restart required: {t}", .{ argv[0], reason }); + // Process restart required. + const term = zp.child.wait(io) catch |e| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); + }; + _ = term; + exited = true; + break :update; + }, + else => |e| return e, + }; + + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + } + + if (s.result_error_msgs.items.len > 0 and result == null) { + // Crash detected. + const term = zp.child.wait(io) catch |e| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + exited = true; + try handleChildProcessTerm(s, term); + return error.MakeFailed; + } + + return result; + } + assert(argv.len != 0); + + try handleChildProcUnsupported(s); + try handleVerbose(s, .inherit, argv); + + const zp = try gpa.create(ZigProcess); + defer if (!watch) gpa.destroy(zp); + + zp.child = std.process.spawn(io, .{ + .argv = argv, + .environ_map = &b.graph.environ_map, + .stdin = .pipe, + .stdout = .pipe, + .stderr = .pipe, + .request_resource_usage_statistics = true, + .progress_node = prog_node, + }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); + + zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ + zp.child.stdout.?, zp.child.stderr.?, + }); + if (watch) s.cast(Compile).?.zig_process = zp; + defer if (!watch) zp.deinit(io); + + const result = result: { + defer if (watch) zp.saveState(prog_node); + break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); + }; + + if (!watch) { + // Send EOF to stdin. + zp.child.stdin.?.close(io); + zp.child.stdin = null; + + const term = zp.child.wait(io) catch |err| { + return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + + // Special handling for Compile step that is expecting compile errors. + if (s.cast(Compile)) |compile| switch (term) { + .exited => { + // Note that the exit code may be 0 in this case due to the + // compiler server protocol. + if (compile.expect_errors != null) { + return error.NeedCompileErrorCheck; + } + }, + else => {}, + }; + + try handleChildProcessTerm(s, term); + } + + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + } + + return result; +} + +/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. +pub fn installFile(s: *Step, src_lazy_path: LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { + const b = s.owner; + const io = b.graph.io; + const src_path = src_lazy_path.getPath3(b, s); + try handleVerbose(s, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); + return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| + return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); +} + +/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. +pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { + const b = s.owner; + const io = b.graph.io; + try handleVerbose(s, .inherit, &.{ "install", "-d", dest_path }); + return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| + return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); +} + +fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebServer, gpa: Allocator) !?Path { + const b = s.owner; + const arena = b.allocator; + const io = b.graph.io; + + const start_ts = Io.Clock.awake.now(io); + + try sendMessage(io, zp.child.stdin.?, .update); + if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); + + var result: ?Path = null; + var eos_err: error{EndOfStream}!void = {}; + + const stdout = zp.multi_reader.fileReader(0); + + while (true) { + const Header = std.zig.Server.Message.Header; + const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) { + error.EndOfStream => break, + error.ReadFailed => return stdout.err.?, + }; + const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) { + error.EndOfStream => |e| { + // Better to report the crash with stderr below, but we set + // this in case the child exits successfully while violating + // this protocol. + eos_err = e; + break; + }, + error.ReadFailed => return stdout.err.?, + }; + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return s.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + } + }, + .error_bundle => { + s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); + // This message indicates the end of the update. + if (watch) break; + }, + .emit_digest => { + const EmitDigest = std.zig.Server.Message.EmitDigest; + const emit_digest: *align(1) const EmitDigest = @ptrCast(body); + s.result_cached = emit_digest.flags.cache_hit; + const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; + result = .{ + .root_dir = b.cache_root, + .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), + }; + }, + .file_system_inputs => { + s.clearWatchInputs(); + var it = std.mem.splitScalar(u8, body, 0); + while (it.next()) |prefixed_path| { + const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); + const sub_path = try arena.dupe(u8, prefixed_path[1..]); + const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; + switch (prefix_index) { + .cwd => { + const path: Cache.Path = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .zig_lib => zl: { + if (s.cast(Step.Compile)) |compile| { + if (compile.zig_lib_dir) |zig_lib_dir| { + const lp = try zig_lib_dir.join(arena, sub_path); + try addWatchInput(s, lp); + break :zl; + } + } + const path: Cache.Path = .{ + .root_dir = s.owner.graph.zig_lib_directory, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .local_cache => { + const path: Cache.Path = .{ + .root_dir = b.cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .global_cache => { + const path: Cache.Path = .{ + .root_dir = s.owner.graph.global_cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + } + } + }, + .time_report => if (web_server) |ws| { + const TimeReport = std.zig.Server.Message.TimeReport; + const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); + ws.updateTimeReportCompile(.{ + .compile = s.cast(Step.Compile).?, + .use_llvm = tr.flags.use_llvm, + .stats = tr.stats, + .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), + .llvm_pass_timings_len = tr.llvm_pass_timings_len, + .files_len = tr.files_len, + .decls_len = tr.decls_len, + .trailing = body[@sizeOf(TimeReport)..], + }); + }, + else => {}, // ignore other messages + } + } + + s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()); + + const stderr_contents = zp.multi_reader.reader(1).buffered(); + if (stderr_contents.len > 0) { + try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); + } + + try eos_err; + + return result; +} + +pub fn getZigProcess(s: *Step) ?*ZigProcess { + if (true) @panic("TODO getZigProcess"); + return switch (s.id) { + .compile => s.cast(Compile).?.zig_process, + else => null, + }; +} + +fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + var w = file.writer(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +pub fn handleVerbose( + s: *Step, + arena: Allocator, + cwd: std.process.Child.Cwd, + opt_env: ?*const std.process.Environ.Map, + argv: []const []const u8, +) error{OutOfMemory}!void { + if (!s.verbose) return; + const graph = s.graph; + // Intention of verbose is to print all sub-process command lines to + // stderr before spawning them. + const text = try allocPrintCmd(arena, cwd, if (opt_env) |env| .{ + .child = env, + .parent = &graph.environ_map, + } else null, argv); + std.log.scoped(.verbose).info("{s}", .{text}); +} + +/// Asserts that the caller has already populated `s.result_failed_command`. +pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { + if (!std.process.can_spawn) { + return s.fail("unable to spawn process: host cannot spawn child processes", .{}); + } +} + +/// Asserts that the caller has already populated `s.result_failed_command`. +pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { + assert(s.result_failed_command != null); + return switch (term) { + .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), + .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), + .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), + .unknown => s.fail("process terminated unexpectedly", .{}), + }; +} + +/// Prefer `cacheHitAndWatch` unless you already added watch inputs +/// separately from using the cache system. +pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool { + s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); + return s.result_cached; +} + +/// Clears previous watch inputs, if any, and then populates watch inputs from +/// the full set of files picked up by the cache manifest. +/// +/// Must be accompanied with `writeManifestAndWatch`. +pub fn cacheHitAndWatch(s: *Step, man: *Cache.Manifest) !bool { + const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); + s.result_cached = is_hit; + // The above call to hit() populates the manifest with files, so in case of + // a hit, we need to populate watch inputs. + if (is_hit) try setWatchInputsFromManifest(s, man); + return is_hit; +} + +fn failWithCacheError( + s: *Step, + man: *const Cache.Manifest, + err: Cache.Manifest.HitError, +) error{ OutOfMemory, Canceled, MakeFailed } { + switch (err) { + error.CacheCheckFailed => switch (man.diagnostic) { + .none => unreachable, + .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ + man.diagnostic, e, + }), + .file_open, .file_stat, .file_read, .file_hash => |op| { + const pp = man.files.keys()[op.file_index].prefixed_path; + const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; + return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ + prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, + }); + }, + }, + error.OutOfMemory, error.Canceled => |e| return e, + error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), + } +} + +/// Prefer `writeManifestAndWatch` unless you already added watch inputs +/// separately from using the cache system. +pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { + if (s.test_results.isSuccess()) { + man.writeManifest() catch |err| { + try s.addError("unable to write cache manifest: {t}", .{err}); + }; + } +} + +/// Clears previous watch inputs, if any, and then populates watch inputs from +/// the full set of files picked up by the cache manifest. +/// +/// Must be accompanied with `cacheHitAndWatch`. +pub fn writeManifestAndWatch(s: *Step, man: *Cache.Manifest) !void { + try writeManifest(s, man); + try setWatchInputsFromManifest(s, man); +} + +fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void { + const arena = s.owner.allocator; + const prefixes = man.cache.prefixes(); + clearWatchInputs(s); + for (man.files.keys()) |file| { + // The file path data is freed when the cache manifest is cleaned up at the end of `make`. + const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); + try addWatchInputFromPath(s, .{ + .root_dir = prefixes[file.prefixed_path.prefix], + .sub_path = std.fs.path.dirname(sub_path) orelse "", + }, std.fs.path.basename(sub_path)); + } +} + +/// For steps that have a single input that never changes when re-running `make`. +pub fn singleUnchangingWatchInput(step: *Step, lazy_path: LazyPath) Allocator.Error!void { + if (!step.inputs.populated()) try step.addWatchInput(lazy_path); +} + +pub fn clearWatchInputs(step: *Step) void { + const gpa = step.owner.allocator; + step.inputs.clear(gpa); +} + +/// Places a *file* dependency on the path. +pub fn addWatchInput(step: *Step, lazy_file: LazyPath) Allocator.Error!void { + switch (lazy_file) { + .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), + .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), + .cwd_relative => |path_string| { + try addWatchInputFromPath(step, .{ + .root_dir = .{ + .path = null, + .handle = Io.Dir.cwd(), + }, + .sub_path = std.fs.path.dirname(path_string) orelse "", + }, std.fs.path.basename(path_string)); + }, + // Nothing to watch because this dependency edge is modeled instead via `dependants`. + .generated => {}, + } +} + +/// Any changes inside the directory will trigger invalidation. +/// +/// See also `addDirectoryWatchInputFromPath` which takes a `Cache.Path` instead. +/// +/// Paths derived from this directory should also be manually added via +/// `addDirectoryWatchInputFromPath` if and only if this function returns +/// `true`. +pub fn addDirectoryWatchInput(step: *Step, lazy_directory: LazyPath) Allocator.Error!bool { + switch (lazy_directory) { + .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), + .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), + .cwd_relative => |path_string| { + try addDirectoryWatchInputFromPath(step, .{ + .root_dir = .{ + .path = null, + .handle = Io.Dir.cwd(), + }, + .sub_path = path_string, + }); + }, + // Nothing to watch because this dependency edge is modeled instead via `dependants`. + .generated => return false, + } + return true; +} + +/// Any changes inside the directory will trigger invalidation. +/// +/// See also `addDirectoryWatchInput` which takes a `LazyPath` instead. +/// +/// This function should only be called when it has been verified that the +/// dependency on `path` is not already accounted for by a `Step` dependency. +/// In other words, before calling this function, first check that the +/// `LazyPath` which this `path` is derived from is not `generated`. +pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void { + return addWatchInputFromPath(step, path, "."); +} + +fn addWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { + return addWatchInputFromPath(step, .{ + .root_dir = package.build_root, + .sub_path = std.fs.path.dirname(sub_path) orelse "", + }, std.fs.path.basename(sub_path)); +} + +fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { + return addDirectoryWatchInputFromPath(step, .{ + .root_dir = package.build_root, + .sub_path = sub_path, + }); +} + +fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !void { + const gpa = step.owner.allocator; + const gop = try step.inputs.table.getOrPut(gpa, path); + if (!gop.found_existing) gop.value_ptr.* = .empty; + try gop.value_ptr.append(gpa, basename); +} + +pub fn allocPrintCmd( + gpa: Allocator, + cwd: std.process.Child.Cwd, + opt_env: ?struct { + child: *const std.process.Environ.Map, + parent: *const std.process.Environ.Map, + }, + argv: []const []const u8, +) Allocator.Error![]u8 { + const shell = struct { + fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { + for (string) |c| { + if (switch (c) { + else => true, + '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, + '=' => is_argv0, + }) break; + } else return writer.writeAll(string); + + try writer.writeByte('"'); + for (string) |c| { + if (switch (c) { + std.ascii.control_code.nul => break, + '!', '"', '$', '\\', '`' => true, + else => !std.ascii.isPrint(c), + }) try writer.writeByte('\\'); + switch (c) { + std.ascii.control_code.nul => unreachable, + std.ascii.control_code.bel => try writer.writeByte('a'), + std.ascii.control_code.bs => try writer.writeByte('b'), + std.ascii.control_code.ht => try writer.writeByte('t'), + std.ascii.control_code.lf => try writer.writeByte('n'), + std.ascii.control_code.vt => try writer.writeByte('v'), + std.ascii.control_code.ff => try writer.writeByte('f'), + std.ascii.control_code.cr => try writer.writeByte('r'), + std.ascii.control_code.esc => try writer.writeByte('E'), + ' '...'~' => try writer.writeByte(c), + else => try writer.print("{o:0>3}", .{c}), + } + } + try writer.writeByte('"'); + } + }; + + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + const writer = &aw.writer; + switch (cwd) { + .inherit => {}, + .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, + .dir => @panic("TODO"), + } + if (opt_env) |env| { + var it = env.child.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + if (env.parent.get(key)) |process_value| { + if (std.mem.eql(u8, value, process_value)) continue; + } + writer.print("{s}=", .{key}) catch return error.OutOfMemory; + shell.escape(writer, value, false) catch return error.OutOfMemory; + writer.writeByte(' ') catch return error.OutOfMemory; + } + } + shell.escape(writer, argv[0], true) catch return error.OutOfMemory; + for (argv[1..]) |arg| { + writer.writeByte(' ') catch return error.OutOfMemory; + shell.escape(writer, arg, false) catch return error.OutOfMemory; + } + return aw.toOwnedSlice(); +} diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig new file mode 100644 index 0000000000000000000000000000000000000000..5ffbc23cfc15d6d6de5df4f4585585364052da6d --- /dev/null +++ b/lib/compiler/Maker/Step/Compile.zig @@ -0,0 +1,1200 @@ +/// Populated during the make phase when there is a long-lived compiler process. +/// Managed by the build runner, not user build script. +zig_process: ?*Step.ZigProcess, + +fn make(step: *Step, options: Step.MakeOptions) !void { + const b = step.owner; + const compile: *Compile = @fieldParentPtr("step", step); + + const zig_args = try getZigArgs(compile, false); + + const maybe_output_dir = step.evalZigProcess( + zig_args, + options.progress_node, + (b.graph.incremental == true) and (options.watch or options.web_server != null), + options.web_server, + options.gpa, + ) catch |err| switch (err) { + error.NeedCompileErrorCheck => { + assert(compile.expect_errors != null); + try checkCompileErrors(compile); + return; + }, + else => |e| return e, + }; + + // Update generated files + if (maybe_output_dir) |output_dir| { + if (compile.emit_directory) |lp| { + lp.path = b.fmt("{f}", .{output_dir}); + } + + // zig fmt: off + if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); + if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); + // hack for stage2_x86_64 + coff + if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); + if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); + if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); + if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); + if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); + if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); + if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); + // zig fmt: on + } + + if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and + compile.version != null and compile.generated_bin != null and + std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) + { + try doAtomicSymLinks( + step, + compile.getEmittedBin().getPath2(b, step), + compile.major_only_filename.?, + compile.name_only_filename.?, + ); + } +} + +fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { + const step = &compile.step; + const b = step.owner; + const arena = b.allocator; + + var zig_args = std.array_list.Managed([]const u8).init(arena); + defer zig_args.deinit(); + + try zig_args.append(b.graph.zig_exe); + + const cmd = switch (compile.kind) { + .lib => "build-lib", + .exe => "build-exe", + .obj => "build-obj", + .@"test" => "test", + .test_obj => "test-obj", + }; + try zig_args.append(cmd); + + if (b.reference_trace) |some| { + try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); + } + try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); + + try addFlag(&zig_args, "llvm", compile.use_llvm); + try addFlag(&zig_args, "lld", compile.use_lld); + try addFlag(&zig_args, "new-linker", compile.use_new_linker); + + if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { + try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); + } + + switch (compile.entry) { + .default => {}, + .disabled => try zig_args.append("-fno-entry"), + .enabled => try zig_args.append("-fentry"), + .symbol_name => |entry_name| { + try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); + }, + } + + { + for (compile.force_undefined_symbols.keys()) |symbol_name| { + try zig_args.append("--force_undefined"); + try zig_args.append(symbol_name.*); + } + } + + if (compile.stack_size) |stack_size| { + try zig_args.append("--stack"); + try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); + } + + if (fuzz) { + try zig_args.append("-ffuzz"); + } + + { + // Stores system libraries that have already been seen for at least one + // module, along with any arguments that need to be passed to the + // compiler for each module individually. + var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; + var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; + + var prev_has_cflags = false; + var prev_has_rcflags = false; + var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; + var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; + // Track the number of positional arguments so that a nice error can be + // emitted if there is nothing to link. + var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); + + // Fully recursive iteration including dynamic libraries to detect + // libc and libc++ linkage. + for (compile.getCompileDependencies(true)) |some_compile| { + for (some_compile.root_module.getGraph().modules) |mod| { + if (mod.link_libc == true) compile.is_linking_libc = true; + if (mod.link_libcpp == true) compile.is_linking_libcpp = true; + } + } + + var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); + + // For this loop, don't chase dynamic libraries because their link + // objects are already linked. + for (compile.getCompileDependencies(false)) |dep_compile| { + for (dep_compile.root_module.getGraph().modules) |mod| { + // While walking transitive dependencies, if a given link object is + // already included in a library, it should not redundantly be + // placed on the linker line of the dependee. + const my_responsibility = dep_compile == compile; + const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); + + // Inherit dependencies on darwin frameworks. + if (!already_linked) { + for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { + try frameworks.put(arena, name, info); + } + } + + // Inherit dependencies on system libraries and static libraries. + for (mod.link_objects.items) |link_object| { + switch (link_object) { + .static_path => |static_path| { + if (my_responsibility) { + try zig_args.append(static_path.getPath2(mod.owner, step)); + total_linker_objects += 1; + } + }, + .system_lib => |system_lib| { + const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); + if (system_lib_gop.found_existing) { + try zig_args.appendSlice(system_lib_gop.value_ptr.*); + continue; + } else { + system_lib_gop.value_ptr.* = &.{}; + } + + if (already_linked) + continue; + + if ((system_lib.search_strategy != prev_search_strategy or + system_lib.preferred_link_mode != prev_preferred_link_mode) and + compile.linkage != .static) + { + switch (system_lib.search_strategy) { + .no_fallback => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_dylibs_only"), + .static => try zig_args.append("-search_static_only"), + }, + .paths_first => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_paths_first"), + .static => try zig_args.append("-search_paths_first_static"), + }, + .mode_first => switch (system_lib.preferred_link_mode) { + .dynamic => try zig_args.append("-search_dylibs_first"), + .static => try zig_args.append("-search_static_first"), + }, + } + prev_search_strategy = system_lib.search_strategy; + prev_preferred_link_mode = system_lib.preferred_link_mode; + } + + const prefix: []const u8 = prefix: { + if (system_lib.needed) break :prefix "-needed-l"; + if (system_lib.weak) break :prefix "-weak-l"; + break :prefix "-l"; + }; + switch (system_lib.use_pkg_config) { + .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), + .yes, .force => { + if (compile.runPkgConfig(system_lib.name)) |result| { + try zig_args.appendSlice(result.cflags); + try zig_args.appendSlice(result.libs); + try seen_system_libs.put(arena, system_lib.name, result.cflags); + } else |err| switch (err) { + error.PkgConfigInvalidOutput, + error.PkgConfigCrashed, + error.PkgConfigFailed, + error.PkgConfigNotInstalled, + error.PackageNotFound, + => switch (system_lib.use_pkg_config) { + .yes => { + // pkg-config failed, so fall back to linking the library + // by name directly. + try zig_args.append(b.fmt("{s}{s}", .{ + prefix, + system_lib.name, + })); + }, + .force => { + panic("pkg-config failed for library {s}", .{system_lib.name}); + }, + .no => unreachable, + }, + + else => |e| return e, + } + }, + } + }, + .other_step => |other| { + switch (other.kind) { + .exe => return step.fail("cannot link with an executable build artifact", .{}), + .@"test" => return step.fail("cannot link with a test", .{}), + .obj, .test_obj => { + const included_in_lib_or_obj = !my_responsibility and + (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); + if (!already_linked and !included_in_lib_or_obj) { + try zig_args.append(other.getEmittedBin().getPath2(b, step)); + total_linker_objects += 1; + } + }, + .lib => l: { + const other_produces_implib = other.producesImplib(); + const other_is_static = other_produces_implib or other.isStaticLibrary(); + + if (compile.isStaticLibrary() and other_is_static) { + // Avoid putting a static library inside a static library. + break :l; + } + + // For DLLs, we must link against the implib. + // For everything else, we directly link + // against the library file. + const full_path_lib = if (other_produces_implib) + try other.getGeneratedFilePath("generated_implib", &compile.step) + else + try other.getGeneratedFilePath("generated_bin", &compile.step); + + try zig_args.append(full_path_lib); + total_linker_objects += 1; + + if (other.linkage == .dynamic and + compile.rootModuleTarget().os.tag != .windows) + { + if (fs.path.dirname(full_path_lib)) |dirname| { + try zig_args.append("-rpath"); + try zig_args.append(dirname); + } + } + }, + } + }, + .assembly_file => |asm_file| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags) { + try zig_args.append("-cflags"); + try zig_args.append("--"); + prev_has_cflags = false; + } + try zig_args.append(asm_file.getPath2(mod.owner, step)); + total_linker_objects += 1; + }, + + .c_source_file => |c_source_file| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags or c_source_file.flags.len != 0) { + try zig_args.append("-cflags"); + for (c_source_file.flags) |arg| { + try zig_args.append(arg); + } + try zig_args.append("--"); + } + prev_has_cflags = (c_source_file.flags.len != 0); + + if (c_source_file.language) |lang| { + try zig_args.append("-x"); + try zig_args.append(lang.internalIdentifier()); + } + + try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); + + if (c_source_file.language != null) { + try zig_args.append("-x"); + try zig_args.append("none"); + } + total_linker_objects += 1; + }, + + .c_source_files => |c_source_files| l: { + if (!my_responsibility) break :l; + + if (prev_has_cflags or c_source_files.flags.len != 0) { + try zig_args.append("-cflags"); + for (c_source_files.flags) |arg| { + try zig_args.append(arg); + } + try zig_args.append("--"); + } + prev_has_cflags = (c_source_files.flags.len != 0); + + if (c_source_files.language) |lang| { + try zig_args.append("-x"); + try zig_args.append(lang.internalIdentifier()); + } + + const root_path = c_source_files.root.getPath2(mod.owner, step); + for (c_source_files.files) |file| { + try zig_args.append(b.pathJoin(&.{ root_path, file })); + } + + if (c_source_files.language != null) { + try zig_args.append("-x"); + try zig_args.append("none"); + } + + total_linker_objects += c_source_files.files.len; + }, + + .win32_resource_file => |rc_source_file| l: { + if (!my_responsibility) break :l; + + if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { + if (prev_has_rcflags) { + try zig_args.append("-rcflags"); + try zig_args.append("--"); + prev_has_rcflags = false; + } + } else { + try zig_args.append("-rcflags"); + for (rc_source_file.flags) |arg| { + try zig_args.append(arg); + } + for (rc_source_file.include_paths) |include_path| { + try zig_args.append("/I"); + try zig_args.append(include_path.getPath2(mod.owner, step)); + } + try zig_args.append("--"); + prev_has_rcflags = true; + } + try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); + total_linker_objects += 1; + }, + } + } + + // We need to emit the --mod argument here so that the above link objects + // have the correct parent module, but only if the module is part of + // this compilation. + if (!my_responsibility) continue; + if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { + const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; + try mod.appendZigProcessFlags(&zig_args, step); + + // --dep arguments + try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); + for (mod.import_table.keys(), mod.import_table.values()) |name, import| { + const import_index = cli_named_modules.modules.getIndex(import).?; + const import_cli_name = cli_named_modules.names.keys()[import_index]; + zig_args.appendAssumeCapacity("--dep"); + if (std.mem.eql(u8, import_cli_name, name)) { + zig_args.appendAssumeCapacity(import_cli_name); + } else { + zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); + } + } + + // When the CLI sees a -M argument, it determines whether it + // implies the existence of a Zig compilation unit based on + // whether there is a root source file. If there is no root + // source file, then this is not a zig compilation unit - it is + // perhaps a set of linker objects, or C source files instead. + // Linker objects are added to the CLI globally, while C source + // files must have a module parent. + if (mod.root_source_file) |lp| { + const src = lp.getPath2(mod.owner, step); + try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); + } else if (moduleNeedsCliArg(mod)) { + try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); + } + } + } + } + + if (total_linker_objects == 0) { + return step.fail("the linker needs one or more objects to link", .{}); + } + + for (frameworks.keys(), frameworks.values()) |name, info| { + if (info.needed) { + try zig_args.append("-needed_framework"); + } else if (info.weak) { + try zig_args.append("-weak_framework"); + } else { + try zig_args.append("-framework"); + } + try zig_args.append(name); + } + + if (compile.is_linking_libcpp) { + try zig_args.append("-lc++"); + } + + if (compile.is_linking_libc) { + try zig_args.append("-lc"); + } + } + + if (compile.win32_manifest) |manifest_file| { + try zig_args.append(manifest_file.getPath2(b, step)); + } + + if (compile.win32_module_definition) |module_file| { + try zig_args.append(module_file.getPath2(b, step)); + } + + if (compile.image_base) |image_base| { + try zig_args.append("--image-base"); + try zig_args.append(b.fmt("0x{x}", .{image_base})); + } + + for (compile.filters) |filter| { + try zig_args.append("--test-filter"); + try zig_args.append(filter); + } + + if (compile.test_runner) |test_runner| { + try zig_args.append("--test-runner"); + try zig_args.append(test_runner.path.getPath2(b, step)); + } + + for (b.debug_log_scopes) |log_scope| { + try zig_args.append("--debug-log"); + try zig_args.append(log_scope); + } + + if (b.debug_compile_errors) { + try zig_args.append("--debug-compile-errors"); + } + + if (b.debug_incremental) { + try zig_args.append("--debug-incremental"); + } + + if (b.verbose_air) try zig_args.append("--verbose-air"); + if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); + if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); + if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); + if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); + if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); + if (b.graph.time_report) try zig_args.append("--time-report"); + + if (compile.generated_asm != null) try zig_args.append("-femit-asm"); + if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); + if (compile.generated_docs != null) try zig_args.append("-femit-docs"); + if (compile.generated_implib != null) try zig_args.append("-femit-implib"); + if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); + if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); + if (compile.generated_h != null) try zig_args.append("-femit-h"); + + try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); + + switch (compile.compress_debug_sections) { + .none => {}, + .zlib => try zig_args.append("--compress-debug-sections=zlib"), + .zstd => try zig_args.append("--compress-debug-sections=zstd"), + } + + if (compile.link_eh_frame_hdr) { + try zig_args.append("--eh-frame-hdr"); + } + if (compile.link_emit_relocs) { + try zig_args.append("--emit-relocs"); + } + if (compile.link_function_sections) { + try zig_args.append("-ffunction-sections"); + } + if (compile.link_data_sections) { + try zig_args.append("-fdata-sections"); + } + if (compile.link_gc_sections) |x| { + try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); + } + if (!compile.linker_dynamicbase) { + try zig_args.append("--no-dynamicbase"); + } + if (compile.linker_allow_shlib_undefined) |x| { + try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); + } + if (compile.link_z_notext) { + try zig_args.append("-z"); + try zig_args.append("notext"); + } + if (!compile.link_z_relro) { + try zig_args.append("-z"); + try zig_args.append("norelro"); + } + if (compile.link_z_lazy) { + try zig_args.append("-z"); + try zig_args.append("lazy"); + } + if (compile.link_z_common_page_size) |size| { + try zig_args.append("-z"); + try zig_args.append(b.fmt("common-page-size={d}", .{size})); + } + if (compile.link_z_max_page_size) |size| { + try zig_args.append("-z"); + try zig_args.append(b.fmt("max-page-size={d}", .{size})); + } + if (compile.link_z_defs) { + try zig_args.append("-z"); + try zig_args.append("defs"); + } + + if (compile.libc_file) |libc_file| { + try zig_args.append("--libc"); + try zig_args.append(libc_file.getPath2(b, step)); + } else if (b.libc_file) |libc_file| { + try zig_args.append("--libc"); + try zig_args.append(libc_file); + } + + try zig_args.append("--cache-dir"); + try zig_args.append(b.cache_root.path orelse "."); + + try zig_args.append("--global-cache-dir"); + try zig_args.append(b.graph.global_cache_root.path orelse "."); + + if (b.graph.debug_compiler_runtime_libs) |mode| + try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); + + try zig_args.append("--name"); + try zig_args.append(compile.name); + + if (compile.linkage) |some| switch (some) { + .dynamic => try zig_args.append("-dynamic"), + .static => try zig_args.append("-static"), + }; + if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { + if (compile.version) |version| { + try zig_args.append("--version"); + try zig_args.append(b.fmt("{f}", .{version})); + } + + if (compile.rootModuleTarget().os.tag.isDarwin()) { + const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ + compile.rootModuleTarget().libPrefix(), + compile.name, + compile.rootModuleTarget().dynamicLibSuffix(), + }); + try zig_args.append("-install_name"); + try zig_args.append(install_name); + } + } + + if (compile.entitlements) |entitlements| { + try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); + } + if (compile.pagezero_size) |pagezero_size| { + const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); + try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); + } + if (compile.headerpad_size) |headerpad_size| { + const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); + try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); + } + if (compile.headerpad_max_install_names) { + try zig_args.append("-headerpad_max_install_names"); + } + if (compile.dead_strip_dylibs) { + try zig_args.append("-dead_strip_dylibs"); + } + if (compile.force_load_objc) { + try zig_args.append("-ObjC"); + } + if (compile.discard_local_symbols) { + try zig_args.append("--discard-all"); + } + + try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); + try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); + try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); + if (compile.rdynamic) { + try zig_args.append("-rdynamic"); + } + if (compile.import_memory) { + try zig_args.append("--import-memory"); + } + if (compile.export_memory) { + try zig_args.append("--export-memory"); + } + if (compile.import_symbols) { + try zig_args.append("--import-symbols"); + } + if (compile.import_table) { + try zig_args.append("--import-table"); + } + if (compile.export_table) { + try zig_args.append("--export-table"); + } + if (compile.initial_memory) |initial_memory| { + try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); + } + if (compile.max_memory) |max_memory| { + try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); + } + if (compile.shared_memory) { + try zig_args.append("--shared-memory"); + } + if (compile.global_base) |global_base| { + try zig_args.append(b.fmt("--global-base={d}", .{global_base})); + } + + if (compile.wasi_exec_model) |model| { + try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); + } + if (compile.linker_script) |linker_script| { + try zig_args.append("--script"); + try zig_args.append(linker_script.getPath2(b, step)); + } + + if (compile.version_script) |version_script| { + try zig_args.append("--version-script"); + try zig_args.append(version_script.getPath2(b, step)); + } + if (compile.linker_allow_undefined_version) |x| { + try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); + } + + if (compile.linker_enable_new_dtags) |enabled| { + try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); + } + + if (compile.kind == .@"test") { + if (compile.exec_cmd_args) |exec_cmd_args| { + for (exec_cmd_args) |cmd_arg| { + if (cmd_arg) |arg| { + try zig_args.append("--test-cmd"); + try zig_args.append(arg); + } else { + try zig_args.append("--test-cmd-bin"); + } + } + } + } + + if (b.sysroot) |sysroot| { + try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); + } + + // -I and -L arguments that appear after the last --mod argument apply to all modules. + const cwd: Io.Dir = .cwd(); + const io = b.graph.io; + + for (b.search_prefixes.items) |search_prefix| { + var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { + return step.fail("unable to open prefix directory '{s}': {s}", .{ + search_prefix, @errorName(err), + }); + }; + defer prefix_dir.close(io); + + // Avoid passing -L and -I flags for nonexistent directories. + // This prevents a warning, that should probably be upgraded to an error in Zig's + // CLI parsing code, when the linker sees an -L directory that does not exist. + + if (prefix_dir.access(io, "lib", .{})) |_| { + try zig_args.appendSlice(&.{ + "-L", b.pathJoin(&.{ search_prefix, "lib" }), + }); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ + search_prefix, @errorName(e), + }), + } + + if (prefix_dir.access(io, "include", .{})) |_| { + try zig_args.appendSlice(&.{ + "-I", b.pathJoin(&.{ search_prefix, "include" }), + }); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ + search_prefix, @errorName(e), + }), + } + } + + if (compile.rc_includes != .any) { + try zig_args.append("-rcincludes"); + try zig_args.append(@tagName(compile.rc_includes)); + } + + try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); + + if (compile.build_id orelse b.build_id) |build_id| { + try zig_args.append(switch (build_id) { + .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), + .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), + }); + } + + const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| + dir.getPath2(b, step) + else if (b.graph.zig_lib_directory.path) |_| + b.fmt("{f}", .{b.graph.zig_lib_directory}) + else + null; + + if (opt_zig_lib_dir) |zig_lib_dir| { + try zig_args.append("--zig-lib-dir"); + try zig_args.append(zig_lib_dir); + } + + try addFlag(&zig_args, "PIE", compile.pie); + + if (compile.lto) |lto| { + try zig_args.append(switch (lto) { + .full => "-flto=full", + .thin => "-flto=thin", + .none => "-fno-lto", + }); + } + + try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); + + if (compile.subsystem) |subsystem| { + try zig_args.append("--subsystem"); + try zig_args.append(@tagName(subsystem)); + } + + if (compile.mingw_unicode_entry_point) { + try zig_args.append("-municode"); + } + + if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ + "--error-limit", b.fmt("{d}", .{err_limit}), + }); + + try addFlag(&zig_args, "incremental", b.graph.incremental); + + try zig_args.append("--listen=-"); + + // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux + // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and + // pass that to zig, e.g. via 'zig build-lib @args.rsp' + // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html + var args_length: usize = 0; + for (zig_args.items) |arg| { + args_length += arg.len + 1; // +1 to account for null terminator + } + if (args_length >= 30 * 1024) { + try b.cache_root.handle.createDirPath(io, "args"); + + const args_to_escape = zig_args.items[2..]; + var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); + arg_blk: for (args_to_escape) |arg| { + for (arg, 0..) |c, arg_idx| { + if (c == '\\' or c == '"') { + // Slow path for arguments that need to be escaped. We'll need to allocate and copy + var escaped: std.ArrayList(u8) = .empty; + try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1); + try escaped.appendSlice(arena, arg[0..arg_idx]); + for (arg[arg_idx..]) |to_escape| { + if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\'); + try escaped.append(arena, to_escape); + } + escaped_args.appendAssumeCapacity(escaped.items); + continue :arg_blk; + } + } + escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument + } + + // Write the args to zig-cache/args/ to avoid conflicts with + // other zig build commands running in parallel. + const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); + const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); + + var args_hash: [Sha256.digest_length]u8 = undefined; + Sha256.hash(args, &args_hash, .{}); + var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; + _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); + + const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; + if (b.cache_root.handle.access(io, args_file, .{})) |_| { + // The args file is already present from a previous run. + } else |err| switch (err) { + error.FileNotFound => { + var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ + .replace = false, + .make_path = true, + }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ + b.cache_root, args_file, e, + }); + defer af.deinit(io); + + af.file.writeStreamingAll(io, args) catch |e| { + return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ + b.cache_root, args_file, e, + }); + }; + // Note we can't clean up this file, not even after build + // success, because that might interfere with another build + // process that needs the same file. + af.link(io) catch |e| switch (e) { + error.PathAlreadyExists => { + // The args file was created by another concurrent build process. + }, + else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ + b.cache_root, args_file, other_err, + }), + }; + }, + else => |other_err| return other_err, + } + + const resolved_args_file = try mem.concat(arena, u8, &.{ + "@", + try b.cache_root.join(arena, &.{args_file}), + }); + + zig_args.shrinkRetainingCapacity(2); + try zig_args.append(resolved_args_file); + } + + return try zig_args.toOwnedSlice(); +} + +pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { + c.step.result_error_msgs.clearRetainingCapacity(); + c.step.result_stderr = ""; + + c.step.result_error_bundle.deinit(gpa); + c.step.result_error_bundle = std.zig.ErrorBundle.empty; + + if (c.step.result_failed_command) |cmd| { + gpa.free(cmd); + c.step.result_failed_command = null; + } + + const zig_args = try getZigArgs(c, true); + const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); + return maybe_output_bin_path.?; +} + +pub fn doAtomicSymLinks( + step: *Step, + output_path: []const u8, + filename_major_only: []const u8, + filename_name_only: []const u8, +) !void { + const b = step.owner; + const io = b.graph.io; + const out_dir = fs.path.dirname(output_path) orelse "."; + const out_basename = fs.path.basename(output_path); + // sym link for libfoo.so.1 to libfoo.so.1.2.3 + const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); + const cwd: Io.Dir = .cwd(); + cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { + return step.fail("unable to symlink {s} -> {s}: {s}", .{ + major_only_path, out_basename, @errorName(err), + }); + }; + // sym link for libfoo.so to libfoo.so.1 + const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); + cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { + return step.fail("Unable to symlink {s} -> {s}: {s}", .{ + name_only_path, filename_major_only, @errorName(err), + }); + }; +} + +fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { + const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); + var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); + errdefer list.deinit(); + var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); + while (line_it.next()) |line| { + if (mem.trim(u8, line, " \t").len == 0) continue; + var tok_it = mem.tokenizeAny(u8, line, " \t"); + try list.append(PkgConfigPkg{ + .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, + .desc = tok_it.rest(), + }); + } + return list.toOwnedSlice(); +} + +fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { + if (b.pkg_config_pkg_list) |res| { + return res; + } + var code: u8 = undefined; + if (execPkgConfigList(b, &code)) |list| { + b.pkg_config_pkg_list = list; + return list; + } else |err| { + const result = switch (err) { + error.ProcessTerminated => error.PkgConfigCrashed, + error.ExecNotSupported => error.PkgConfigFailed, + error.ExitCodeFailure => error.PkgConfigFailed, + error.FileNotFound => error.PkgConfigNotInstalled, + error.InvalidName => error.PkgConfigNotInstalled, + error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, + else => return err, + }; + b.pkg_config_pkg_list = result; + return result; + } +} + +fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { + const cond = opt orelse return; + try args.ensureUnusedCapacity(1); + if (cond) { + args.appendAssumeCapacity("-f" ++ name); + } else { + args.appendAssumeCapacity("-fno-" ++ name); + } +} + +const PkgConfigResult = struct { + cflags: []const []const u8, + libs: []const []const u8, +}; + +/// Run pkg-config for the given library name and parse the output, returning the arguments +/// that should be passed to zig to link the given library. +fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult { + const wl_rpath_prefix = "-Wl,-rpath,"; + + const b = compile.step.owner; + const arena = b.allocator; + const pkg_name = match: { + // First we have to map the library name to pkg config name. Unfortunately, + // there are several examples where this is not straightforward: + // -lSDL2 -> pkg-config sdl2 + // -lgdk-3 -> pkg-config gdk-3.0 + // -latk-1.0 -> pkg-config atk + // -lpulse -> pkg-config libpulse + const pkgs = try getPkgConfigList(b); + + // Exact match means instant winner. + for (pkgs) |pkg| { + if (mem.eql(u8, pkg.name, lib_name)) { + break :match pkg.name; + } + } + + // Next we'll try ignoring case. + for (pkgs) |pkg| { + if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { + break :match pkg.name; + } + } + + // Prefixed "lib" or suffixed ".0". + for (pkgs) |pkg| { + if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { + const prefix = pkg.name[0..pos]; + const suffix = pkg.name[pos + lib_name.len ..]; + if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; + if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; + break :match pkg.name; + } + } + + // Trimming "-1.0". + if (mem.endsWith(u8, lib_name, "-1.0")) { + const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; + for (pkgs) |pkg| { + if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { + break :match pkg.name; + } + } + } + + return error.PackageNotFound; + }; + + var code: u8 = undefined; + const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = if (b.runAllowFail(&[_][]const u8{ + pkg_config_exe, + pkg_name, + "--cflags", + "--libs", + }, &code, .ignore)) |stdout| stdout else |err| switch (err) { + error.ProcessTerminated => return error.PkgConfigCrashed, + error.ExecNotSupported => return error.PkgConfigFailed, + error.ExitCodeFailure => return error.PkgConfigFailed, + error.FileNotFound => return error.PkgConfigNotInstalled, + else => return err, + }; + + var zig_cflags: std.ArrayList([]const u8) = .empty; + defer zig_cflags.deinit(arena); + var zig_libs: std.ArrayList([]const u8) = .empty; + defer zig_libs.deinit(arena); + + var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); + while (arg_it.next()) |arg| { + if (mem.eql(u8, arg, "-I")) { + const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_cflags.appendSlice(arena, &.{ "-I", dir }); + } else if (mem.startsWith(u8, arg, "-I")) { + try zig_cflags.append(arena, arg); + } else if (mem.eql(u8, arg, "-L")) { + const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_libs.appendSlice(arena, &.{ "-L", dir }); + } else if (mem.startsWith(u8, arg, "-L")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-l")) { + const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_libs.appendSlice(arena, &.{ "-l", lib }); + } else if (mem.startsWith(u8, arg, "-l")) { + try zig_libs.append(arena, arg); + } else if (mem.eql(u8, arg, "-D")) { + const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; + try zig_cflags.appendSlice(arena, &.{ "-D", macro }); + } else if (mem.startsWith(u8, arg, "-D")) { + try zig_cflags.append(arena, arg); + } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { + try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); + } else if (b.debug_pkg_config) { + return compile.step.fail("unknown pkg-config flag '{s}'", .{arg}); + } + } + + try zig_cflags.shrinkToLen(arena); + try zig_libs.shrinkToLen(arena); + + return .{ + .cflags = zig_cflags.toOwnedSliceAssert(), + .libs = zig_libs.toOwnedSliceAssert(), + }; +} + +fn checkCompileErrors(compile: *Compile) !void { + // Clear this field so that it does not get printed by the build runner. + const actual_eb = compile.step.result_error_bundle; + compile.step.result_error_bundle = .empty; + + const arena = compile.step.owner.allocator; + + const actual_errors = ae: { + var aw: std.Io.Writer.Allocating = .init(arena); + defer aw.deinit(); + try actual_eb.renderToWriter(.{ + .include_reference_trace = false, + .include_source_line = false, + }, &aw.writer); + break :ae try aw.toOwnedSlice(); + }; + + // Render the expected lines into a string that we can compare verbatim. + var expected_generated: std.ArrayList(u8) = .empty; + const expect_errors = compile.expect_errors.?; + + var actual_line_it = mem.splitScalar(u8, actual_errors, '\n'); + + // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile + switch (expect_errors) { + .starts_with => |expect_starts_with| { + if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return; + return compile.step.fail( + \\ + \\========= should start with: ============ + \\{s} + \\========= but not found: ================ + \\{s} + \\========================================= + , .{ expect_starts_with, actual_errors }); + }, + .contains => |expect_line| { + while (actual_line_it.next()) |actual_line| { + if (!matchCompileError(actual_line, expect_line)) continue; + return; + } + + return compile.step.fail( + \\ + \\========= should contain: =============== + \\{s} + \\========= but not found: ================ + \\{s} + \\========================================= + , .{ expect_line, actual_errors }); + }, + .stderr_contains => |expect_line| { + const actual_stderr: []const u8 = if (compile.step.result_error_msgs.items.len > 0) + compile.step.result_error_msgs.items[0] + else + &.{}; + compile.step.result_error_msgs.clearRetainingCapacity(); + + var stderr_line_it = mem.splitScalar(u8, actual_stderr, '\n'); + + while (stderr_line_it.next()) |actual_line| { + if (!matchCompileError(actual_line, expect_line)) continue; + return; + } + + return compile.step.fail( + \\ + \\========= should contain: =============== + \\{s} + \\========= but not found: ================ + \\{s} + \\========================================= + , .{ expect_line, actual_stderr }); + }, + .exact => |expect_lines| { + for (expect_lines) |expect_line| { + const actual_line = actual_line_it.next() orelse { + try expected_generated.appendSlice(arena, expect_line); + try expected_generated.append(arena, '\n'); + continue; + }; + if (matchCompileError(actual_line, expect_line)) { + try expected_generated.appendSlice(arena, actual_line); + try expected_generated.append(arena, '\n'); + continue; + } + try expected_generated.appendSlice(arena, expect_line); + try expected_generated.append(arena, '\n'); + } + + if (mem.eql(u8, expected_generated.items, actual_errors)) return; + + return compile.step.fail( + \\ + \\========= expected: ===================== + \\{s} + \\========= but found: ==================== + \\{s} + \\========================================= + , .{ expected_generated.items, actual_errors }); + }, + } +} + +fn matchCompileError(actual: []const u8, expected: []const u8) bool { + if (mem.endsWith(u8, actual, expected)) return true; + if (mem.startsWith(u8, expected, ":?:?: ")) { + if (mem.endsWith(u8, actual, expected[":?:?: ".len..])) return true; + } + // We scan for /?/ in expected line and if there is a match, we match everything + // up to and after /?/. + const expected_trim = mem.trim(u8, expected, " "); + if (mem.find(u8, expected_trim, "/?/")) |index| { + const actual_trim = mem.trim(u8, actual, " "); + const lhs = expected_trim[0..index]; + const rhs = expected_trim[index + "/?/".len ..]; + if (mem.startsWith(u8, actual_trim, lhs) and mem.endsWith(u8, actual_trim, rhs)) return true; + } + return false; +} + +fn moduleNeedsCliArg(mod: *const Module) bool { + return for (mod.link_objects.items) |o| switch (o) { + .c_source_file, .c_source_files, .assembly_file, .win32_resource_file => break true, + else => continue, + } else false; +} + diff --git a/lib/compiler/Maker/Step/InstallArtifact.zig b/lib/compiler/Maker/Step/InstallArtifact.zig new file mode 100644 index 0000000000000000000000000000000000000000..ba3846c1a574f9934f4196f03529dac3abcbe275 --- /dev/null +++ b/lib/compiler/Maker/Step/InstallArtifact.zig @@ -0,0 +1,96 @@ + +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; + const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); + const b = step.owner; + const io = b.graph.io; + + var all_cached = true; + + if (install_artifact.dest_dir) |dest_dir| { + const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); + const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); + all_cached = all_cached and p == .fresh; + + if (install_artifact.dylib_symlinks) |dls| { + try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); + } + + install_artifact.artifact.installed_path = full_dest_path; + } + + if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { + const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.implib_dir) |implib_dir| { + const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.pdb_dir) |pdb_dir| { + const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); + const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); + all_cached = all_cached and p == .fresh; + } + + if (install_artifact.h_dir) |h_dir| { + if (install_artifact.emitted_h) |emitted_h| { + const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); + const p = try step.installFile(emitted_h, full_h_path); + all_cached = all_cached and p == .fresh; + } + + for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { + .file => |file| { + const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); + const p = try step.installFile(file.source, full_h_path); + all_cached = all_cached and p == .fresh; + }, + .directory => |dir| { + const src_dir_path = dir.source.getPath3(b, step); + const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); + + var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{f}': {s}", .{ + src_dir_path, @errorName(err), + }); + }; + defer src_dir.close(io); + + var it = try src_dir.walk(b.allocator); + next_entry: while (try it.next(io)) |entry| { + for (dir.options.exclude_extensions) |ext| { + if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; + } + if (dir.options.include_extensions) |incs| { + for (incs) |inc| { + if (std.mem.endsWith(u8, entry.path, inc)) break; + } else { + continue :next_entry; + } + } + + const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); + switch (entry.kind) { + .directory => { + try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); + const p = try step.installDir(full_dest_path); + all_cached = all_cached and p == .existed; + }, + .file => { + const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); + all_cached = all_cached and p == .fresh; + }, + else => continue, + } + } + }, + }; + } + + step.result_cached = all_cached; +} diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig new file mode 100644 index 0000000000000000000000000000000000000000..4ba04092e02221fffc55729a64b44aa41a5b9d91 --- /dev/null +++ b/lib/compiler/Maker/Step/Run.zig @@ -0,0 +1,2130 @@ +const Run = @This(); + +const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; +const Dir = std.Io.Dir; +const mem = std.mem; +const process = std.process; +const EnvMap = std.process.Environ.Map; +const assert = std.debug.assert; +const Cache = std.Build.Cache; +const Path = std.Build.Cache.Path; + +const Step = @import("../Step.zig"); + +/// If this is a Zig unit test binary, this tracks the names of the unit +/// tests that are also fuzz tests. Indexes cannot be used as they may +/// change between reruns. +fuzz_tests: std.ArrayList([]const u8), +cached_test_metadata: ?CachedTestMetadata = null, + +/// Populated during the fuzz phase if this run step corresponds to a unit test +/// executable that contains fuzz tests. +rebuilt_executable: ?Path, + +fn make(step: *Step, options: Step.MakeOptions) !void { + const b = step.owner; + const io = b.graph.io; + const arena = b.allocator; + const run: *Run = @fieldParentPtr("step", step); + const has_side_effects = run.hasSideEffects(); + + var argv_list = std.array_list.Managed([]const u8).init(arena); + var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); + + var man = b.graph.cache.obtain(); + defer man.deinit(); + + if (run.environ_map) |environ_map| { + for (environ_map.keys(), environ_map.values()) |key, value| { + man.hash.addBytes(key); + man.hash.addBytes(value); + } + } + + man.hash.add(run.color); + man.hash.add(run.disable_zig_progress); + + for (run.argv.items) |arg| { + switch (arg) { + .bytes => |bytes| { + try argv_list.append(bytes); + man.hash.addBytes(bytes); + }, + .lazy_path => |file| { + const file_path = file.lazy_path.getPath3(b, step); + try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); + man.hash.addBytes(file.prefix); + _ = try man.addFilePath(file_path, null); + }, + .decorated_directory => |dd| { + const file_path = dd.lazy_path.getPath3(b, step); + const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); + try argv_list.append(resolved_arg); + man.hash.addBytes(resolved_arg); + }, + .file_content => |file_plp| { + const file_path = file_plp.lazy_path.getPath3(b, step); + + var result: std.Io.Writer.Allocating = .init(arena); + errdefer result.deinit(); + result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + + const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { + return step.fail( + "unable to open input file '{f}': {t}", + .{ file_path, err }, + ); + }; + defer file.close(io); + + var buf: [1024]u8 = undefined; + var file_reader = file.reader(io, &buf); + _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { + error.ReadFailed => return step.fail( + "failed to read from '{f}': {t}", + .{ file_path, file_reader.err.? }, + ), + error.WriteFailed => return error.OutOfMemory, + }; + + try argv_list.append(result.written()); + man.hash.addBytes(file_plp.prefix); + _ = try man.addFilePath(file_path, null); + }, + .artifact => |pa| { + const artifact = pa.artifact; + + if (artifact.rootModuleTarget().os.tag == .windows) { + // On Windows we don't have rpaths so we have to add .dll search paths to PATH + run.addPathForDynLibs(artifact); + } + const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; + + try argv_list.append(b.fmt("{s}{s}", .{ + pa.prefix, + run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), + })); + + _ = try man.addFile(file_path, null); + }, + .output_file, .output_directory => |output| { + man.hash.addBytes(output.prefix); + man.hash.addBytes(output.basename); + // Add a placeholder into the argument list because we need the + // manifest hash to be updated with all arguments before the + // object directory is computed. + try output_placeholders.append(.{ + .index = argv_list.items.len, + .tag = arg, + .output = output, + }); + _ = try argv_list.addOne(); + }, + } + } + + switch (run.stdin) { + .bytes => |bytes| { + man.hash.addBytes(bytes); + }, + .lazy_path => |lazy_path| { + const file_path = lazy_path.getPath2(b, step); + _ = try man.addFile(file_path, null); + }, + .none => {}, + } + + if (run.captured_stdout) |captured| { + man.hash.addBytes(captured.output.basename); + man.hash.add(captured.trim_whitespace); + } + + if (run.captured_stderr) |captured| { + man.hash.addBytes(captured.output.basename); + man.hash.add(captured.trim_whitespace); + } + + hashStdIo(&man.hash, run.stdio); + + for (run.file_inputs.items) |lazy_path| { + _ = try man.addFile(lazy_path.getPath2(b, step), null); + } + + if (run.cwd) |cwd| { + const cwd_path = cwd.getPath3(b, step); + _ = man.hash.addBytes(try cwd_path.toString(arena)); + } + + if (!has_side_effects and try step.cacheHitAndWatch(&man)) { + // cache hit, skip running command + const digest = man.final(); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); + + step.result_cached = true; + return; + } + + const dep_output_file = run.dep_output_file orelse { + // We already know the final output paths, use them directly. + const digest = if (has_side_effects) + man.hash.final() + else + man.final(); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); + + const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; + for (output_placeholders.items) |placeholder| { + const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); + const output_sub_dir_path = switch (placeholder.tag) { + .output_file => Dir.path.dirname(output_sub_path).?, + .output_directory => output_sub_path, + else => unreachable, + }; + b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, output_sub_dir_path, @errorName(err), + }); + }; + const arg_output_path = run.convertPathArg(.{ + .root_dir = .cwd(), + .sub_path = placeholder.output.generated_file.getPath(), + }); + argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) + arg_output_path + else + b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); + } + + try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); + if (!has_side_effects) try step.writeManifestAndWatch(&man); + return; + }; + + // We do not know the final output paths yet, use temp paths to run the command. + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + + for (output_placeholders.items) |placeholder| { + const output_components = .{ tmp_dir_path, placeholder.output.basename }; + const output_sub_path = b.pathJoin(&output_components); + const output_sub_dir_path = switch (placeholder.tag) { + .output_file => Dir.path.dirname(output_sub_path).?, + .output_directory => output_sub_path, + else => unreachable, + }; + b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, output_sub_dir_path, @errorName(err), + }); + }; + const raw_output_path: Cache.Path = .{ + .root_dir = b.cache_root, + .sub_path = b.pathJoin(&output_components), + }; + placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); + argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ + placeholder.output.prefix, + run.convertPathArg(raw_output_path), + }); + } + + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); + + const dep_file_dir = Dir.cwd(); + const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); + if (has_side_effects) + try man.addDepFile(dep_file_dir, dep_file_basename) + else + try man.addDepFilePost(dep_file_dir, dep_file_basename); + + const digest = if (has_side_effects) + man.hash.final() + else + man.final(); + + const any_output = output_placeholders.items.len > 0 or + run.captured_stdout != null or run.captured_stderr != null; + + // Rename into place + if (any_output) { + const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; + + b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { + Dir.RenameError.DirNotEmpty => { + b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { + return step.fail("unable to remove dir '{f}'{s}: {t}", .{ + b.cache_root, tmp_dir_path, del_err, + }); + }; + b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { + return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ + b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, + }); + }; + }, + else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ + b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, + }), + }; + } + + if (!has_side_effects) try step.writeManifestAndWatch(&man); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + run.captured_stdout, + run.captured_stderr, + b.cache_root, + &digest, + ); +} + +/// Reads stdout of a Zig test process until a termination condition is reached: +/// * A write fails, indicating the child unexpectedly closed stdin +/// * A test (or a response from the test runner) times out +/// * The wait fails, indicating the child closed stdout and stderr +fn waitZigTest( + run: *Run, + child: *process.Child, + options: Step.MakeOptions, + multi_reader: *Io.File.MultiReader, + opt_metadata: *?TestMetadata, + results: *Step.TestResults, +) !union(enum) { + write_failed: anyerror, + no_poll: struct { + active_test_index: ?u32, + ns_elapsed: u64, + }, + timeout: struct { + active_test_index: ?u32, + ns_elapsed: u64, + }, +} { + const gpa = run.step.owner.allocator; + const arena = run.step.owner.allocator; + const io = run.step.owner.graph.io; + + var sub_prog_node: ?std.Progress.Node = null; + defer if (sub_prog_node) |n| n.end(); + + if (opt_metadata.*) |*md| { + // Previous unit test process died or was killed; we're continuing where it left off + requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; + } else { + // Running unit tests normally + run.fuzz_tests.clearRetainingCapacity(); + sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err }; + } + + var active_test_index: ?u32 = null; + + var last_update: Io.Clock.Timestamp = .now(io, .awake); + + // This timeout is used when we're waiting on the test runner itself rather than a user-specified + // test. For instance, if the test runner leaves this much time between us requesting a test to + // start and it acknowledging the test starting, we terminate the child and raise an error. This + // *should* never happen, but could in theory be caused by some very unlucky IB in a test. + const response_timeout: Io.Clock.Duration = t: { + if (fuzz_context != null) break :t null; // don't timeout fuzz tests + const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); + break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; + }; + const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ + .clock = .awake, + .raw = .fromNanoseconds(ns), + } else null; + + const stdout = multi_reader.reader(0); + const stderr = multi_reader.reader(1); + const Header = std.zig.Server.Message.Header; + + while (true) { + const timeout: Io.Timeout = t: { + const opt_duration = if (active_test_index == null) response_timeout else test_timeout; + const duration = opt_duration orelse break :t .none; + break :t .{ .deadline = last_update.addDuration(duration) }; + }; + + // This block is exited when `stdout` contains enough bytes for a `Header`. + header_ready: { + if (stdout.buffered().len >= @sizeOf(Header)) { + // We already have one, no need to poll! + break :header_ready; + } + + multi_reader.fill(64, timeout) catch |err| switch (err) { + error.Timeout => return .{ .timeout = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + error.EndOfStream => return .{ .no_poll = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + else => |e| return e, + }; + + continue; + } + // There is definitely a header available now -- read it. + const header = stdout.takeStruct(Header, .little) catch unreachable; + + while (stdout.buffered().len < header.bytes_len) { + multi_reader.fill(64, timeout) catch |err| switch (err) { + error.Timeout => return .{ .timeout = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + error.EndOfStream => return .{ .no_poll = .{ + .active_test_index = active_test_index, + .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), + } }, + else => |e| return e, + }; + } + + const body = stdout.take(header.bytes_len) catch unreachable; + var body_r: std.Io.Reader = .fixed(body); + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + }, + .test_metadata => { + // `metadata` would only be populated if we'd already seen a `test_metadata`, but we + // only request it once (and importantly, we don't re-request it if we kill and + // restart the test runner). + assert(opt_metadata.* == null); + + const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable; + results.test_count = tm_hdr.tests_len; + + const names = try arena.alloc(u32, results.test_count); + for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; + + const expected_panic_msgs = try arena.alloc(u32, results.test_count); + for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; + + const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; + + options.progress_node.setEstimatedTotalItems(names.len); + opt_metadata.* = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .ns_per_test = try arena.alloc(u64, results.test_count), + .names = names, + .expected_panic_msgs = expected_panic_msgs, + .next_index = 0, + .prog_node = options.progress_node, + }; + @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); + + active_test_index = null; + last_update = .now(io, .awake); + + requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err }; + }, + .test_started => { + active_test_index = opt_metadata.*.?.next_index - 1; + last_update = .now(io, .awake); + }, + .test_results => { + const md = &opt_metadata.*.?; + + const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable; + assert(tr_hdr.index == active_test_index); + + switch (tr_hdr.flags.status) { + .pass => {}, + .skip => results.skip_count +|= 1, + .fail => results.fail_count +|= 1, + } + const leak_count = tr_hdr.flags.leak_count; + const log_err_count = tr_hdr.flags.log_err_count; + results.leak_count +|= leak_count; + results.log_err_count +|= log_err_count; + + if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index)); + + if (tr_hdr.flags.status == .fail) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + if (stderr_bytes.len == 0) { + try run.step.addError("'{s}' failed without output", .{name}); + } else { + try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); + } + } else if (leak_count > 0) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); + } else if (log_err_count > 0) { + const name = md.testName(tr_hdr.index); + const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); + stderr.tossBuffered(); + try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); + } + + active_test_index = null; + + const now: Io.Clock.Timestamp = .now(io, .awake); + md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds); + last_update = now; + + requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; + }, + else => {}, // ignore other messages + } + } +} + +const FuzzTestRunner = struct { + run: *Run, + ctx: FuzzContext, + coverage_id: ?u64, + + instances: []Instance, + /// The indexes of this are layed out such that it is effectively an array + /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr. + batch: Io.Batch, + /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter. + pending_broadcasts: std.ArrayList(u8), + broadcast: std.ArrayList(u8), + broadcast_undelivered: u32, + + const Instance = struct { + child: process.Child, + message: std.ArrayListAligned(u8, .@"4"), + broadcast_written: usize, + stderr: std.ArrayList(u8), + stdin_vec: [1][]u8, + stdout_vec: [1][]u8, + stderr_vec: [1][]u8, + progress_node: std.Progress.Node, + + fn messageHeader(instance: *Instance) InHeader { + assert(instance.message.items.len >= @sizeOf(InHeader)); + const header_ptr: *InHeader = @ptrCast(instance.message.items); + var header = header_ptr.*; + if (std.builtin.Endian.native != .little) { + std.mem.byteSwapAllFields(InHeader, &header); + } + return header; + } + }; + + const PendingBroadcastFooter = struct { + from_id: u32, + body_len: u32, + }; + + const InHeader = std.zig.Server.Message.Header; + const OutHeader = std.zig.Client.Message.Header; + + const stdin_i = 0; + const stdout_i = 1; + const stderr_i = 2; + + fn init( + run: *Run, + ctx: FuzzContext, + progress_node: std.Progress.Node, + spawn_options: process.SpawnOptions, + ) !FuzzTestRunner { + const step_owner = run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + + const n_instances = switch (ctx.fuzz.mode) { + .forever => step_owner.graph.max_jobs orelse @min( + std.Thread.getCpuCount() catch 1, + (std.math.maxInt(u32) - 2) / 3, + ), + .limit => 1, + }; + const instances = try gpa.alloc(Instance, n_instances); + errdefer gpa.free(instances); + const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3); + errdefer gpa.free(batch_storage); + + @memset(instances, .{ + .child = undefined, + .message = .empty, + .broadcast_written = undefined, + .stderr = .empty, + .stdin_vec = undefined, + .stdout_vec = undefined, + .stderr_vec = undefined, + .progress_node = undefined, + }); + for (0.., instances) |id, *instance| { + errdefer for (instances[0..id]) |*spawned| { + spawned.child.kill(io); + spawned.progress_node.end(); + }; + instance.child = try process.spawn(io, spawn_options); + instance.progress_node = progress_node.start("starting fuzzer", 0); + } + + return .{ + .run = run, + .ctx = ctx, + .coverage_id = null, + + .instances = instances, + .batch = .init(batch_storage), + .pending_broadcasts = .empty, + .broadcast = .empty, + .broadcast_undelivered = 0, + }; + } + + fn deinit(f: *FuzzTestRunner) void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + + f.batch.cancel(io); + gpa.free(f.batch.storage); + var total_rss: usize = 0; + for (f.instances) |*instance| { + instance.child.kill(io); + instance.message.deinit(gpa); + instance.stderr.deinit(gpa); + instance.progress_node.end(); + total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; + } + f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); + gpa.free(f.instances); + } + + fn startInstances(f: *FuzzTestRunner) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + + for (0.., f.instances) |id, *instance| { + const id32: u32 = @intCast(id); + (switch (f.ctx.fuzz.mode) { + .forever => sendRunFuzzTestMessage( + io, + instance.child.stdin.?, + f.run.fuzz_tests.items, + .forever, + id32, + ), + .limit => |limit| sendRunFuzzTestMessage( + io, + instance.child.stdin.?, + f.run.fuzz_tests.items, + .iterations, + limit.amount, + ), + }) catch |write_err| { + // The runner unexpectedly closed stdin, which means it crashed during initialization. + // Clean up everything and wait for the child to exit. + instance.child.stdin.?.close(io); + instance.child.stdin = null; + const term = try instance.child.wait(io); + return f.run.step.fail( + "unable to write stdin ({t}); test process unexpectedly {f}", + .{ write_err, fmtTerm(term) }, + ); + }; + + try f.addStdoutRead(id32, @sizeOf(InHeader)); + try f.addStderrRead(id32); + } + } + + fn listen(f: *FuzzTestRunner) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + + while (true) { + try f.batch.awaitConcurrent(io, .none); + while (f.batch.next()) |completion| { + const id = completion.index / 3; + const result = completion.result; + switch (completion.index % 3) { + 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) { + // Avoid calling `instanceEos` until EndOfStream is seen with stderr so + // that all stderr is collected. + error.BrokenPipe => continue, + else => |write_e| return write_e, + }), + 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) { + // Avoid calling `instanceEos` until EndOfStream is seen with stderr so + // that all stderr is collected. + error.EndOfStream => continue, + else => |read_e| return read_e, + }), + 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { + error.EndOfStream => return f.instanceEos(id), + else => |read_e| return read_e, + }), + else => unreachable, + } + } + } + } + + fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const io = step_owner.graph.io; + const instance = &f.instances[id]; + + instance.message.items.len += n; + const total_read = instance.message.items.len; + if (total_read < @sizeOf(InHeader)) { + try f.addStdoutRead(id, @sizeOf(InHeader)); + return; + } + + const header = instance.messageHeader(); + const body = instance.message.items[@sizeOf(InHeader)..]; + if (body.len != header.bytes_len) { + try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len); + return; + } + + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + }, + .coverage_id => { + var body_r: Io.Reader = .fixed(body); + f.coverage_id = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable; + const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable; + + const fuzz = f.ctx.fuzz; + fuzz.queue_mutex.lockUncancelable(io); + defer fuzz.queue_mutex.unlock(io); + try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ + .id = f.coverage_id.?, + .cumulative = .{ + .runs = cumulative_runs, + .unique = cumulative_unique, + .coverage = cumulative_coverage, + }, + .run = f.run, + } }); + fuzz.queue_cond.signal(io); + }, + .fuzz_start_addr => { + var body_r: Io.Reader = .fixed(body); + const fuzz = f.ctx.fuzz; + const addr = body_r.takeInt(u64, .little) catch unreachable; + + fuzz.queue_mutex.lockUncancelable(io); + defer fuzz.queue_mutex.unlock(io); + try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ + .addr = addr, + .coverage_id = f.coverage_id.?, + } }); + fuzz.queue_cond.signal(io); + }, + .fuzz_test_change => { + const test_i = std.mem.readInt(u32, body[0..4], .little); + instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); + }, + .broadcast_fuzz_input => { + if (f.instances.len == 1) { + // No other processes to broadcast to. + } else if (f.broadcast_undelivered == 0) { + try f.instanceBroadcast(id, body); + } else { + const footer: PendingBroadcastFooter = .{ + .from_id = id, + .body_len = @intCast(body.len), + }; + // There is another broadcast in progress so add this one to the queue. + const size = @sizeOf(PendingBroadcastFooter) + body.len; + try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); + f.pending_broadcasts.appendSliceAssumeCapacity(body); + f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); + } + }, + else => {}, // ignore other messages + } + + instance.message.clearRetainingCapacity(); + try f.addStdoutRead(id, @sizeOf(InHeader)); + } + + fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void { + const instance = &f.instances[id]; + instance.stderr.items.len += n; + try f.addStderrRead(id); + } + + fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void { + const instance = &f.instances[id]; + + instance.broadcast_written += n; + if (instance.broadcast_written == f.broadcast.items.len) { + f.broadcast_undelivered -= 1; + if (f.broadcast_undelivered == 0) { + try f.broadcastComplete(); + } + } else { + f.addStdinWrite(id); + } + } + + fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const instance = &f.instances[id]; + + try instance.message.ensureTotalCapacity(gpa, end); + const start = instance.message.items.len; + instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]}; + f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{ + .file = instance.child.stdout.?, + .data = &instance.stdout_vec, + } }); + } + + fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + const instance = &f.instances[id]; + + try instance.stderr.ensureUnusedCapacity(gpa, 1); + instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()}; + f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{ + .file = instance.child.stderr.?, + .data = &instance.stderr_vec, + } }); + } + + fn addStdinWrite(f: *FuzzTestRunner, id: u32) void { + const instance = &f.instances[id]; + + assert(f.broadcast.items.len != instance.broadcast_written); + instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]}; + f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{ + .file = instance.child.stdin.?, + .data = &instance.stdin_vec, + } }); + } + + fn instanceEos(f: *FuzzTestRunner, id: u32) !void { + const step_owner = f.run.step.owner; + const io = step_owner.graph.io; + const instance = &f.instances[id]; + + instance.child.stdin.?.close(io); + instance.child.stdin = null; + const term = try instance.child.wait(io); + if (!termMatches(.{ .exited = 0 }, term)) { + f.run.step.result_stderr = try f.mergedStderr(); + try f.saveCrash(id, term); + return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); + } + } + + fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { + const step = &f.run.step; + const b = step.owner; + const io = b.graph.io; + + if (f.coverage_id == null) return; + + // Search for the input file corresponding to the instance + const InputHeader = Build.abi.fuzz.MmapInputHeader; + var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; + var in_r: Io.File.Reader = undefined; + var in_f: Io.File = undefined; + var in_name_buf: [12]u8 = undefined; + var in_name: []const u8 = undefined; + var i: u32 = 0; + const header: InputHeader = while (true) : ({ + if (i == std.math.maxInt(u32)) return; + i += 1; + }) { + const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; + in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; + in_f = b.cache_root.handle.openFile(io, in_name, .{ + .lock = .exclusive, + .lock_nonblocking = true, + }) catch |e| switch (e) { + error.FileNotFound => return, + error.WouldBlock => continue, // Can not be from + // the crashed instance since it is still locked. + else => return step.fail("failed to open file '{f}{s}': {t}", .{ + b.cache_root, in_name, e, + }), + }; + + in_r = in_f.readerStreaming(io, &in_r_buf); + const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { + in_f.close(io); + switch (e) { + error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ + b.cache_root, in_name, in_r.err.?, + }), + error.EndOfStream => continue, + } + }; + + if (header.pc_digest == f.coverage_id.? and + header.instance_id == id and + header.test_i < f.run.fuzz_tests.items.len) + { + break header; + } + + in_f.close(io); + }; + defer in_f.close(io); + + // Save it to a seperate file + const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; + const out = b.cache_root.handle.createFile(io, crash_name, .{ + .lock = .exclusive, // Multiple run steps could have found a crash at the same time + }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ + b.cache_root, crash_name, e, + }); + defer out.close(io); + + var out_w_buf: [512]u8 = undefined; + var out_w = out.writerStreaming(io, &out_w_buf); + _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { + error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ + b.cache_root, in_name, in_r.err.?, + }), + error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ + b.cache_root, crash_name, out_w.err.?, + }), + }; + + return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ + f.run.fuzz_tests.items[header.test_i], + fmtTerm(term), + b.cache_root, + crash_name, + }); + } + + fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void { + assert(f.instances.len > 1); + assert(f.broadcast_undelivered == 0); // no other broadcast is progress + assert(f.broadcast.items.len == 0); + assert(from_id < f.instances.len); + + const step_owner = f.run.step.owner; + const gpa = step_owner.allocator; + + var out_header: OutHeader = .{ + .tag = .new_fuzz_input, + .bytes_len = @intCast(bytes.len), + }; + if (std.builtin.Endian.native != .little) { + std.mem.byteSwapAllFields(OutHeader, &out_header); + } + try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len); + f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header)); + f.broadcast.appendSliceAssumeCapacity(bytes); + + f.broadcast_undelivered = @intCast(f.instances.len - 1); + for (0.., f.instances) |to_id, *instance| { + if (to_id == from_id) continue; + instance.broadcast_written = 0; + f.addStdinWrite(@intCast(to_id)); + } + } + + fn broadcastComplete(f: *FuzzTestRunner) !void { + assert(f.instances.len > 1); + assert(f.broadcast_undelivered == 0); + f.broadcast.clearRetainingCapacity(); + + const pending = &f.pending_broadcasts; + if (pending.items.len != 0) { + // Another broadcast is pending; copy it over to `broadcast` + + const footer_len = @sizeOf(PendingBroadcastFooter); + const footer_bytes = pending.items[pending.items.len - footer_len ..]; + const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes); + pending.items.len -= footer_len; + + const body = pending.items[pending.items.len - footer.body_len ..]; + try f.instanceBroadcast(footer.from_id, body); + pending.items.len -= body.len; + } + } + + fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { + const step_owner = f.run.step.owner; + const arena = step_owner.allocator; + + // Collect any available stderr + while (f.batch.next()) |completion| { + if (completion.index % 3 != 2) continue; + const len = completion.result.file_read_streaming catch continue; + f.instances[completion.index / 3].stderr.items.len += len; + } + + var stderr_len: usize = 0; + for (f.instances) |*instance| stderr_len += instance.stderr.items.len; + const stderr = try arena.alloc(u8, stderr_len); + + stderr_len = 0; + for (f.instances) |*instance| { + @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items); + stderr_len += instance.stderr.items.len; + } + return stderr; + } +}; + +fn evalFuzzTest( + run: *Run, + spawn_options: process.SpawnOptions, + options: Step.MakeOptions, + fuzz_context: FuzzContext, +) !void { + var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); + defer f.deinit(); + try f.startInstances(); + try f.listen(); +} + +const StdioPollEnum = enum { stdout, stderr }; + +fn evalZigTest( + run: *Run, + spawn_options: process.SpawnOptions, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !void { + if (fuzz_context != null) { + try evalFuzzTest(run, spawn_options, options, fuzz_context.?); + return; + } + + const step_owner = run.step.owner; + const gpa = step_owner.allocator; + const arena = step_owner.allocator; + const io = step_owner.graph.io; + + // We will update this every time a child runs. + run.step.result_peak_rss = 0; + + var test_results: Step.TestResults = .{ + .test_count = 0, + .skip_count = 0, + .fail_count = 0, + .crash_count = 0, + .timeout_count = 0, + .leak_count = 0, + .log_err_count = 0, + }; + var test_metadata: ?TestMetadata = null; + + while (true) { + var child = try process.spawn(io, spawn_options); + var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; + var multi_reader: Io.File.MultiReader = undefined; + multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); + var child_killed = false; + defer if (!child_killed) { + child.kill(io); + multi_reader.deinit(); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + }; + + switch (try waitZigTest( + run, + &child, + options, + &multi_reader, + &test_metadata, + &test_results, + )) { + .write_failed => |err| { + // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured + // all available stderr to make our error output as useful as possible. + const stderr_fr = multi_reader.fileReader(1); + while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) { + error.ReadFailed => return stderr_fr.err.?, + error.EndOfStream => {}, + } + run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); + + // Clean up everything and wait for the child to exit. + child.stdin.?.close(io); + child.stdin = null; + multi_reader.deinit(); + child_killed = true; + const term = try child.wait(io); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + + // The individual unit test results are irrelevant: the test runner itself broke! + // Fail immediately without populating `s.test_results`. + return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); + }, + .no_poll => |no_poll| { + // This might be a success (we requested exit and the child dutifully closed stdout) or + // a crash of some kind. Either way, the child will terminate by itself -- wait for it. + const stderr_reader = multi_reader.reader(1); + const stderr_owned = try arena.dupe(u8, stderr_reader.buffered()); + + // Clean up everything and wait for the child to exit. + child.stdin.?.close(io); + child.stdin = null; + multi_reader.deinit(); + child_killed = true; + const term = try child.wait(io); + run.step.result_peak_rss = @max( + run.step.result_peak_rss, + child.resource_usage_statistics.getMaxRss() orelse 0, + ); + + if (no_poll.active_test_index) |test_index| { + // A test was running, so this is definitely a crash. Report it against that + // test, and continue to the next test. + test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; + test_results.crash_count += 1; + try run.step.addError("'{s}' {f}{s}{s}", .{ + test_metadata.?.testName(test_index), + fmtTerm(term), + if (stderr_owned.len != 0) " with stderr:\n" else "", + std.mem.trim(u8, stderr_owned, "\n"), + }); + continue; + } + + // Report an error if the child terminated uncleanly or if we were still trying to run more tests. + run.step.result_stderr = stderr_owned; + const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); + if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { + // The individual unit test results are irrelevant: the test runner itself broke! + // Fail immediately without populating `s.test_results`. + return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); + } + + // We're done with all of the tests! Commit the test results and return. + run.step.test_results = test_results; + if (test_metadata) |tm| { + run.cached_test_metadata = tm.toCachedTestMetadata(); + if (options.web_server) |ws| { + if (run.step.owner.graph.time_report) { + ws.updateTimeReportRunTest( + run, + &run.cached_test_metadata.?, + tm.ns_per_test, + ); + } + } + } + return; + }, + .timeout => |timeout| { + const stderr_reader = multi_reader.reader(1); + const stderr = stderr_reader.buffered(); + stderr_reader.tossBuffered(); + if (timeout.active_test_index) |test_index| { + // A test was running. Report the timeout against that test, and continue on to + // the next test. + test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; + test_results.timeout_count += 1; + try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ + test_metadata.?.testName(test_index), + Io.Duration{ .nanoseconds = timeout.ns_elapsed }, + if (stderr.len != 0) " with stderr:\n" else "", + std.mem.trim(u8, stderr, "\n"), + }); + continue; + } + // Just log an error and let the child be killed. + run.step.result_stderr = try arena.dupe(u8, stderr); + // The individual unit test results in `results` are irrelevant: the test runner + // is broken! Fail immediately without populating `s.test_results`. + return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); + }, + } + comptime unreachable; + } +} + +const TestMetadata = struct { + names: []const u32, + ns_per_test: []u64, + expected_panic_msgs: []const u32, + string_bytes: []const u8, + next_index: u32, + prog_node: std.Progress.Node, + + fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { + return .{ + .names = tm.names, + .string_bytes = tm.string_bytes, + }; + } + + fn testName(tm: TestMetadata, index: u32) []const u8 { + return tm.toCachedTestMetadata().testName(index); + } +}; + +pub const CachedTestMetadata = struct { + names: []const u32, + string_bytes: []const u8, + + pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { + return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); + } +}; + +fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { + while (metadata.next_index < metadata.names.len) { + const i = metadata.next_index; + metadata.next_index += 1; + + if (metadata.expected_panic_msgs[i] != 0) continue; + + const name = metadata.testName(i); + if (sub_prog_node.*) |n| n.end(); + sub_prog_node.* = metadata.prog_node.start(name, 0); + + try sendRunTestMessage(io, in, .run_test, i); + return; + } else { + metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done + try sendMessage(io, in, .exit); + } +} + +fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 4, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u32, index, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; +} + +fn sendRunFuzzTestMessage( + io: Io, + file: Io.File, + test_names: []const []const u8, + kind: std.Build.abi.fuzz.LimitKind, + amount_or_instance: u64, +) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = .start_fuzzing, + .bytes_len = 1 + 8 + 4 + count: { + var c: u32 = @intCast(test_names.len * 4); + for (test_names) |name| { + c += @intCast(name.len); + } + break :count c; + }, + }; + var w = file.writerStreaming(io, &.{}); + w.interface.writeStruct(header, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + for (test_names) |test_name| { + w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeAll(test_name) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + } +} + +fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { + const b = run.step.owner; + const io = b.graph.io; + const arena = b.allocator; + const gpa = b.allocator; + + var child = try process.spawn(io, spawn_options); + defer child.kill(io); + + switch (run.stdin) { + .bytes => |bytes| { + child.stdin.?.writeStreamingAll(io, bytes) catch |err| { + return run.step.fail("unable to write stdin: {t}", .{err}); + }; + child.stdin.?.close(io); + child.stdin = null; + }, + .lazy_path => |lazy_path| { + const path = lazy_path.getPath3(b, &run.step); + const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { + return run.step.fail("unable to open stdin file: {t}", .{err}); + }; + defer file.close(io); + // TODO https://github.com/ziglang/zig/issues/23955 + var read_buffer: [1024]u8 = undefined; + var file_reader = file.reader(io, &read_buffer); + var write_buffer: [1024]u8 = undefined; + var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); + _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { + error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ + path, file_reader.err.?, + }), + error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ + stdin_writer.err.?, + }), + }; + stdin_writer.interface.flush() catch |err| switch (err) { + error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ + stdin_writer.err.?, + }), + }; + child.stdin.?.close(io); + child.stdin = null; + }, + .none => {}, + } + + var stdout_bytes: ?[]const u8 = null; + var stderr_bytes: ?[]const u8 = null; + + if (child.stdout) |stdout| { + if (child.stderr) |stderr| { + var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; + var multi_reader: Io.File.MultiReader = undefined; + multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); + defer multi_reader.deinit(); + + const stdout_reader = multi_reader.reader(0); + const stderr_reader = multi_reader.reader(1); + + while (multi_reader.fill(64, .none)) |_| { + if (run.stdio_limit.toInt()) |limit| { + if (stdout_reader.buffered().len > limit) + return error.StdoutStreamTooLong; + if (stderr_reader.buffered().len > limit) + return error.StderrStreamTooLong; + } + } else |err| switch (err) { + error.Timeout => unreachable, + error.EndOfStream => {}, + else => |e| return e, + } + + try multi_reader.checkAnyError(); + + // TODO: this string can leak since alloc below can return error. + stdout_bytes = try multi_reader.toOwnedSlice(0); + // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. + stderr_bytes = try multi_reader.toOwnedSlice(1); + } else { + var stdout_reader = stdout.readerStreaming(io, &.{}); + stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return stdout_reader.err.?, + error.StreamTooLong => return error.StdoutStreamTooLong, + }; + } + } else if (child.stderr) |stderr| { + var stderr_reader = stderr.readerStreaming(io, &.{}); + stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return stderr_reader.err.?, + error.StreamTooLong => return error.StderrStreamTooLong, + }; + } + + if (stderr_bytes) |bytes| if (bytes.len > 0) { + // Treat stderr as an error message. + const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { + .check => |checks| !checksContainStderr(checks.items), + else => true, + }; + if (stderr_is_diagnostic) { + run.step.result_stderr = bytes; + } + }; + + run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; + + return .{ + .term = try child.wait(io), + .stdout = stdout_bytes, + .stderr = stderr_bytes, + }; +} + +const IndexedOutput = struct { + index: usize, + tag: @typeInfo(Arg).@"union".tag_type.?, + output: *Output, +}; + +pub fn rerunInFuzzMode( + run: *Run, + fuzz: *std.Build.Fuzz, + prog_node: std.Progress.Node, +) !void { + const step = &run.step; + const b = step.owner; + const io = b.graph.io; + const arena = b.allocator; + var argv_list: std.ArrayList([]const u8) = .empty; + for (run.argv.items) |arg| { + switch (arg) { + .bytes => |bytes| { + try argv_list.append(arena, bytes); + }, + .lazy_path => |file| { + const file_path = file.lazy_path.getPath3(b, step); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); + }, + .decorated_directory => |dd| { + const file_path = dd.lazy_path.getPath3(b, step); + try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); + }, + .file_content => |file_plp| { + const file_path = file_plp.lazy_path.getPath3(b, step); + + var result: std.Io.Writer.Allocating = .init(arena); + errdefer result.deinit(); + result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + + const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); + defer file.close(io); + + var buf: [1024]u8 = undefined; + var file_reader = file.reader(io, &buf); + _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { + error.ReadFailed => return file_reader.err.?, + error.WriteFailed => return error.OutOfMemory, + }; + + try argv_list.append(arena, result.written()); + }, + .artifact => |pa| { + const artifact = pa.artifact; + const file_path: []const u8 = p: { + if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); + break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; + }; + try argv_list.append(arena, b.fmt("{s}{s}", .{ + pa.prefix, + run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), + })); + }, + .output_file, .output_directory => unreachable, + } + } + + if (run.step.result_failed_command) |cmd| { + fuzz.gpa.free(cmd); + run.step.result_failed_command = null; + } + + const has_side_effects = false; + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ + .progress_node = prog_node, + .watch = undefined, // not used by `runCommand` + .web_server = null, // only needed for time reports + .unit_test_timeout_ns = null, // don't time out fuzz tests for now + .gpa = fuzz.gpa, + }, .{ + .fuzz = fuzz, + }); +} + +fn populateGeneratedPaths( + arena: std.mem.Allocator, + output_placeholders: []const IndexedOutput, + captured_stdout: ?*CapturedStdIo, + captured_stderr: ?*CapturedStdIo, + cache_root: Cache.Directory, + digest: *const Cache.HexDigest, +) !void { + for (output_placeholders) |placeholder| { + placeholder.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, placeholder.output.basename, + }); + } + + if (captured_stdout) |captured| { + captured.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, captured.output.basename, + }); + } + + if (captured_stderr) |captured| { + captured.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, captured.output.basename, + }); + } +} + +fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void { + if (term) |t| switch (t) { + .exited => |code| try w.print("exited with code {d}", .{code}), + .signal => |sig| try w.print("terminated with signal {t}", .{sig}), + .stopped => |sig| try w.print("stopped with signal {t}", .{sig}), + .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}), + } else { + try w.writeAll("exited with any code"); + } +} +fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) { + return .{ .data = term }; +} + +const FuzzContext = struct { + fuzz: *std.Build.Fuzz, +}; + +fn runCommand( + run: *Run, + argv: []const []const u8, + has_side_effects: bool, + output_dir_path: []const u8, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !void { + const step = &run.step; + const b = step.owner; + const arena = b.allocator; + const gpa = options.gpa; + const io = b.graph.io; + + const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; + + try step.handleChildProcUnsupported(); + try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); + + const allow_skip = switch (run.stdio) { + .check, .zig_test => run.skip_foreign_checks, + else => false, + }; + + var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); + defer interp_argv.deinit(); + + var environ_map: EnvMap = env: { + const orig = run.environ_map orelse &b.graph.environ_map; + break :env try orig.clone(gpa); + }; + defer environ_map.deinit(); + + const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { + // InvalidExe: cpu arch mismatch + // FileNotFound: can happen with a wrong dynamic linker path + if (err == error.InvalidExe or err == error.FileNotFound) interpret: { + // TODO: learn the target from the binary directly rather than from + // relying on it being a Compile step. This will make this logic + // work even for the edge case that the binary was produced by a + // third party. + const exe = switch (run.argv.items[0]) { + .artifact => |exe| exe.artifact, + else => break :interpret, + }; + switch (exe.kind) { + .exe, .@"test" => {}, + else => break :interpret, + } + + const root_target = exe.rootModuleTarget(); + const need_cross_libc = exe.is_linking_libc and + (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); + const other_target = exe.root_module.resolved_target.?.result; + switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ + .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, + .link_libc = exe.is_linking_libc, + })) { + .native, .rosetta => { + if (allow_skip) return error.MakeSkipped; + break :interpret; + }, + .wine => |bin_name| { + if (b.enable_wine) { + try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); + + // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but + // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. + if (environ_map.get("WINEDEBUG") == null) { + try environ_map.put("WINEDEBUG", "-all"); + } + } else { + return failForeign(run, "-fwine", argv[0], exe); + } + }, + .qemu => |bin_name| { + if (b.enable_qemu) { + try interp_argv.append(bin_name); + + if (need_cross_libc) { + if (b.libc_runtimes_dir) |dir| { + try interp_argv.append("-L"); + try interp_argv.append(b.pathJoin(&.{ + dir, + try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( + b.allocator, + root_target.cpu.arch, + root_target.os.tag, + root_target.abi, + ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( + b.allocator, + root_target.cpu.arch, + root_target.abi, + ) else unreachable, + })); + } else return failForeign(run, "--libc-runtimes", argv[0], exe); + } + + try interp_argv.appendSlice(argv); + } else return failForeign(run, "-fqemu", argv[0], exe); + }, + .darling => |bin_name| { + if (b.enable_darling) { + try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); + } else { + return failForeign(run, "-fdarling", argv[0], exe); + } + }, + .wasmtime => |bin_name| { + if (b.enable_wasmtime) { + try interp_argv.append(bin_name); + try interp_argv.append("--dir=."); + // Wasmtime doeesn't inherit environment variables from the parent process + // by default. '-S inherit-env' was added in Wasmtime version 20. + try interp_argv.append("-Sinherit-env"); + try interp_argv.append(argv[0]); + try interp_argv.appendSlice(argv[1..]); + } else { + return failForeign(run, "-fwasmtime", argv[0], exe); + } + }, + .bad_dl => |foreign_dl| { + if (allow_skip) return error.MakeSkipped; + + const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; + + return step.fail( + \\the host system is unable to execute binaries from the target + \\ because the host dynamic linker is '{s}', + \\ while the target dynamic linker is '{s}'. + \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step + , .{ host_dl, foreign_dl }); + }, + .bad_os_or_cpu => { + if (allow_skip) return error.MakeSkipped; + + const host_name = try b.graph.host.result.zigTriple(b.allocator); + const foreign_name = try root_target.zigTriple(b.allocator); + + return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ + host_name, foreign_name, + }); + }, + } + + if (root_target.os.tag == .windows) { + // On Windows we don't have rpaths so we have to add .dll search paths to PATH + run.addPathForDynLibs(exe); + } + + gpa.free(step.result_failed_command.?); + step.result_failed_command = null; + try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); + + break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { + if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; + if (e == error.MakeFailed) return error.MakeFailed; // error already reported + return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); + }; + } + if (err == error.MakeFailed) return error.MakeFailed; // error already reported + + return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); + }; + + const generic_result = opt_generic_result orelse { + assert(run.stdio == .zig_test); + // Specific errors have already been reported, and test results are populated. All we need + // to do is report step failure if any test failed. + if (!step.test_results.isSuccess()) return error.MakeFailed; + return; + }; + + assert(fuzz_context == null); + assert(run.stdio != .zig_test); + + // Capture stdout and stderr to GeneratedFile objects. + const Stream = struct { + captured: ?*CapturedStdIo, + bytes: ?[]const u8, + }; + for ([_]Stream{ + .{ + .captured = run.captured_stdout, + .bytes = generic_result.stdout, + }, + .{ + .captured = run.captured_stderr, + .bytes = generic_result.stderr, + }, + }) |stream| { + if (stream.captured) |captured| { + const output_components = .{ output_dir_path, captured.output.basename }; + const output_path = try b.cache_root.join(arena, &output_components); + captured.output.generated_file.path = output_path; + + const sub_path = b.pathJoin(&output_components); + const sub_path_dirname = Dir.path.dirname(sub_path).?; + b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { + return step.fail("unable to make path '{f}{s}': {s}", .{ + b.cache_root, sub_path_dirname, @errorName(err), + }); + }; + const data = switch (captured.trim_whitespace) { + .none => stream.bytes.?, + .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), + .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), + .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), + }; + b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { + return step.fail("unable to write file '{f}{s}': {s}", .{ + b.cache_root, sub_path, @errorName(err), + }); + }; + } + } + + switch (run.stdio) { + .zig_test => unreachable, + .check => |checks| for (checks.items) |check| switch (check) { + .expect_stderr_exact => |expected_bytes| { + if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { + return step.fail( + \\========= expected this stderr: ========= + \\{s} + \\========= but found: ==================== + \\{s} + , .{ + expected_bytes, + generic_result.stderr.?, + }); + } + }, + .expect_stderr_match => |match| { + if (mem.find(u8, generic_result.stderr.?, match) == null) { + return step.fail( + \\========= expected to find in stderr: ========= + \\{s} + \\========= but stderr does not contain it: ===== + \\{s} + , .{ + match, + generic_result.stderr.?, + }); + } + }, + .expect_stdout_exact => |expected_bytes| { + if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { + return step.fail( + \\========= expected this stdout: ========= + \\{s} + \\========= but found: ==================== + \\{s} + , .{ + expected_bytes, + generic_result.stdout.?, + }); + } + }, + .expect_stdout_match => |match| { + if (mem.find(u8, generic_result.stdout.?, match) == null) { + return step.fail( + \\========= expected to find in stdout: ========= + \\{s} + \\========= but stdout does not contain it: ===== + \\{s} + , .{ + match, + generic_result.stdout.?, + }); + } + }, + .expect_term => |expected_term| { + if (!termMatches(expected_term, generic_result.term)) { + return step.fail("process {f} (expected {f})", .{ + fmtTerm(generic_result.term), + fmtTerm(expected_term), + }); + } + }, + }, + else => { + // On failure, report captured stderr like normal standard error output. + const bad_exit = switch (generic_result.term) { + .exited => |code| code != 0, + .signal, .stopped, .unknown => true, + }; + if (bad_exit) { + if (generic_result.stderr) |bytes| { + run.step.result_stderr = bytes; + } + } + + try step.handleChildProcessTerm(generic_result.term); + }, + } +} + +const EvalGenericResult = struct { + term: process.Child.Term, + stdout: ?[]const u8, + stderr: ?[]const u8, +}; + +fn spawnChildAndCollect( + run: *Run, + argv: []const []const u8, + environ_map: *EnvMap, + has_side_effects: bool, + options: Step.MakeOptions, + fuzz_context: ?FuzzContext, +) !?EvalGenericResult { + const b = run.step.owner; + const graph = b.graph; + const io = graph.io; + + if (fuzz_context != null) { + assert(!has_side_effects); + assert(run.stdio == .zig_test); + } + + const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; + + // If an error occurs, it's caused by this command: + assert(run.step.result_failed_command == null); + run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ + .child = environ_map, + .parent = &graph.environ_map, + }, argv); + + var spawn_options: process.SpawnOptions = .{ + .argv = argv, + .cwd = child_cwd, + .environ_map = environ_map, + .request_resource_usage_statistics = true, + .stdin = if (run.stdin != .none) s: { + assert(run.stdio != .inherit); + break :s .pipe; + } else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .ignore, + .inherit => .inherit, + .check => .ignore, + .zig_test => .pipe, + }, + .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .ignore, + .inherit => .inherit, + .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, + .zig_test => .pipe, + }, + .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { + .infer_from_args => if (has_side_effects) .inherit else .pipe, + .inherit => .inherit, + .check => .pipe, + .zig_test => .pipe, + }, + }; + + if (run.stdio == .zig_test) { + const started: Io.Clock.Timestamp = .now(io, .awake); + const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| e, + }; + run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + try result; + return null; + } else { + const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; + if (!run.disable_zig_progress and !inherit) { + spawn_options.progress_node = options.progress_node; + } + const terminal_mode: Io.Terminal.Mode = if (inherit) m: { + const stderr = try io.lockStderr(&.{}, graph.stderr_mode); + break :m stderr.terminal_mode; + } else .no_color; + defer if (inherit) io.unlockStderr(); + try setColorEnvironmentVariables(run, environ_map, terminal_mode); + + const started: Io.Clock.Timestamp = .now(io, .awake); + const result = evalGeneric(run, spawn_options) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| e, + }; + run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + return try result; + } +} + +fn hashStdIo(hh: *Cache.HashHelper, stdio: StdIo) void { + switch (stdio) { + .infer_from_args, .inherit, .zig_test => {}, + .check => |checks| for (checks.items) |check| { + hh.add(@as(std.meta.Tag(StdIo.Check), check)); + switch (check) { + .expect_stderr_exact, + .expect_stderr_match, + .expect_stdout_exact, + .expect_stdout_match, + => |s| hh.addBytes(s), + + .expect_term => |term| { + hh.add(@as(std.meta.Tag(process.Child.Term), term)); + switch (term) { + inline .exited, .signal, .stopped => |x| hh.add(x), + .unknown => |x| hh.add(x), + } + }, + } + }, + } +} +fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { + return if (expected) |e| switch (e) { + .exited => |expected_code| switch (actual) { + .exited => |actual_code| expected_code == actual_code, + else => false, + }, + .signal => |expected_sig| switch (actual) { + .signal => |actual_sig| expected_sig == actual_sig, + else => false, + }, + .stopped => |expected_sig| switch (actual) { + .stopped => |actual_sig| expected_sig == actual_sig, + else => false, + }, + .unknown => |expected_code| switch (actual) { + .unknown => |actual_code| expected_code == actual_code, + else => false, + }, + } else switch (actual) { + .exited => true, + else => false, + }; +} + +fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { + color: switch (run.color) { + .manual => {}, + .enable => { + try environ_map.put("CLICOLOR_FORCE", "1"); + _ = environ_map.swapRemove("NO_COLOR"); + }, + .disable => { + try environ_map.put("NO_COLOR", "1"); + _ = environ_map.swapRemove("CLICOLOR_FORCE"); + }, + .inherit => switch (terminal_mode) { + .no_color, .windows_api => continue :color .disable, + .escape_codes => continue :color .enable, + }, + .auto => { + const capture_stderr = run.captured_stderr != null or switch (run.stdio) { + .check => |checks| checksContainStderr(checks.items), + .infer_from_args, .inherit, .zig_test => false, + }; + if (capture_stderr) { + continue :color .disable; + } else { + continue :color .inherit; + } + }, + } +} + +fn checksContainStdout(checks: []const StdIo.Check) bool { + for (checks) |check| switch (check) { + .expect_stderr_exact, + .expect_stderr_match, + .expect_term, + => continue, + + .expect_stdout_exact, + .expect_stdout_match, + => return true, + }; + return false; +} + +fn checksContainStderr(checks: []const StdIo.Check) bool { + for (checks) |check| switch (check) { + .expect_stdout_exact, + .expect_stdout_match, + .expect_term, + => continue, + + .expect_stderr_exact, + .expect_stderr_match, + => return true, + }; + return false; +} + +/// Returns whether the Run step has side effects *other than* updating the output arguments. +fn hasSideEffects(run: Run) bool { + if (run.has_side_effects) return true; + return switch (run.stdio) { + .infer_from_args => !run.hasAnyOutputArgs(), + .inherit => true, + .check => false, + .zig_test => false, + }; +} + +fn hasAnyOutputArgs(run: Run) bool { + if (run.captured_stdout != null) return true; + if (run.captured_stderr != null) return true; + for (run.argv.items) |arg| switch (arg) { + .output_file, .output_directory => return true, + else => continue, + }; + return false; +} + +/// If `path` is cwd-relative, make it relative to the cwd of the child instead. +/// +/// Whenever a path is included in the argv of a child, it should be put through this function first +/// to make sure the child doesn't see paths relative to a cwd other than its own. +fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { + const b = run.step.owner; + const graph = b.graph; + const arena = graph.arena; + + const path_str = path.toString(arena) catch @panic("OOM"); + if (Dir.path.isAbsolute(path_str)) { + // Absolute paths don't need changing. + return path_str; + } + const child_cwd_rel: []const u8 = rel: { + const child_lazy_cwd = run.cwd orelse break :rel path_str; + const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); + // Convert it from relative to *our* cwd, to relative to the *child's* cwd. + break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); + }; + // Not every path can be made relative, e.g. if the path and the child cwd are on different + // disk designators on Windows. In that case, `relative` will return an absolute path which we can + // just return. + if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel; + + // We're not done yet. In some cases this path must be prefixed with './': + // * On POSIX, the executable name cannot be a single component like 'foo' + // * Some executables might treat a leading '-' like a flag, which we must avoid + // There's no harm in it, so just *always* apply this prefix. + return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); +} + +fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { + const b = run.step.owner; + const compiles = artifact.getCompileDependencies(true); + for (compiles) |compile| { + if (compile.root_module.resolved_target.?.result.os.tag == .windows and + compile.isDynamicLibrary()) + { + addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); + } + } +} + +fn failForeign( + run: *Run, + suggested_flag: []const u8, + argv0: []const u8, + exe: *Step.Compile, +) error{ MakeFailed, MakeSkipped, OutOfMemory } { + switch (run.stdio) { + .check, .zig_test => { + if (run.skip_foreign_checks) + return error.MakeSkipped; + + const b = run.step.owner; + const host_name = try b.graph.host.result.zigTriple(b.allocator); + const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); + + return run.step.fail( + \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) + \\ consider using {s} or enabling skip_foreign_checks in the Run step + , .{ argv0, foreign_name, host_name, suggested_flag }); + }, + else => { + return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); + }, + } +} diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig new file mode 100644 index 0000000000000000000000000000000000000000..d594f8983fe5ead949a5efdd0ba106c176777d34 --- /dev/null +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -0,0 +1,206 @@ + +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; + const b = step.owner; + const graph = b.graph; + const io = graph.io; + const arena = b.allocator; + const gpa = graph.cache.gpa; + const write_file: *WriteFile = @fieldParentPtr("step", step); + + const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); + var open_dirs_count: usize = 0; + defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); + + switch (write_file.mode) { + .whole_cached => { + step.clearWatchInputs(); + + // The cache is used here not really as a way to speed things up - because writing + // the data to a file would probably be very fast - but as a way to find a canonical + // location to put build artifacts. + + // If, for example, a hard-coded path was used as the location to put WriteFile + // files, then two WriteFiles executing in parallel might clobber each other. + + var man = b.graph.cache.obtain(); + defer man.deinit(); + + for (write_file.files.items) |file| { + man.hash.addBytes(file.sub_path); + + switch (file.contents) { + .bytes => |bytes| { + man.hash.addBytes(bytes); + }, + .copy => |lazy_path| { + const path = lazy_path.getPath3(b, step); + _ = try man.addFilePath(path, null); + try step.addWatchInput(lazy_path); + }, + } + } + + for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { + man.hash.addBytes(dir.sub_path); + for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); + if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); + + const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); + const src_dir_path = dir.source.getPath3(b, step); + + var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{f}': {s}", .{ + src_dir_path, @errorName(err), + }); + }; + open_dir_cache_elem.* = src_dir; + open_dirs_count += 1; + + var it = try src_dir.walk(gpa); + defer it.deinit(); + while (try it.next(io)) |entry| { + if (!dir.options.pathIncluded(entry.path)) continue; + + switch (entry.kind) { + .directory => { + if (need_derived_inputs) { + const entry_path = try src_dir_path.join(arena, entry.path); + try step.addDirectoryWatchInputFromPath(entry_path); + } + }, + .file => { + const entry_path = try src_dir_path.join(arena, entry.path); + _ = try man.addFilePath(entry_path, null); + }, + else => continue, + } + } + } + + if (try step.cacheHit(&man)) { + const digest = man.final(); + write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); + assert(step.result_cached); + return; + } + + const digest = man.final(); + const cache_path = "o" ++ Dir.path.sep_str ++ digest; + + write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); + + try operate(write_file, open_dir_cache, .{ + .root_dir = b.cache_root, + .sub_path = cache_path, + }); + + try step.writeManifest(&man); + }, + .tmp => { + step.result_cached = false; + + var rand_int: u64 = undefined; + io.random(@ptrCast(&rand_int)); + const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + + write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); + + try operate(write_file, open_dir_cache, .{ + .root_dir = b.cache_root, + .sub_path = tmp_dir_sub_path, + }); + }, + .mutate => |lp| { + step.result_cached = false; + const root_path = try lp.getPath4(b, step); + write_file.generated_directory.path = try root_path.toString(arena); + try operate(write_file, open_dir_cache, root_path); + }, + } +} + +fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { + const step = &write_file.step; + const b = step.owner; + const io = b.graph.io; + const gpa = b.graph.cache.gpa; + const arena = b.allocator; + + var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| + return step.fail("unable to make path {f}: {t}", .{ root_path, err }); + defer cache_dir.close(io); + + for (write_file.files.items) |file| { + if (Dir.path.dirname(file.sub_path)) |dirname| { + cache_dir.createDirPath(io, dirname) catch |err| { + return step.fail("unable to make path '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, dirname, err, + }); + }; + } + switch (file.contents) { + .bytes => |bytes| { + cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { + return step.fail("unable to write file '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, file.sub_path, err, + }); + }; + }, + .copy => |file_source| { + const source_path = file_source.getPath2(b, step); + const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { + return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ + source_path, root_path, Dir.path.sep, file.sub_path, err, + }); + }; + // At this point we already will mark the step as a cache miss. + // But this is kind of a partial cache hit since individual + // file copies may be avoided. Oh well, this information is + // discarded. + _ = prev_status; + }, + } + } + + for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { + const src_dir_path = dir.source.getPath3(b, step); + const dest_dirname = dir.sub_path; + + if (dest_dirname.len != 0) { + cache_dir.createDirPath(io, dest_dirname) catch |err| { + return step.fail("unable to make path '{f}{c}{s}': {t}", .{ + root_path, Dir.path.sep, dest_dirname, err, + }); + }; + } + + var it = try already_open_dir.walk(gpa); + defer it.deinit(); + while (try it.next(io)) |entry| { + if (!dir.options.pathIncluded(entry.path)) continue; + + const src_entry_path = try src_dir_path.join(arena, entry.path); + const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); + switch (entry.kind) { + .directory => try cache_dir.createDirPath(io, dest_path), + .file => { + const prev_status = Io.Dir.updateFile( + src_entry_path.root_dir.handle, + io, + src_entry_path.sub_path, + cache_dir, + dest_path, + .{}, + ) catch |err| { + return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ + src_entry_path, root_path, Dir.path.sep, dest_path, err, + }); + }; + _ = prev_status; + }, + else => continue, + } + } + } +} diff --git a/lib/compiler/Maker/Watch.zig b/lib/compiler/Maker/Watch.zig new file mode 100644 index 0000000000000000000000000000000000000000..907e6536a132863603d70e423041b41c5b677a5c --- /dev/null +++ b/lib/compiler/Maker/Watch.zig @@ -0,0 +1,976 @@ +const Watch = @This(); +const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const fatal = std.process.fatal; +const Configuration = std.Build.Configuration; + +const FsEvents = @import("Watch/FsEvents.zig"); +const Step = @import("Step.zig"); + +os: Os, +/// The number to show as the number of directories being watched. +dir_count: usize, +// These fields are common to most implementations so are kept here for simplicity. +// They are `undefined` on implementations which do not utilize then. +dir_table: DirTable, +generation: Generation, +configuration: *const Configuration, +make_steps: []Step, + +pub const have_impl = Os != void; + +/// Key is the directory to watch which contains one or more files we are +/// interested in noticing changes to. +/// +/// Value is generation. +const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false); + +/// Special key of "." means any changes in this directory trigger the steps. +const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); +const StepSet = std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, Generation); + +const Generation = u8; + +const Hash = std.hash.Wyhash; +const Cache = std.Build.Cache; + +const Os = switch (builtin.os.tag) { + .linux => struct { + const posix = std.posix; + + /// Keyed differently but indexes correspond 1:1 with `dir_table`. + handle_table: HandleTable, + /// fanotify file descriptors are keyed by mount id since marks + /// are limited to a single filesystem. + poll_fds: std.AutoArrayHashMapUnmanaged(MountId, posix.pollfd), + + const MountId = i32; + const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, struct { mount_id: MountId, reaction_set: ReactionSet }, FileHandle.Adapter, false); + + const fan_mask: std.os.linux.fanotify.MarkMask = .{ + .CLOSE_WRITE = true, + .CREATE = true, + .DELETE = true, + .DELETE_SELF = true, + .EVENT_ON_CHILD = true, + .MOVED_FROM = true, + .MOVED_TO = true, + .MOVE_SELF = true, + .ONDIR = true, + }; + + const FileHandle = struct { + handle: *align(1) std.os.linux.file_handle, + + fn clone(lfh: FileHandle, gpa: Allocator) Allocator.Error!FileHandle { + const bytes = lfh.slice(); + const new_ptr = try gpa.alignedAlloc( + u8, + .of(std.os.linux.file_handle), + @sizeOf(std.os.linux.file_handle) + bytes.len, + ); + const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); + new_header.* = lfh.handle.*; + const new: FileHandle = .{ .handle = new_header }; + @memcpy(new.slice(), lfh.slice()); + return new; + } + + fn destroy(lfh: FileHandle, gpa: Allocator) void { + const ptr: [*]u8 = @ptrCast(lfh.handle); + const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; + return gpa.free(allocated_slice); + } + + fn slice(lfh: FileHandle) []u8 { + const ptr: [*]u8 = &lfh.handle.f_handle; + return ptr[0..lfh.handle.handle_bytes]; + } + + const Adapter = struct { + pub fn hash(self: Adapter, a: FileHandle) u32 { + _ = self; + const unsigned_type: u32 = @bitCast(a.handle.handle_type); + return @truncate(Hash.hash(unsigned_type, a.slice())); + } + pub fn eql(self: Adapter, a: FileHandle, b: FileHandle, b_index: usize) bool { + _ = self; + _ = b_index; + return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice()); + } + }; + }; + + fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { + _ = cwd_path; + return .{ + .dir_table = .{}, + .dir_count = 0, + .os = switch (builtin.os.tag) { + .linux => .{ + .handle_table = .{}, + .poll_fds = .{}, + }, + else => {}, + }, + .generation = 0, + .make_steps = make_steps, + .configuration = configuration, + }; + } + + fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path, mount_id: *MountId) !FileHandle { + var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; + var buf: [std.fs.max_path_bytes]u8 = undefined; + const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{ + path.sub_path, + }) catch return error.NameTooLong; + const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); + stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); + try posix.name_to_handle_at(path.root_dir.handle.handle, adjusted_path, stack_ptr, mount_id, std.os.linux.AT.HANDLE_FID); + const stack_lfh: FileHandle = .{ .handle = stack_ptr }; + return stack_lfh.clone(gpa); + } + + fn markDirtySteps(w: *Watch, gpa: Allocator, fan_fd: posix.fd_t) !bool { + const fanotify = std.os.linux.fanotify; + const M = fanotify.event_metadata; + var events_buf: [256 + 4096]u8 = undefined; + var any_dirty = false; + while (true) { + var len = posix.read(fan_fd, &events_buf) catch |err| switch (err) { + error.WouldBlock => return any_dirty, + else => |e| return e, + }; + var meta: [*]align(1) M = @ptrCast(&events_buf); + while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ + len -= meta[0].event_len; + meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); + }) { + assert(meta[0].vers == M.VERSION); + if (meta[0].mask.Q_OVERFLOW) { + any_dirty = true; + std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); + markAllFilesDirty(w, gpa); + return true; + } + const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); + switch (fid.hdr.info_type) { + .DFID_NAME => { + const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); + const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); + const file_name = std.mem.span(file_name_z); + const lfh: FileHandle = .{ .handle = file_handle }; + if (w.os.handle_table.getPtr(lfh)) |value| { + if (value.reaction_set.getPtr(".")) |glob_set| + any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty); + if (value.reaction_set.getPtr(file_name)) |step_set| + any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty); + } + }, + else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}), + } + } + } + } + + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + // Add missing marks and note persisted ones. + for (steps) |step_index| { + const step = &w.make_steps[@intFromEnum(step_index)]; + for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { + const reaction_set = rs: { + const gop = try w.dir_table.getOrPut(gpa, path); + if (!gop.found_existing) { + var mount_id: MountId = undefined; + const dir_handle = getDirHandle(gpa, path, &mount_id) catch |err| switch (err) { + error.FileNotFound => { + std.debug.assert(w.dir_table.swapRemove(path)); + continue; + }, + else => return err, + }; + const fan_fd = blk: { + const fd_gop = try w.os.poll_fds.getOrPut(gpa, mount_id); + if (!fd_gop.found_existing) { + const fan_fd = std.posix.fanotify_init(.{ + .CLASS = .NOTIF, + .CLOEXEC = true, + .NONBLOCK = true, + .REPORT_NAME = true, + .REPORT_DIR_FID = true, + .REPORT_FID = true, + .REPORT_TARGET_FID = true, + }, 0) catch |err| switch (err) { + error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}), + else => |e| return e, + }; + fd_gop.value_ptr.* = .{ + .fd = fan_fd, + .events = std.posix.POLL.IN, + .revents = undefined, + }; + } + break :blk fd_gop.value_ptr.*.fd; + }; + // `dir_handle` may already be present in the table in + // the case that we have multiple Cache.Path instances + // that compare inequal but ultimately point to the same + // directory on the file system. + // In such case, we must revert adding this directory, but keep + // the additions to the step set. + const dh_gop = try w.os.handle_table.getOrPut(gpa, dir_handle); + if (dh_gop.found_existing) { + _ = w.dir_table.pop(); + } else { + assert(dh_gop.index == gop.index); + dh_gop.value_ptr.* = .{ .mount_id = mount_id, .reaction_set = .{} }; + posix.fanotify_mark(fan_fd, .{ + .ADD = true, + .ONLYDIR = true, + }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| { + fatal("unable to watch {f}: {s}", .{ path, @errorName(err) }); + }; + } + break :rs &dh_gop.value_ptr.reaction_set; + } + break :rs &w.os.handle_table.values()[gop.index].reaction_set; + }; + for (files.items) |basename| { + const gop = try reaction_set.getOrPut(gpa, basename); + if (!gop.found_existing) gop.value_ptr.* = .{}; + try gop.value_ptr.put(gpa, step_index, w.generation); + } + } + } + + { + // Remove marks for files that are no longer inputs. + var i: usize = 0; + while (i < w.os.handle_table.entries.len) { + { + const reaction_set = &w.os.handle_table.values()[i].reaction_set; + var step_set_i: usize = 0; + while (step_set_i < reaction_set.entries.len) { + const step_set = &reaction_set.values()[step_set_i]; + var dirent_i: usize = 0; + while (dirent_i < step_set.entries.len) { + const generations = step_set.values(); + if (generations[dirent_i] == w.generation) { + dirent_i += 1; + continue; + } + step_set.swapRemoveAt(dirent_i); + } + if (step_set.entries.len > 0) { + step_set_i += 1; + continue; + } + reaction_set.swapRemoveAt(step_set_i); + } + if (reaction_set.entries.len > 0) { + i += 1; + continue; + } + } + + const path = w.dir_table.keys()[i]; + + const mount_id = w.os.handle_table.values()[i].mount_id; + const fan_fd = w.os.poll_fds.getEntry(mount_id).?.value_ptr.fd; + posix.fanotify_mark(fan_fd, .{ + .REMOVE = true, + .ONLYDIR = true, + }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| switch (err) { + error.FileNotFound => {}, // Expected, harmless. + else => |e| std.log.warn("unable to unwatch '{f}': {s}", .{ path, @errorName(e) }), + }; + + w.dir_table.swapRemoveAt(i); + w.os.handle_table.swapRemoveAt(i); + } + w.generation +%= 1; + } + w.dir_count = w.dir_table.count(); + } + + fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { + _ = io; + const events_len = try std.posix.poll(w.os.poll_fds.values(), timeout.to_i32_ms()); + if (events_len == 0) + return .timeout; + for (w.os.poll_fds.values()) |poll_fd| { + if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try markDirtySteps(w, gpa, poll_fd.fd)) + return .dirty; + } + return .clean; + } + }, + .windows => struct { + const windows = std.os.windows; + + /// Keyed differently but indexes correspond 1:1 with `dir_table`. + handle_table: std.ArrayHashMapUnmanaged(*Directory, void, Directory.TableAdapter, false), + ready_dirs: std.DoublyLinkedList, + + const FileId = struct { + volumeSerialNumber: windows.ULONG, + indexNumber: windows.LARGE_INTEGER, + }; + + const Directory = struct { + reaction_set: ReactionSet, + id: FileId, + file: Io.File, + state: enum { idle, listening, ready }, + iosb: windows.IO_STATUS_BLOCK, + // 64 KB is the packet size limit when monitoring over a network. + // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw#remarks + buffer: [64 * 1024]u8 align(@alignOf(windows.FILE.NOTIFY.INFORMATION)), + ready_node: std.DoublyLinkedList.Node, + + /// Start listening for events, buffer field will be overwritten eventually. + fn startListening(dir: *Directory, w: *Watch) !void { + assert(dir.file.flags.nonblocking); + assert(dir.state == .idle); + switch (windows.ntdll.NtNotifyChangeDirectoryFileEx( + dir.file.handle, + null, + ¬ifyApc, + w, + &dir.iosb, + &dir.buffer, + dir.buffer.len, + .{ + .FILE_NAME = true, + .DIR_NAME = true, + .SIZE = true, + .LAST_WRITE = true, + .CREATION = true, + }, + .FALSE, + .Notify, + )) { + .SUCCESS, .PENDING => dir.state = .listening, + .ILLEGAL_FUNCTION => return error.ReadDirectoryChangesUnsupported, + else => |status| return windows.unexpectedStatus(status), + } + } + + fn notifyApc(apc_context: ?*anyopaque, iosb: *windows.IO_STATUS_BLOCK, _: windows.ULONG) align(std.Io.Threaded.apc_align) callconv(.winapi) void { + const w: *Watch = @ptrCast(@alignCast(apc_context)); + const dir: *Directory = @fieldParentPtr("iosb", iosb); + assert(iosb.u.Status != .PENDING); + assert(dir.state == .listening); + w.os.ready_dirs.append(&dir.ready_node); + dir.state = .ready; + } + + fn init(gpa: Allocator, path: Cache.Path) !*Directory { + // The following code is a drawn out NtCreateFile call. (mostly adapted from Io.Dir.makeOpenDirAccessMaskW) + // It's necessary in order to get the specific flags that are required when calling ReadDirectoryChangesW. + var dir_handle: windows.HANDLE = undefined; + const root_fd = path.root_dir.handle.handle; + const sub_path = path.subPathOrDot(); + const sub_path_w = try Io.Threaded.sliceToPrefixedFileW(root_fd, sub_path, .{}); // TODO eliminate this call + var iosb: windows.IO_STATUS_BLOCK = undefined; + switch (windows.ntdll.NtCreateFile( + &dir_handle, + .{ + .SPECIFIC = .{ .FILE_DIRECTORY = .{ + .LIST = true, + } }, + .STANDARD = .{ .SYNCHRONIZE = true }, + .GENERIC = .{ .READ = true }, + }, + &.{ + .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w.span())) null else root_fd, + .ObjectName = @constCast(&sub_path_w.string()), + }, + &iosb, + null, + .{}, + .VALID_FLAGS, + .OPEN, + .{ + .DIRECTORY_FILE = true, + .IO = .ASYNCHRONOUS, + .OPEN_FOR_BACKUP_INTENT = true, + }, + null, + 0, + )) { + .SUCCESS => {}, + .OBJECT_NAME_INVALID => return error.BadPathName, + .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, + .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, + .NOT_A_DIRECTORY => return error.NotDir, + // This can happen if the directory has 'List folder contents' permission set to 'Deny' + .ACCESS_DENIED => return error.AccessDenied, + .INVALID_PARAMETER => unreachable, + else => |rc| return windows.unexpectedStatus(rc), + } + assert(dir_handle != windows.INVALID_HANDLE_VALUE); + errdefer windows.CloseHandle(dir_handle); + + const dir_id = try getFileId(dir_handle); + + const dir = try gpa.create(Directory); + dir.* = .{ + .reaction_set = .empty, + .id = dir_id, + .file = .{ .handle = dir_handle, .flags = .{ .nonblocking = true } }, + .state = .idle, + .iosb = undefined, + .buffer = undefined, + .ready_node = undefined, + }; + return dir; + } + + fn deinit(dir: *Directory, gpa: Allocator, w: *Watch) void { + state: switch (dir.state) { + .idle => {}, + .listening => { + var cancel_iosb: windows.IO_STATUS_BLOCK = undefined; + _ = windows.ntdll.NtCancelIoFileEx(dir.file.handle, &dir.iosb, &cancel_iosb); + while (switch (dir.state) { + .idle => unreachable, + .listening => true, + .ready => false, + }) Io.Threaded.waitForApcOrAlert(); + continue :state .ready; + }, + .ready => w.os.ready_dirs.remove(&dir.ready_node), + } + windows.CloseHandle(dir.file.handle); + gpa.destroy(dir); + } + + /// Useful to make `*Directory` a key in `std.ArrayHashMap`. + const TableAdapter = struct { + pub fn hash(_: TableAdapter, lhs_dir: *Directory) u32 { + return @truncate(Hash.hash(lhs_dir.id.volumeSerialNumber, @ptrCast(&lhs_dir.id.indexNumber))); + } + pub fn eql(_: TableAdapter, lhs_dir: *Directory, rhs_dir: *Directory, rhs_index: usize) bool { + _ = rhs_index; + return lhs_dir.id.volumeSerialNumber == rhs_dir.id.volumeSerialNumber and + lhs_dir.id.indexNumber == rhs_dir.id.indexNumber; + } + }; + }; + + fn init(cwd_path: []const u8) !Watch { + _ = cwd_path; + return .{ + .dir_table = .{}, + .dir_count = 0, + .os = switch (builtin.os.tag) { + .windows => .{ + .handle_table = .empty, + .ready_dirs = .{}, + }, + else => {}, + }, + .generation = 0, + }; + } + + fn getFileId(handle: windows.HANDLE) !FileId { + var file_id: FileId = undefined; + var io_status: windows.IO_STATUS_BLOCK = undefined; + var volume_info: windows.FILE.FS_VOLUME_INFORMATION = undefined; + switch (windows.ntdll.NtQueryVolumeInformationFile( + handle, + &io_status, + &volume_info, + @sizeOf(windows.FILE.FS_VOLUME_INFORMATION), + .Volume, + )) { + .SUCCESS => {}, + // Buffer overflow here indicates that there is more information available than was able to be stored in the buffer + // size provided. This is treated as success because the type of variable-length information that this would be relevant for + // (name, volume name, etc) we don't care about. + .BUFFER_OVERFLOW => {}, + else => |rc| return windows.unexpectedStatus(rc), + } + file_id.volumeSerialNumber = volume_info.VolumeSerialNumber; + var internal_info: windows.FILE.INTERNAL_INFORMATION = undefined; + switch (windows.ntdll.NtQueryInformationFile( + handle, + &io_status, + &internal_info, + @sizeOf(windows.FILE.INTERNAL_INFORMATION), + .Internal, + )) { + .SUCCESS => {}, + else => |rc| return windows.unexpectedStatus(rc), + } + file_id.indexNumber = internal_info.IndexNumber; + return file_id; + } + + fn markDirtySteps(w: *Watch, gpa: Allocator, dir: *Directory) !bool { + var any_dirty = false; + const bytes_returned = dir.iosb.Information; + if (bytes_returned == 0) { + std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); + markAllFilesDirty(w, gpa); + try dir.startListening(w); + return true; + } + var file_name_buf: [std.fs.max_path_bytes]u8 = undefined; + var offset: usize = 0; + while (true) { + const notify: *windows.FILE.NOTIFY.INFORMATION = @ptrCast(@alignCast(&dir.buffer[offset])); + const file_name = file_name_buf[0..std.unicode.wtf16LeToWtf8(&file_name_buf, notify.fileName())]; + if (dir.reaction_set.getPtr(".")) |glob_set| + any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); + if (dir.reaction_set.getPtr(file_name)) |step_set| + any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + if (notify.NextEntryOffset == 0) + break; + + offset += notify.NextEntryOffset; + } + + // We call this now since at this point we have finished reading dir.buffer. + try dir.startListening(w); + return any_dirty; + } + + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + // Add missing marks and note persisted ones. + for (steps) |step| { + for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { + const dir = dir: { + const gop = try w.dir_table.getOrPut(gpa, path); + if (!gop.found_existing) { + const dir: *Directory = try .init(gpa, path); + errdefer dir.deinit(gpa, w); + // `dir.id` may already be present in the table in + // the case that we have multiple Cache.Path instances + // that compare inequal but ultimately point to the same + // directory on the file system. + // In such case, we must revert adding this directory, but keep + // the additions to the step set. + const dh_gop = try w.os.handle_table.getOrPut(gpa, dir); + if (dh_gop.found_existing) { + dir.deinit(gpa, w); + _ = w.dir_table.pop(); + break :dir w.os.handle_table.keys()[dh_gop.index]; + } else { + assert(dh_gop.index == gop.index); + try dir.startListening(w); + break :dir dir; + } + } + break :dir w.os.handle_table.keys()[gop.index]; + }; + for (files.items) |basename| { + const gop = try dir.reaction_set.getOrPut(gpa, basename); + if (!gop.found_existing) gop.value_ptr.* = .{}; + try gop.value_ptr.put(gpa, step, w.generation); + } + } + } + + { + // Remove marks for files that are no longer inputs. + var i: usize = 0; + while (i < w.os.handle_table.entries.len) { + const dir = w.os.handle_table.keys()[i]; + { + var step_set_i: usize = 0; + while (step_set_i < dir.reaction_set.entries.len) { + const step_set = &dir.reaction_set.values()[step_set_i]; + var dirent_i: usize = 0; + while (dirent_i < step_set.entries.len) { + const generations = step_set.values(); + if (generations[dirent_i] == w.generation) { + dirent_i += 1; + continue; + } + step_set.swapRemoveAt(dirent_i); + } + if (step_set.entries.len > 0) { + step_set_i += 1; + continue; + } + dir.reaction_set.swapRemoveAt(step_set_i); + } + if (dir.reaction_set.entries.len > 0) { + i += 1; + continue; + } + } + + w.dir_table.swapRemoveAt(i); + w.os.handle_table.swapRemoveAt(i); + dir.deinit(gpa, w); + } + w.generation +%= 1; + } + w.dir_count = w.dir_table.count(); + } + + fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { + for (0..2) |attempt| { + while (w.os.ready_dirs.popFirst()) |ready_node| { + const dir: *Directory = @fieldParentPtr("ready_node", ready_node); + assert(dir.state == .ready); + dir.state = .idle; + switch (dir.iosb.u.Status) { + .SUCCESS => return if (try markDirtySteps(w, gpa, dir)) .dirty else .clean, + .PENDING => unreachable, + .CANCELLED => {}, + else => |status| return windows.unexpectedStatus(status), + } + try dir.startListening(w); + } + try io.checkCancel(); + if (attempt == 1) return .timeout; + const delay_interval: windows.LARGE_INTEGER = switch (timeout) { + .none => std.math.minInt(windows.LARGE_INTEGER), + .ms => |ms| -@as(windows.LARGE_INTEGER, ms) * (std.time.ns_per_ms / 100), + }; + _ = windows.ntdll.NtDelayExecution(.TRUE, &delay_interval); + } else unreachable; + } + }, + .dragonfly, .freebsd, .netbsd, .openbsd, .ios, .tvos, .visionos, .watchos => struct { + const posix = std.posix; + + kq_fd: i32, + /// Indexes correspond 1:1 with `dir_table`. + handles: std.MultiArrayList(struct { + rs: ReactionSet, + /// If the corresponding dir_table Path has sub_path == "", then it + /// suffices as the open directory handle, and this value will be + /// -1. Otherwise, it needs to be opened in update(), and will be + /// stored here. + dir_fd: i32, + }), + + const dir_open_flags: posix.O = f: { + var f: posix.O = .{ + .ACCMODE = .RDONLY, + .NOFOLLOW = false, + .DIRECTORY = true, + .CLOEXEC = true, + }; + if (@hasField(posix.O, "EVTONLY")) f.EVTONLY = true; + if (@hasField(posix.O, "PATH")) f.PATH = true; + break :f f; + }; + + const EV = std.c.EV; + const NOTE = std.c.NOTE; + + fn init(cwd_path: []const u8) !Watch { + _ = cwd_path; + return .{ + .dir_table = .{}, + .dir_count = 0, + .os = .{ + .kq_fd = try Io.Kqueue.createFileDescriptor(), + .handles = .empty, + }, + .generation = 0, + }; + } + + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + const handles = &w.os.handles; + for (steps) |step| { + for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { + const reaction_set = rs: { + const gop = try w.dir_table.getOrPut(gpa, path); + if (!gop.found_existing) { + const skip_open_dir = path.sub_path.len == 0; + const dir_fd = if (skip_open_dir) + path.root_dir.handle.handle + else + posix.openat(path.root_dir.handle.handle, path.sub_path, dir_open_flags, 0) catch |err| { + fatal("failed to open directory {f}: {t}", .{ path, err }); + }; + // Empirically the dir has to stay open or else no events are triggered. + errdefer if (!skip_open_dir) std.Io.Threaded.closeFd(dir_fd); + const changes = [1]posix.Kevent{.{ + .ident = @bitCast(@as(isize, dir_fd)), + .filter = std.c.EVFILT.VNODE, + .flags = EV.ADD | EV.ENABLE | EV.CLEAR, + .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE, + .data = 0, + .udata = gop.index, + }}; + _ = try Io.Kqueue.kevent(w.os.kq_fd, &changes, &.{}, null); + assert(handles.len == gop.index); + try handles.append(gpa, .{ + .rs = .{}, + .dir_fd = if (skip_open_dir) -1 else dir_fd, + }); + } + + break :rs &handles.items(.rs)[gop.index]; + }; + for (files.items) |basename| { + const gop = try reaction_set.getOrPut(gpa, basename); + if (!gop.found_existing) gop.value_ptr.* = .{}; + try gop.value_ptr.put(gpa, step, w.generation); + } + } + } + + { + // Remove marks for files that are no longer inputs. + var i: usize = 0; + while (i < handles.len) { + { + const reaction_set = &handles.items(.rs)[i]; + var step_set_i: usize = 0; + while (step_set_i < reaction_set.entries.len) { + const step_set = &reaction_set.values()[step_set_i]; + var dirent_i: usize = 0; + while (dirent_i < step_set.entries.len) { + const generations = step_set.values(); + if (generations[dirent_i] == w.generation) { + dirent_i += 1; + continue; + } + step_set.swapRemoveAt(dirent_i); + } + if (step_set.entries.len > 0) { + step_set_i += 1; + continue; + } + reaction_set.swapRemoveAt(step_set_i); + } + if (reaction_set.entries.len > 0) { + i += 1; + continue; + } + } + + // If the sub_path == "" then this patch has already the + // dir fd that we need to use as the ident to remove the + // event. If it was opened above with openat() then we need + // to access that data via the dir_fd field. + const path = w.dir_table.keys()[i]; + const dir_fd = if (path.sub_path.len == 0) + path.root_dir.handle.handle + else + handles.items(.dir_fd)[i]; + assert(dir_fd != -1); + + // The changelist also needs to update the udata field of the last + // event, since we are doing a swap remove, and we store the dir_table + // index in the udata field. + const last_dir_fd = fd: { + const last_path = w.dir_table.keys()[handles.len - 1]; + const last_dir_fd = if (last_path.sub_path.len == 0) + last_path.root_dir.handle.handle + else + handles.items(.dir_fd)[handles.len - 1]; + assert(last_dir_fd != -1); + break :fd last_dir_fd; + }; + const changes = [_]posix.Kevent{ + .{ + .ident = @bitCast(@as(isize, dir_fd)), + .filter = std.c.EVFILT.VNODE, + .flags = EV.DELETE, + .fflags = 0, + .data = 0, + .udata = i, + }, + .{ + .ident = @bitCast(@as(isize, last_dir_fd)), + .filter = std.c.EVFILT.VNODE, + .flags = EV.ADD, + .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE, + .data = 0, + .udata = i, + }, + }; + const filtered_changes = if (i == handles.len - 1) changes[0..1] else &changes; + _ = try Io.Kqueue.kevent(w.os.kq_fd, filtered_changes, &.{}, null); + if (path.sub_path.len != 0) std.Io.Threaded.closeFd(dir_fd); + + w.dir_table.swapRemoveAt(i); + handles.swapRemove(i); + } + w.generation +%= 1; + } + w.dir_count = w.dir_table.count(); + } + + fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { + _ = io; + var timespec_buffer: posix.timespec = undefined; + var event_buffer: [100]posix.Kevent = undefined; + var n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, timeout.toTimespec(×pec_buffer)); + if (n == 0) return .timeout; + const reaction_sets = w.os.handles.items(.rs); + var any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], false); + timespec_buffer = .{ .sec = 0, .nsec = 0 }; + while (n == event_buffer.len) { + n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, ×pec_buffer); + if (n == 0) break; + any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], any_dirty); + } + return if (any_dirty) .dirty else .clean; + } + + fn markDirtySteps( + gpa: Allocator, + reaction_sets: []ReactionSet, + events: []const std.c.Kevent, + start_any_dirty: bool, + ) bool { + var any_dirty = start_any_dirty; + for (events) |event| { + const index: usize = @intCast(event.udata); + const reaction_set = &reaction_sets[index]; + // If we knew the basename of the changed file, here we would + // mark only the step set dirty, and possibly the glob set: + //if (reaction_set.getPtr(".")) |glob_set| + // any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); + //if (reaction_set.getPtr(file_name)) |step_set| + // any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + // However we don't know the file name so just mark all the + // sets dirty for this directory. + for (reaction_set.values()) |*step_set| { + any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + } + } + return any_dirty; + } + }, + .macos => struct { + fse: FsEvents, + + fn init(cwd_path: []const u8) !Watch { + return .{ + .os = .{ .fse = try .init(cwd_path) }, + .dir_count = 0, + .dir_table = undefined, + .generation = undefined, + }; + } + fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + try w.os.fse.setPaths(gpa, steps); + w.dir_count = w.os.fse.watch_roots.len; + } + fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { + _ = io; + return w.os.fse.wait(gpa, switch (timeout) { + .none => null, + .ms => |ms| @as(u64, ms) * std.time.ns_per_ms, + }); + } + }, + else => void, +}; + +pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { + return Os.init(cwd_path, configuration, make_steps); +} + +pub const Match = struct { + /// Relative to the watched directory, the file path that triggers this + /// match. + basename: []const u8, + /// The step to re-run when file corresponding to `basename` is changed. + step_index: Configuration.Step.Index, + + pub const Context = struct { + pub fn hash(self: Context, a: Match) u32 { + _ = self; + var hasher = Hash.init(@intFromEnum(a.step_index)); + hasher.update(a.basename); + return @truncate(hasher.final()); + } + pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { + _ = self; + _ = b_index; + return a.step_index == b.step_index and std.mem.eql(u8, a.basename, b.basename); + } + }; +}; + +fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { + for (switch (builtin.os.tag) { + .windows => w.os.handle_table.keys(), + else => w.os.handle_table.values(), + }) |item| { + const reaction_set = switch (builtin.os.tag) { + .linux, .windows => item.reaction_set, + else => item, + }; + for (reaction_set.values()) |step_set| { + for (step_set.keys()) |step_index| { + const step = &w.make_steps[@intFromEnum(step_index)]; + _ = step.invalidateResult(gpa); + } + } + } +} + +fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool { + var this_any_dirty = false; + for (step_set.keys()) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + if (step.invalidateResult(gpa)) this_any_dirty = true; + } + return any_dirty or this_any_dirty; +} + +pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { + return Os.update(w, gpa, steps); +} + +pub const Timeout = union(enum) { + none, + ms: u16, + + pub fn to_i32_ms(t: Timeout) i32 { + return switch (t) { + .none => -1, + .ms => |ms| ms, + }; + } + + pub fn toTimespec(t: Timeout, buf: *std.posix.timespec) ?*std.posix.timespec { + return switch (t) { + .none => null, + .ms => |ms_u16| { + const ms: isize = ms_u16; + buf.* = .{ + .sec = @divTrunc(ms, std.time.ms_per_s), + .nsec = @rem(ms, std.time.ms_per_s) * std.time.ns_per_ms, + }; + return buf; + }, + }; + } +}; + +pub const WaitResult = enum { + timeout, + /// File system watching triggered on files that were marked as inputs to at least one Step. + /// Relevant steps have been marked dirty. + dirty, + /// File system watching triggered but none of the events were relevant to + /// what we are listening to. There is nothing to do. + clean, +}; + +pub fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { + return Os.wait(w, gpa, io, timeout); +} diff --git a/lib/compiler/Maker/Watch/FsEvents.zig b/lib/compiler/Maker/Watch/FsEvents.zig new file mode 100644 index 0000000000000000000000000000000000000000..0a56ce182255176222ef4b9a7a7bbb22abc9350d --- /dev/null +++ b/lib/compiler/Maker/Watch/FsEvents.zig @@ -0,0 +1,479 @@ +//! An implementation of file-system watching based on the `FSEventStream` API in macOS. +//! While macOS supports kqueue, it does not allow detecting changes to files without +//! placing watches on each individual file, meaning FD limits are reached incredibly +//! quickly. The File System Events API works differently: it implements *recursive* +//! directory watches, managed by a system service. Rather than being in libc, the API is +//! exposed by the CoreServices framework. To avoid a compile dependency on the framework +//! bundle, we dynamically load CoreServices with `std.DynLib`. +//! +//! While the logic in this file *is* specialized to `std.Build.Watch`, efforts have been +//! made to keep that specialization to a minimum. Other use cases could be served with +//! relatively minimal modifications to the `watch_paths` field and its usages (in +//! particular the `setPaths` function). We avoid using the global GCD dispatch queue in +//! favour of creating our own and synchronizing with an explicit semaphore, meaning this +//! logic is thread-safe and does not affect process-global state. +//! +//! In theory, this API is quite good at avoiding filesystem race conditions. In practice, +//! the logic that would avoid them is currently disabled, because the build system kind +//! of relies on them at the time of writing to avoid redundant work -- see the comment at +//! the top of `wait` for details. + +const enable_debug_logs = false; + +core_services: std.DynLib, +resolved_symbols: ResolvedSymbols, + +paths_arena: std.heap.ArenaAllocator.State, +/// The roots of the recursive watches. FSEvents has relatively small limits on the number +/// of watched paths, so this slice must not be too long. The paths themselves are allocated +/// into `paths_arena`, but this slice is allocated into the GPA. +watch_roots: [][:0]const u8, +/// All of the paths being watched. Value is the set of steps which depend on the file/directory. +/// Keys and values are in `paths_arena`, but this map is allocated into the GPA. +watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step), + +/// The semaphore we use to block the thread calling `wait` until the callback determines a relevant +/// event has occurred. This is retained across `wait` calls for simplicity and efficiency. +waiting_semaphore: dispatch.semaphore_t, +/// This dispatch queue is created by us and executes serially. It exists exclusively to trigger the +/// callbacks of the FSEventStream we create. This is not in use outside of `wait`, but is retained +/// across `wait` calls for simplicity and efficiency. +dispatch_queue: dispatch.queue_t, +/// In theory, this field avoids race conditions. In practice, it is essentially unused at the time +/// of writing. See the comment at the start of `wait` for details. +since_event: FSEventStreamEventId, + +cwd_path: []const u8, + +/// All of the symbols we pull from the `dlopen`ed CoreServices framework. If any of these symbols +/// is not present, `init` will close the framework and return an error. +const ResolvedSymbols = struct { + FSEventStreamCreate: *const fn ( + allocator: CFAllocatorRef, + callback: FSEventStreamCallback, + ctx: ?*const FSEventStreamContext, + paths_to_watch: CFArrayRef, + since_when: FSEventStreamEventId, + latency: CFTimeInterval, + flags: FSEventStreamCreateFlags, + ) callconv(.c) FSEventStreamRef, + FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch.queue_t) callconv(.c) void, + FSEventStreamStart: *const fn (stream: FSEventStreamRef) callconv(.c) bool, + FSEventStreamStop: *const fn (stream: FSEventStreamRef) callconv(.c) void, + FSEventStreamInvalidate: *const fn (stream: FSEventStreamRef) callconv(.c) void, + FSEventStreamRelease: *const fn (stream: FSEventStreamRef) callconv(.c) void, + FSEventStreamGetLatestEventId: *const fn (stream: ConstFSEventStreamRef) callconv(.c) FSEventStreamEventId, + FSEventsGetCurrentEventId: *const fn () callconv(.c) FSEventStreamEventId, + CFRelease: *const fn (cf: *const anyopaque) callconv(.c) void, + CFArrayCreate: *const fn ( + allocator: CFAllocatorRef, + values: [*]const usize, + num_values: CFIndex, + call_backs: ?*const CFArrayCallBacks, + ) callconv(.c) CFArrayRef, + CFStringCreateWithCString: *const fn ( + alloc: CFAllocatorRef, + c_str: [*:0]const u8, + encoding: CFStringEncoding, + ) callconv(.c) CFStringRef, + CFAllocatorCreate: *const fn (allocator: CFAllocatorRef, context: *const CFAllocatorContext) callconv(.c) CFAllocatorRef, + kCFAllocatorUseContext: *const CFAllocatorRef, +}; + +pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol, SystemResources }!FsEvents { + var core_services = std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/CoreServices") catch + return error.OpenFrameworkFailed; + errdefer core_services.close(); + + var resolved_symbols: ResolvedSymbols = undefined; + inline for (@typeInfo(ResolvedSymbols).@"struct".fields) |f| { + @field(resolved_symbols, f.name) = core_services.lookup(f.type, f.name) orelse return error.MissingCoreServicesSymbol; + } + + return .{ + .core_services = core_services, + .resolved_symbols = resolved_symbols, + .paths_arena = .{}, + .watch_roots = &.{}, + .watch_paths = .empty, + .waiting_semaphore = dispatch.semaphore_create(0) orelse return error.SystemResources, + .dispatch_queue = dispatch.queue_create("zig-watch", .SERIAL()) orelse return error.SystemResources, + // Not `.since_now`, because this means we can init `FsEvents` *before* we do work in order + // to notice any changes which happened during said work. + .since_event = resolved_symbols.FSEventsGetCurrentEventId(), + .cwd_path = cwd_path, + }; +} + +pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void { + fse.waiting_semaphore.as_object().release(); + fse.dispatch_queue.as_object().release(); + fse.core_services.close(io); + + gpa.free(fse.watch_roots); + fse.watch_paths.deinit(gpa); + { + var paths_arena = fse.paths_arena.promote(gpa); + paths_arena.deinit(); + } +} + +pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) !void { + var paths_arena_instance = fse.paths_arena.promote(gpa); + defer fse.paths_arena = paths_arena_instance.state; + const paths_arena = paths_arena_instance.allocator(); + + var need_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; + defer need_dirs.deinit(gpa); + + fse.watch_paths.clearRetainingCapacity(); + + // We take `step` by pointer for a slight memory optimization in a moment. + for (steps) |*step| { + for (step.*.inputs.table.keys(), step.*.inputs.table.values()) |path, *files| { + const resolved_dir = try std.fs.path.resolvePosix(paths_arena, &.{ + fse.cwd_path, path.root_dir.path orelse ".", path.sub_path, + }); + try need_dirs.put(gpa, resolved_dir, {}); + for (files.items) |file_name| { + const watch_path = if (std.mem.eql(u8, file_name, ".")) + resolved_dir + else + try std.fs.path.join(paths_arena, &.{ resolved_dir, file_name }); + const gop = try fse.watch_paths.getOrPut(gpa, watch_path); + if (gop.found_existing) { + const old_steps = gop.value_ptr.*; + const new_steps = try paths_arena.alloc(*std.Build.Step, old_steps.len + 1); + @memcpy(new_steps[0..old_steps.len], old_steps); + new_steps[old_steps.len] = step.*; + gop.value_ptr.* = new_steps; + } else { + // This is why we captured `step` by pointer! We can avoid allocating a slice of one + // step in the arena in the common case where a file is referenced by only one step. + gop.value_ptr.* = step[0..1]; + } + } + } + } + + { + // There's no point looking at directories inside other ones (e.g. "/foo" and "/foo/bar"). + // To eliminate these, we'll re-add directories in order of path length with a redundancy check. + const old_dirs = try gpa.dupe([]const u8, need_dirs.keys()); + defer gpa.free(old_dirs); + std.mem.sort([]const u8, old_dirs, {}, struct { + fn lessThan(ctx: void, a: []const u8, b: []const u8) bool { + ctx; + return std.mem.lessThan(u8, a, b); + } + }.lessThan); + need_dirs.clearRetainingCapacity(); + for (old_dirs) |dir_path| { + var it: std.fs.path.ComponentIterator(.posix, u8) = .init(dir_path); + while (it.next()) |component| { + if (need_dirs.contains(component.path)) { + // this path is '/foo/bar/qux', but '/foo' or '/foo/bar' was already added + break; + } + } else { + need_dirs.putAssumeCapacityNoClobber(dir_path, {}); + } + } + } + + // `need_dirs` is now a set of directories to watch with no redundancy. In practice, this is very + // likely to have reduced it to a quite small set (e.g. it'll typically coalesce a full `src/` + // directory into one entry). However, the FSEventStream API has a fairly low undocumented limit + // on total watches (supposedly 4096), so we should handle the case where we exceed it. To be + // safe, because this API can be a little unpredictable, we'll cap ourselves a little *below* + // that known limit. + if (need_dirs.count() > 2048) { + // Fallback: watch the whole filesystem. This is excessive, but... it *works* :P + if (enable_debug_logs) watch_log.debug("too many dirs; recursively watching root", .{}); + fse.watch_roots = try gpa.realloc(fse.watch_roots, 1); + fse.watch_roots[0] = "/"; + } else { + fse.watch_roots = try gpa.realloc(fse.watch_roots, need_dirs.count()); + for (fse.watch_roots, need_dirs.keys()) |*out, in| { + out.* = try paths_arena.dupeSentinel(u8, in, 0); + } + } + if (enable_debug_logs) { + watch_log.debug("watching {d} paths using {d} recursive watches:", .{ fse.watch_paths.count(), fse.watch_roots.len }); + for (fse.watch_roots) |dir_path| { + watch_log.debug("- '{s}'", .{dir_path}); + } + } +} + +pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory, StartFailed }!std.Build.Watch.WaitResult { + if (fse.watch_roots.len == 0) @panic("nothing to watch"); + + const rs = fse.resolved_symbols; + + // At the time of writing, using `since_event` in the obvious way causes redundant rebuilds + // to occur, because one step modifies a file which is an input to another step. The solution + // to this problem will probably be either: + // + // a) Don't include the output of one step as a watch input of another; only mark external + // files as watch inputs. Or... + // + // b) Note the current event ID when a step begins, and disregard events preceding that ID + // when considering whether to dirty that step in `eventCallback`. + // + // For now, to avoid the redundant rebuilds, we bypass this `since_event` mechanism. This does + // introduce race conditions, but the other `std.Build.Watch` implementations suffer from those + // too at the time of writing, so this is kind of expected. + fse.since_event = .since_now; + + const cf_allocator = rs.CFAllocatorCreate(rs.kCFAllocatorUseContext.*, &.{ + .version = 0, + .info = @constCast(&gpa), + .retain = null, + .release = null, + .copy_description = null, + .allocate = &cf_alloc_callbacks.allocate, + .reallocate = &cf_alloc_callbacks.reallocate, + .deallocate = &cf_alloc_callbacks.deallocate, + .preferred_size = null, + }) orelse return error.OutOfMemory; + defer rs.CFRelease(cf_allocator); + + const cf_paths = try gpa.alloc(?CFStringRef, fse.watch_roots.len); + @memset(cf_paths, null); + defer { + for (cf_paths) |o| if (o) |p| rs.CFRelease(p); + gpa.free(cf_paths); + } + for (fse.watch_roots, cf_paths) |raw_path, *cf_path| { + cf_path.* = rs.CFStringCreateWithCString(cf_allocator, raw_path, .utf8); + } + const cf_paths_array = rs.CFArrayCreate(cf_allocator, @ptrCast(cf_paths), @intCast(cf_paths.len), null); + defer rs.CFRelease(cf_paths_array); + + const callback_ctx: EventCallbackCtx = .{ + .fse = fse, + .gpa = gpa, + }; + const event_stream = rs.FSEventStreamCreate( + null, + &eventCallback, + &.{ + .version = 0, + .info = @constCast(&callback_ctx), + .retain = null, + .release = null, + .copy_description = null, + }, + cf_paths_array, + fse.since_event, + 0.05, // 0.05s latency; higher values increase efficiency by coalescing more events + .{ .watch_root = true, .file_events = true }, + ); + defer rs.FSEventStreamRelease(event_stream); + rs.FSEventStreamSetDispatchQueue(event_stream, fse.dispatch_queue); + defer rs.FSEventStreamInvalidate(event_stream); + if (!rs.FSEventStreamStart(event_stream)) return error.StartFailed; + defer rs.FSEventStreamStop(event_stream); + const result = fse.waiting_semaphore.wait(timeout: { + const ns = timeout_ns orelse break :timeout .FOREVER; + break :timeout .time(.NOW, @intCast(ns)); + }); + return switch (result) { + 0 => .dirty, + else => .timeout, + }; +} + +const cf_alloc_callbacks = struct { + const log = std.log.scoped(.cf_alloc); + fn allocate(size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque { + if (enable_debug_logs) log.debug("allocate {d}", .{size}); + _ = hint; + const gpa: *const Allocator = @ptrCast(@alignCast(info)); + const mem = gpa.alignedAlloc(u8, .of(usize), @intCast(size + @sizeOf(usize))) catch return null; + const metadata: *usize = @ptrCast(mem); + metadata.* = @intCast(size); + return mem[@sizeOf(usize)..].ptr; + } + fn reallocate(ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque { + if (enable_debug_logs) log.debug("reallocate @{*} {d}", .{ ptr, new_size }); + _ = hint; + if (ptr == null or new_size == 0) return null; // not a bug: documentation explicitly states that realloc on NULL should return NULL + const gpa: *const Allocator = @ptrCast(@alignCast(info)); + const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize)); + const old_size = @as(*const usize, @ptrCast(old_base)).*; + const old_mem = old_base[0 .. old_size + @sizeOf(usize)]; + const new_mem = gpa.realloc(old_mem, @intCast(new_size + @sizeOf(usize))) catch return null; + const metadata: *usize = @ptrCast(new_mem); + metadata.* = @intCast(new_size); + return new_mem[@sizeOf(usize)..].ptr; + } + fn deallocate(ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void { + if (enable_debug_logs) log.debug("deallocate @{*}", .{ptr}); + const gpa: *const Allocator = @ptrCast(@alignCast(info)); + const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize)); + const old_size = @as(*const usize, @ptrCast(old_base)).*; + const old_mem = old_base[0 .. old_size + @sizeOf(usize)]; + gpa.free(old_mem); + } +}; + +const EventCallbackCtx = struct { + fse: *FsEvents, + gpa: Allocator, +}; + +fn eventCallback( + stream: ConstFSEventStreamRef, + client_callback_info: ?*anyopaque, + num_events: usize, + events_paths_ptr: *anyopaque, + events_flags_ptr: [*]const FSEventStreamEventFlags, + events_ids_ptr: [*]const FSEventStreamEventId, +) callconv(.c) void { + const ctx: *const EventCallbackCtx = @ptrCast(@alignCast(client_callback_info)); + const fse = ctx.fse; + const gpa = ctx.gpa; + const rs = fse.resolved_symbols; + const events_paths_ptr_casted: [*]const [*:0]const u8 = @ptrCast(@alignCast(events_paths_ptr)); + const events_paths = events_paths_ptr_casted[0..num_events]; + const events_ids = events_ids_ptr[0..num_events]; + const events_flags = events_flags_ptr[0..num_events]; + var any_dirty = false; + for (events_paths, events_ids, events_flags) |event_path_nts, event_id, event_flags| { + _ = event_id; + if (event_flags.history_done) continue; // sentinel + const event_path = std.mem.span(event_path_nts); + switch (event_flags.must_scan_sub_dirs) { + false => { + if (fse.watch_paths.get(event_path)) |steps| { + assert(steps.len > 0); + for (steps) |s| { + if (s.invalidateResult(gpa)) any_dirty = true; + } + } + if (std.fs.path.dirname(event_path)) |event_dirname| { + // Modifying '/foo/bar' triggers the watch on '/foo'. + if (fse.watch_paths.get(event_dirname)) |steps| { + assert(steps.len > 0); + for (steps) |s| { + if (s.invalidateResult(gpa)) any_dirty = true; + } + } + } + }, + true => { + // This is unlikely, but can occasionally happen when bottlenecked: events have been + // coalesced into one. We want to see if any of these events are actually relevant + // to us. The only way we can reasonably do that in this rare edge case is iterate + // the watch paths and see if any is under this directory. That's acceptable because + // we would otherwise kick off a rebuild which would be clearing those paths anyway. + const changed_path = std.fs.path.dirname(event_path) orelse event_path; + for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| { + if (dirStartsWith(watching_path, changed_path)) { + for (steps) |s| { + if (s.invalidateResult(gpa)) any_dirty = true; + } + } + } + }, + } + } + if (any_dirty) { + fse.since_event = rs.FSEventStreamGetLatestEventId(stream); + _ = fse.waiting_semaphore.signal(); + } +} +fn dirStartsWith(path: []const u8, prefix: []const u8) bool { + if (std.mem.eql(u8, path, prefix)) return true; + if (!std.mem.startsWith(u8, path, prefix)) return false; + if (path[prefix.len] != '/') return false; // `path` is `/foo/barx`, `prefix` is `/foo/bar` + return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar` +} + +const CFAllocatorRef = ?*const opaque {}; +const CFArrayRef = *const opaque {}; +const CFStringRef = *const opaque {}; +const CFTimeInterval = f64; +const CFIndex = i32; +const CFOptionFlags = enum(u32) { _ }; +const CFAllocatorRetainCallBack = *const fn (info: ?*const anyopaque) callconv(.c) *const anyopaque; +const CFAllocatorReleaseCallBack = *const fn (info: ?*const anyopaque) callconv(.c) void; +const CFAllocatorCopyDescriptionCallBack = *const fn (info: ?*const anyopaque) callconv(.c) CFStringRef; +const CFAllocatorAllocateCallBack = *const fn (alloc_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque; +const CFAllocatorReallocateCallBack = *const fn (ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque; +const CFAllocatorDeallocateCallBack = *const fn (ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void; +const CFAllocatorPreferredSizeCallBack = *const fn (size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) CFIndex; +const CFAllocatorContext = extern struct { + version: CFIndex, + info: ?*anyopaque, + retain: ?CFAllocatorRetainCallBack, + release: ?CFAllocatorReleaseCallBack, + copy_description: ?CFAllocatorCopyDescriptionCallBack, + allocate: CFAllocatorAllocateCallBack, + reallocate: ?CFAllocatorReallocateCallBack, + deallocate: ?CFAllocatorDeallocateCallBack, + preferred_size: ?CFAllocatorPreferredSizeCallBack, +}; +const CFArrayCallBacks = opaque {}; +const CFStringEncoding = enum(u32) { + invalid_id = std.math.maxInt(u32), + mac_roman = 0, + windows_latin_1 = 0x500, + iso_latin_1 = 0x201, + next_step_latin = 0xB01, + ascii = 0x600, + unicode = 0x100, + utf8 = 0x8000100, + non_lossy_ascii = 0xBFF, +}; + +const FSEventStreamRef = *opaque {}; +const ConstFSEventStreamRef = *const @typeInfo(FSEventStreamRef).pointer.child; +const FSEventStreamCallback = *const fn ( + stream: ConstFSEventStreamRef, + client_callback_info: ?*anyopaque, + num_events: usize, + event_paths: *anyopaque, + event_flags: [*]const FSEventStreamEventFlags, + event_ids: [*]const FSEventStreamEventId, +) callconv(.c) void; +const FSEventStreamContext = extern struct { + version: CFIndex, + info: ?*anyopaque, + retain: ?CFAllocatorRetainCallBack, + release: ?CFAllocatorReleaseCallBack, + copy_description: ?CFAllocatorCopyDescriptionCallBack, +}; +const FSEventStreamEventId = enum(u64) { + since_now = std.math.maxInt(u64), + _, +}; +const FSEventStreamCreateFlags = packed struct(u32) { + use_cf_types: bool = false, + no_defer: bool = false, + watch_root: bool = false, + ignore_self: bool = false, + file_events: bool = false, + _: u27 = 0, +}; +const FSEventStreamEventFlags = packed struct(u32) { + must_scan_sub_dirs: bool, + user_dropped: bool, + kernel_dropped: bool, + event_ids_wrapped: bool, + history_done: bool, + root_changed: bool, + mount: bool, + unmount: bool, + _: u24 = 0, +}; + +const dispatch = std.c.dispatch; +const std = @import("std"); +const Io = std.Io; +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const watch_log = std.log.scoped(.watch); +const FsEvents = @This(); diff --git a/lib/compiler/Maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig new file mode 100644 index 0000000000000000000000000000000000000000..fd3806e8b2b925f6badad2830ddd8d780abab2b4 --- /dev/null +++ b/lib/compiler/Maker/WebServer.zig @@ -0,0 +1,940 @@ +const WebServer = @This(); + +const builtin = @import("builtin"); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; +const Io = std.Io; +const abi = std.Build.abi; +const assert = std.debug.assert; +const http = std.http; +const log = std.log.scoped(.web_server); +const mem = std.mem; +const net = std.Io.net; + +const Fuzz = @import("Fuzz.zig"); +const Graph = @import("Graph.zig"); +const Step = @import("Step.zig"); + +gpa: Allocator, +graph: *const Graph, +all_steps: []const Configuration.Step.Index, +listen_address: net.IpAddress, +root_prog_node: std.Progress.Node, +watch: bool, + +tcp_server: ?net.Server, +serve_task: ?Io.Future(Io.Cancelable!void), + +/// Uses `Io.Clock.awake`. +base_timestamp: Io.Timestamp, +/// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`. +step_names_trailing: []u8, + +/// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps. +/// Accessed atomically. +step_status_bits: []u8, + +fuzz: ?Fuzz, +time_report_mutex: Io.Mutex, +time_report_msgs: [][]u8, +time_report_update_times: []i64, + +build_status: std.atomic.Value(abi.BuildStatus), +/// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate` +/// to increment this value. Each client thread waits for this increment with `Io.futexWaitTimeout`, so +/// `notifyUpdate` will wake those threads. Updates are sent on a short interval regardless, so it +/// is recommended to only use `notifyUpdate` for changes which the user should see immediately. For +/// instance, we do not call `notifyUpdate` when the number of "unique runs" in the fuzzer changes, +/// because this value changes quickly so this would result in constantly spamming all clients with +/// an unreasonable number of packets. +update_id: std.atomic.Value(u32), + +runner_request_mutex: Io.Mutex, +runner_request_ready_cond: Io.Condition, +runner_request_empty_cond: Io.Condition, +runner_request: ?RunnerRequest, + +/// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates +/// on a fixed interval of this many milliseconds. +const default_update_interval_ms = 500; + +pub const base_clock: Io.Clock = .awake; + +/// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`. +pub fn notifyUpdate(ws: *WebServer) void { + _ = ws.update_id.rmw(.Add, 1, .release); + ws.graph.io.futexWake(u32, &ws.update_id.raw, 16); +} + +pub const Options = struct { + gpa: Allocator, + graph: *const Graph, + all_steps: []const Configuration.Step.Index, + root_prog_node: std.Progress.Node, + watch: bool, + listen_address: net.IpAddress, + base_timestamp: Io.Clock.Timestamp, + configuration: *const Configuration, +}; +pub fn init(opts: Options) WebServer { + // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` + // instead of threads, so that the web server can function in single-threaded builds. + comptime assert(!builtin.single_threaded); + assert(opts.base_timestamp.clock == base_clock); + + const all_steps = opts.all_steps; + const c = opts.configuration; + + const step_names_trailing = opts.gpa.alloc(u8, len: { + var name_bytes: usize = 0; + for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len; + break :len name_bytes + all_steps.len * 4; + }) catch @panic("out of memory"); + { + const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]); + var idx: usize = all_steps.len * 4; + for (all_steps, step_name_lens) |step_index, *name_len| { + const step_name = step_index.ptr(c).name.slice(c); + name_len.* = @intCast(step_name.len); + @memcpy(step_names_trailing[idx..][0..step_name.len], step_name); + idx += step_name.len; + } + assert(idx == step_names_trailing.len); + } + + const step_status_bits = opts.gpa.alloc( + u8, + std.math.divCeil(usize, all_steps.len, 4) catch unreachable, + ) catch @panic("out of memory"); + @memset(step_status_bits, 0); + + const time_reports_len: usize = if (opts.graph.time_report) all_steps.len else 0; + const time_report_msgs = opts.gpa.alloc([]u8, time_reports_len) catch @panic("out of memory"); + const time_report_update_times = opts.gpa.alloc(i64, time_reports_len) catch @panic("out of memory"); + @memset(time_report_msgs, &.{}); + @memset(time_report_update_times, std.math.minInt(i64)); + + return .{ + .gpa = opts.gpa, + .graph = opts.graph, + .all_steps = all_steps, + .listen_address = opts.listen_address, + .root_prog_node = opts.root_prog_node, + .watch = opts.watch, + + .tcp_server = null, + .serve_task = null, + + .base_timestamp = opts.base_timestamp.raw, + .step_names_trailing = step_names_trailing, + + .step_status_bits = step_status_bits, + + .fuzz = null, + .time_report_mutex = .init, + .time_report_msgs = time_report_msgs, + .time_report_update_times = time_report_update_times, + + .build_status = .init(.idle), + .update_id = .init(0), + + .runner_request_mutex = .init, + .runner_request_ready_cond = .init, + .runner_request_empty_cond = .init, + .runner_request = null, + }; +} +pub fn deinit(ws: *WebServer) void { + const gpa = ws.gpa; + const io = ws.graph.io; + + gpa.free(ws.step_names_trailing); + gpa.free(ws.step_status_bits); + + if (ws.fuzz) |*f| f.deinit(); + for (ws.time_report_msgs) |msg| gpa.free(msg); + gpa.free(ws.time_report_msgs); + gpa.free(ws.time_report_update_times); + + if (ws.serve_task) |t| { + if (ws.tcp_server) |*s| s.stream.close(io); + t.await(); + } + if (ws.tcp_server) |*s| s.deinit(); + + gpa.free(ws.step_names_trailing); +} +pub fn start(ws: *WebServer) error{AlreadyReported}!void { + assert(ws.tcp_server == null); + assert(ws.serve_task == null); + const io = ws.graph.io; + + ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| { + log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err }); + return error.AlreadyReported; + }; + ws.serve_task = io.concurrent(serve, .{ws}) catch |err| { + log.err("unable to spawn web server thread: {t}", .{err}); + ws.tcp_server.?.deinit(io); + ws.tcp_server = null; + return error.AlreadyReported; + }; + + log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.socket.address}); + if (ws.listen_address.getPort() == 0) { + log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.socket.address}); + } +} +fn serve(ws: *WebServer) Io.Cancelable!void { + const io = ws.graph.io; + var group: Io.Group = .init; + defer group.cancel(io); + while (true) { + var stream = ws.tcp_server.?.accept(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + log.err("failed to accept connection: {t}", .{e}); + return; + }, + }; + group.concurrent(io, accept, .{ ws, stream }) catch |err| { + log.err("unable to spawn connection thread: {t}", .{err}); + stream.close(io); + continue; + }; + } +} + +pub fn startBuild(ws: *WebServer) void { + if (ws.fuzz) |*fuzz| { + fuzz.deinit(); + ws.fuzz = null; + } + for (ws.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic); + ws.build_status.store(.running, .monotonic); + ws.notifyUpdate(); +} + +pub fn updateStepStatus( + ws: *WebServer, + step_index: Configuration.Step.Index, + new_status: abi.StepUpdate.Status, +) void { + // TODO don't do linear search, especially in a hot loop like this + const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + if (s == step_index) break @intCast(i); + } else unreachable; + const ptr = &ws.step_status_bits[step_idx / 4]; + const bit_offset: u3 = @intCast((step_idx % 4) * 2); + const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset); + const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset; + _ = @atomicRmw(u8, ptr, .Xor, mask, .monotonic); + ws.notifyUpdate(); +} + +pub fn finishBuild(ws: *WebServer, opts: struct { + fuzz: bool, +}) void { + if (opts.fuzz) { + switch (builtin.os.tag) { + // Current implementation depends on two things that need to be ported to Windows: + // * Memory-mapping to share data between the fuzzer and build runner. + // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving + // many addresses to source locations). + .windows => std.process.fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), + else => {}, + } + if (@bitSizeOf(usize) != 64) { + // Current implementation depends on posix.mmap()'s second + // parameter, `length: usize`, being compatible with file system's + // u64 return value. This is not the case on 32-bit platforms. + // Affects or affected by issues #5185, #22523, and #22464. + std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); + } + + assert(ws.fuzz == null); + + ws.build_status.store(.fuzz_init, .monotonic); + ws.notifyUpdate(); + + ws.fuzz = Fuzz.init( + ws.gpa, + ws.graph.io, + ws.all_steps, + ws.root_prog_node, + .{ .forever = .{ .ws = ws } }, + ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)}); + ws.fuzz.?.start(); + } + + ws.build_status.store(if (ws.watch) .watching else .idle, .monotonic); + ws.notifyUpdate(); +} + +pub fn now(s: *const WebServer) i64 { + const io = s.graph.io; + const ts = base_clock.now(io); + return @intCast(s.base_timestamp.durationTo(ts).toNanoseconds()); +} + +fn accept(ws: *WebServer, stream: net.Stream) void { + const io = ws.graph.io; + defer { + // `net.Stream.close` wants to helpfully overwrite `stream` with + // `undefined`, but it cannot do so since it is an immutable parameter. + var copy = stream; + copy.close(io); + } + var send_buffer: [4096]u8 = undefined; + var recv_buffer: [4096]u8 = undefined; + var connection_reader = stream.reader(io, &recv_buffer); + var connection_writer = stream.writer(io, &send_buffer); + var server: http.Server = .init(&connection_reader.interface, &connection_writer.interface); + + while (true) { + var request = server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return log.err("failed to receive http request: {t}", .{err}), + }; + switch (request.upgradeRequested()) { + .websocket => |opt_key| { + const key = opt_key orelse return log.err("missing websocket key", .{}); + var web_socket = request.respondWebSocket(.{ .key = key }) catch { + return log.err("failed to respond web socket: {t}", .{connection_writer.err.?}); + }; + ws.serveWebSocket(&web_socket) catch |err| { + log.err("failed to serve websocket: {t}", .{err}); + return; + }; + comptime unreachable; + }, + .other => |name| return log.err("unknown upgrade request: {s}", .{name}), + .none => { + ws.serveRequest(&request) catch |err| switch (err) { + error.AlreadyReported => return, + else => { + log.err("failed to serve '{s}': {t}", .{ request.head.target, err }); + return; + }, + }; + }, + } + } +} + +fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { + const io = ws.graph.io; + + var prev_build_status = ws.build_status.load(.monotonic); + + const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len); + defer ws.gpa.free(prev_step_status_bits); + for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| { + copy.* = @atomicLoad(u8, shared, .monotonic); + } + + var recv_thread = try io.concurrent(recvWebSocketMessages, .{ ws, sock }); + defer recv_thread.cancel(io); + + { + const hello_header: abi.Hello = .{ + .status = prev_build_status, + .flags = .{ + .time_report = ws.graph.time_report, + }, + .timestamp = ws.now(), + .steps_len = @intCast(ws.all_steps.len), + }; + var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits }; + try sock.writeMessageVec(&bufs, .binary); + } + + var prev_fuzz: Fuzz.Previous = .init; + var prev_time: i64 = std.math.minInt(i64); + while (true) { + const start_time = ws.now(); + const start_update_id = ws.update_id.load(.acquire); + + if (ws.fuzz) |*fuzz| { + try fuzz.sendUpdate(sock, &prev_fuzz); + } + + { + try ws.time_report_mutex.lock(io); + defer ws.time_report_mutex.unlock(io); + for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| { + if (update_time <= prev_time) continue; + // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so + // that we don't hold up the build system on the client accepting this packet. + const owned_msg = try ws.gpa.dupe(u8, msg); + defer ws.gpa.free(owned_msg); + // Temporarily unlock, then re-lock after the message is sent. + ws.time_report_mutex.unlock(io); + defer ws.time_report_mutex.lockUncancelable(io); + try sock.writeMessage(owned_msg, .binary); + } + } + + { + const build_status = ws.build_status.load(.monotonic); + if (build_status != prev_build_status) { + prev_build_status = build_status; + const msg: abi.StatusUpdate = .{ .new = build_status }; + try sock.writeMessage(@ptrCast(&msg), .binary); + } + } + + for (prev_step_status_bits, ws.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| { + const cur_byte = @atomicLoad(u8, shared, .monotonic); + if (prev_byte.* == cur_byte) continue; + const cur: [4]abi.StepUpdate.Status = .{ + @enumFromInt(@as(u2, @truncate(cur_byte >> 0))), + @enumFromInt(@as(u2, @truncate(cur_byte >> 2))), + @enumFromInt(@as(u2, @truncate(cur_byte >> 4))), + @enumFromInt(@as(u2, @truncate(cur_byte >> 6))), + }; + const prev: [4]abi.StepUpdate.Status = .{ + @enumFromInt(@as(u2, @truncate(prev_byte.* >> 0))), + @enumFromInt(@as(u2, @truncate(prev_byte.* >> 2))), + @enumFromInt(@as(u2, @truncate(prev_byte.* >> 4))), + @enumFromInt(@as(u2, @truncate(prev_byte.* >> 6))), + }; + for (cur, prev, byte_idx * 4..) |cur_status, prev_status, step_idx| { + const msg: abi.StepUpdate = .{ .step_idx = @intCast(step_idx), .bits = .{ .status = cur_status } }; + if (cur_status != prev_status) try sock.writeMessage(@ptrCast(&msg), .binary); + } + prev_byte.* = cur_byte; + } + + prev_time = start_time; + + const old_cp = io.swapCancelProtection(.blocked); + defer _ = io.swapCancelProtection(old_cp); + io.futexWaitTimeout( + u32, + &ws.update_id.raw, + start_update_id, + .{ .duration = .{ + .clock = .awake, + .raw = .fromMilliseconds(default_update_interval_ms), + } }, + ) catch |err| switch (err) { + error.Canceled => unreachable, + }; + } +} +fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void { + const io = ws.graph.io; + + while (true) { + const msg = sock.readSmallMessage() catch return; + if (msg.opcode != .binary) continue; + if (msg.data.len == 0) continue; + const tag: abi.ToServerTag = @enumFromInt(msg.data[0]); + switch (tag) { + _ => continue, + .rebuild => while (true) { + ws.runner_request_mutex.lock(io) catch |err| switch (err) { + error.Canceled => return, + }; + defer ws.runner_request_mutex.unlock(io); + if (ws.runner_request == null) { + ws.runner_request = .rebuild; + ws.runner_request_ready_cond.signal(io); + break; + } + ws.runner_request_empty_cond.wait(io, &ws.runner_request_mutex) catch return; + }, + } + } +} + +fn serveRequest(ws: *WebServer, req: *http.Server.Request) !void { + // Strip an optional leading '/debug' component from the request. + const target: []const u8, const debug: bool = target: { + if (mem.eql(u8, req.head.target, "/debug")) break :target .{ "/", true }; + if (mem.eql(u8, req.head.target, "/debug/")) break :target .{ "/", true }; + if (mem.startsWith(u8, req.head.target, "/debug/")) break :target .{ req.head.target["/debug".len..], true }; + break :target .{ req.head.target, false }; + }; + + if (mem.eql(u8, target, "/")) return serveLibFile(ws, req, "build-web/index.html", "text/html"); + if (mem.eql(u8, target, "/main.js")) return serveLibFile(ws, req, "build-web/main.js", "application/javascript"); + if (mem.eql(u8, target, "/style.css")) return serveLibFile(ws, req, "build-web/style.css", "text/css"); + if (mem.eql(u8, target, "/time_report.css")) return serveLibFile(ws, req, "build-web/time_report.css", "text/css"); + if (mem.eql(u8, target, "/main.wasm")) return serveClientWasm(ws, req, if (debug) .Debug else .ReleaseFast); + + if (ws.fuzz) |*fuzz| { + if (mem.eql(u8, target, "/sources.tar")) return fuzz.serveSourcesTar(req); + } + + try req.respond("not found", .{ + .status = .not_found, + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "text/plain" }, + }, + }); +} + +fn serveLibFile( + ws: *WebServer, + request: *http.Server.Request, + sub_path: []const u8, + content_type: []const u8, +) !void { + return serveFile(ws, request, .{ + .root_dir = ws.graph.zig_lib_directory, + .sub_path = sub_path, + }, content_type); +} +fn serveClientWasm( + ws: *WebServer, + req: *http.Server.Request, + optimize_mode: std.builtin.OptimizeMode, +) !void { + var arena_state: std.heap.ArenaAllocator = .init(ws.gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // We always rebuild the wasm on-the-fly, so that if it is edited the user can just refresh the page. + const bin_path = try buildClientWasm(ws, arena, optimize_mode); + return serveFile(ws, req, bin_path, "application/wasm"); +} + +pub fn serveFile( + ws: *WebServer, + request: *http.Server.Request, + path: Cache.Path, + content_type: []const u8, +) !void { + const gpa = ws.gpa; + const io = ws.graph.io; + // The desired API is actually sendfile, which will require enhancing http.Server. + // We load the file with every request so that the user can make changes to the file + // and refresh the HTML page without restarting this server. + const file_contents = path.root_dir.handle.readFileAlloc(io, path.sub_path, gpa, .limited(10 * 1024 * 1024)) catch |err| { + log.err("failed to read '{f}': {t}", .{ path, err }); + return error.AlreadyReported; + }; + defer gpa.free(file_contents); + try request.respond(file_contents, .{ + .extra_headers = &.{ + .{ .name = "Content-Type", .value = content_type }, + cache_control_header, + }, + }); +} +pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void { + const graph = ws.graph; + const io = graph.io; + + var send_buffer: [0x4000]u8 = undefined; + var response = try request.respondStreaming(&send_buffer, .{ + .respond_options = .{ + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "application/x-tar" }, + cache_control_header, + }, + }, + }); + + var archiver: std.tar.Writer = .{ .underlying_writer = &response.writer }; + + for (paths) |path| { + var file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| { + log.err("failed to open '{f}': {s}", .{ path, @errorName(err) }); + continue; + }; + defer file.close(io); + const stat = try file.stat(io); + var read_buffer: [1024]u8 = undefined; + var file_reader: Io.File.Reader = .initSize(file, io, &read_buffer, stat.size); + + // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can + // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI: + // it turns out the WASM treats the first path component as the module name, typically + // resulting in modules named "" and "src". The compiler needs to tell the build system + // about the module graph so that the build system can correctly encode this information in + // the tar file. + // + // Additionally, this needs to ensure that all path separators for both prefix and + // sub_path are using the POSIX-style `/` on platforms that don't use it as their native + // path separator. + archiver.prefix = path.root_dir.path orelse graph.cache.cwd; + try archiver.writeFile(path.sub_path, &file_reader, @intCast(stat.mtime.toSeconds())); + } + + // intentionally not calling `archiver.finishPedantically` + try response.end(); +} + +fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.OptimizeMode) !Cache.Path { + const root_name = "build-web"; + const arch_os_abi = "wasm32-freestanding"; + const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; + + const gpa = ws.gpa; + const graph = ws.graph; + const io = graph.io; + + const main_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "build-web/main.zig", + }; + const walk_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "docs/wasm/Walk.zig", + }; + const html_render_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "docs/wasm/html_render.zig", + }; + + var argv: std.ArrayList([]const u8) = .empty; + + try argv.appendSlice(arena, &.{ + graph.zig_exe, "build-exe", // + "-fno-entry", // + "-O", @tagName(optimize), // + "-target", arch_os_abi, // + "-mcpu", cpu_features, // + "--cache-dir", graph.global_cache_root.path orelse ".", // + "--global-cache-dir", graph.global_cache_root.path orelse ".", // + "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // + "--name", root_name, // + "-rdynamic", // + "-fsingle-threaded", // + "--dep", "Walk", // + "--dep", "html_render", // + try std.fmt.allocPrint(arena, "-Mroot={f}", .{main_src_path}), // + try std.fmt.allocPrint(arena, "-MWalk={f}", .{walk_src_path}), // + "--dep", "Walk", // + try std.fmt.allocPrint(arena, "-Mhtml_render={f}", .{html_render_src_path}), // + "--listen=-", + }); + + var child = try std.process.spawn(io, .{ + .argv = argv.items, + .environ_map = &graph.environ_map, + .stdin = .pipe, + .stdout = .pipe, + .stderr = .pipe, + }); + defer child.kill(io); + + var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited }); + defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {}; + + var stdout_buffer: [512]u8 = undefined; + var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer); + const stdout = &stdout_reader.interface; + + { + var w = child.stdin.?.writer(io, &.{}); + w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch |err| switch (err) { + error.WriteFailed => return w.err.?, + }; + } + + const Header = std.zig.Server.Message.Header; + + var result: ?Cache.Path = null; + var result_error_bundle = std.zig.ErrorBundle.empty; + var body_buffer: std.ArrayList(u8) = .empty; + defer body_buffer.deinit(gpa); + + while (true) { + const header = stdout.takeStruct(Header, .little) catch |err| switch (err) { + error.ReadFailed => |e| return e, + error.EndOfStream => break, + }; + body_buffer.clearRetainingCapacity(); + try stdout.appendExact(gpa, &body_buffer, header.bytes_len); + const body = body_buffer.items; + + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return error.ZigProtocolVersionMismatch; + } + }, + .error_bundle => { + result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body); + }, + .emit_digest => { + const EmitDigest = std.zig.Server.Message.EmitDigest; + const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body); + if (!ebp_hdr.flags.cache_hit) { + log.info("source changes detected; rebuilt wasm component", .{}); + } + const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; + result = .{ + .root_dir = graph.global_cache_root, + .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), + }; + }, + else => {}, // ignore other messages + } + } + + const stderr_contents = try stderr_task.await(io); + if (stderr_contents.len > 0) { + std.debug.print("{s}", .{stderr_contents}); + } + + // Send EOF to stdin. + child.stdin.?.close(io); + child.stdin = null; + + switch (try child.wait(io)) { + .exited => |code| { + if (code != 0) { + log.err( + "the following command exited with error code {d}:\n{s}", + .{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + ); + return error.WasmCompilationFailed; + } + }, + .signal => |sig| { + log.err( + "the following command terminated with signal {t}:\n{s}", + .{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + ); + return error.WasmCompilationFailed; + }, + .stopped => |sig| { + log.err( + "the following command stopped unexpectedly with signal {t}:\n{s}", + .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, + ); + return error.WasmCompilationFailed; + }, + .unknown => { + log.err( + "the following command terminated unexpectedly:\n{s}", + .{try Step.allocPrintCmd(arena, .inherit, null, argv.items)}, + ); + return error.WasmCompilationFailed; + }, + } + + if (result_error_bundle.errorMessageCount() > 0) { + try result_error_bundle.renderToStderr(io, .{}, .auto); + log.err("the following command failed with {d} compilation errors:\n{s}", .{ + result_error_bundle.errorMessageCount(), + try Step.allocPrintCmd(arena, .inherit, null, argv.items), + }); + return error.WasmCompilationFailed; + } + + const base_path = result orelse { + log.err("child process failed to report result\n{s}", .{ + try Step.allocPrintCmd(arena, .inherit, null, argv.items), + }); + return error.WasmCompilationFailed; + }; + const bin_name = try std.zig.binNameAlloc(arena, .{ + .root_name = root_name, + .target = &(std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ + .arch_os_abi = arch_os_abi, + .cpu_features = cpu_features, + }) catch unreachable) catch unreachable), + .output_mode = .Exe, + }); + return base_path.join(arena, bin_name); +} + +fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 { + var file_reader: Io.File.Reader = .initStreaming(file, io, &.{}); + return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { + error.ReadFailed => return file_reader.err.?, + else => |e| return e, + }; +} + +pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { + compile_step: Configuration.Step.Index, + + use_llvm: bool, + stats: abi.time_report.CompileResult.Stats, + ns_total: u64, + + llvm_pass_timings_len: u32, + files_len: u32, + decls_len: u32, + + /// The trailing data of `abi.time_report.CompileResult`, except the step name. + trailing: []const u8, +}) void { + const gpa = ws.gpa; + const io = ws.graph.io; + + // TODO don't do linear search + const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + if (s == opts.compile_step) break @intCast(i); + } else unreachable; + + const old_buf = old: { + ws.time_report_mutex.lock(io) catch return; + defer ws.time_report_mutex.unlock(io); + const old = ws.time_report_msgs[step_idx]; + ws.time_report_msgs[step_idx] = &.{}; + break :old old; + }; + const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory"); + + const out_header: *align(1) abi.time_report.CompileResult = @ptrCast(buf[0..@sizeOf(abi.time_report.CompileResult)]); + out_header.* = .{ + .step_idx = step_idx, + .flags = .{ + .use_llvm = opts.use_llvm, + }, + .stats = opts.stats, + .ns_total = opts.ns_total, + .llvm_pass_timings_len = opts.llvm_pass_timings_len, + .files_len = opts.files_len, + .decls_len = opts.decls_len, + }; + @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing); + + { + ws.time_report_mutex.lock(io) catch return; + defer ws.time_report_mutex.unlock(io); + assert(ws.time_report_msgs[step_idx].len == 0); + ws.time_report_msgs[step_idx] = buf; + ws.time_report_update_times[step_idx] = ws.now(); + } + ws.notifyUpdate(); +} + +pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void { + const gpa = ws.gpa; + const io = ws.graph.io; + + // TODO don't do linear search + const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + if (s == step_index) break @intCast(i); + } else unreachable; + + const old_buf = old: { + ws.time_report_mutex.lock(io) catch return; + defer ws.time_report_mutex.unlock(io); + const old = ws.time_report_msgs[step_idx]; + ws.time_report_msgs[step_idx] = &.{}; + break :old old; + }; + const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory"); + const out: *align(1) abi.time_report.GenericResult = @ptrCast(buf); + out.* = .{ + .step_idx = step_idx, + .ns_total = @intCast(duration.toNanoseconds()), + }; + { + ws.time_report_mutex.lock(io) catch return; + defer ws.time_report_mutex.unlock(io); + assert(ws.time_report_msgs[step_idx].len == 0); + ws.time_report_msgs[step_idx] = buf; + ws.time_report_update_times[step_idx] = ws.now(); + } + ws.notifyUpdate(); +} + +pub fn updateTimeReportRunTest( + ws: *WebServer, + run_step_index: Configuration.Step.Index, + tests: *const Step.Run.CachedTestMetadata, + ns_per_test: []const u64, +) void { + const gpa = ws.gpa; + const io = ws.graph.io; + + // TODO don't do linear search + const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + if (s == run_step_index) break @intCast(i); + } else unreachable; + + assert(tests.names.len == ns_per_test.len); + const tests_len: u32 = @intCast(tests.names.len); + + const new_len: u64 = len: { + var names_len: u64 = 0; + for (0..tests_len) |i| { + names_len += tests.testName(@intCast(i)).len + 1; + } + break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len; + }; + const old_buf = old: { + ws.time_report_mutex.lock(io) catch return; + defer ws.time_report_mutex.unlock(io); + const old = ws.time_report_msgs[step_idx]; + ws.time_report_msgs[step_idx] = &.{}; + break :old old; + }; + const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory"); + + const out_header: *align(1) abi.time_report.RunTestResult = @ptrCast(buf[0..@sizeOf(abi.time_report.RunTestResult)]); + out_header.* = .{ + .step_idx = step_idx, + .tests_len = tests_len, + }; + var offset: usize = @sizeOf(abi.time_report.RunTestResult); + const ns_per_test_out: []align(1) u64 = @ptrCast(buf[offset..][0 .. tests_len * 8]); + @memcpy(ns_per_test_out, ns_per_test); + offset += tests_len * 8; + for (0..tests_len) |i| { + const name = tests.testName(@intCast(i)); + @memcpy(buf[offset..][0..name.len], name); + buf[offset..][name.len] = 0; + offset += name.len + 1; + } + assert(offset == buf.len); + + { + ws.time_report_mutex.lock(io) catch return; + defer ws.time_report_mutex.unlock(io); + assert(ws.time_report_msgs[step_idx].len == 0); + ws.time_report_msgs[step_idx] = buf; + ws.time_report_update_times[step_idx] = ws.now(); + } + ws.notifyUpdate(); +} + +const RunnerRequest = union(enum) { + rebuild, +}; +pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { + const io = ws.graph.io; + ws.runner_request_mutex.lock(io) catch return; + defer ws.runner_request_mutex.unlock(io); + if (ws.runner_request) |req| { + ws.runner_request = null; + ws.runner_request_empty_cond.signal(); + return req; + } + return null; +} +pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest { + const io = ws.graph.io; + try ws.runner_request_mutex.lock(io); + defer ws.runner_request_mutex.unlock(io); + while (true) { + if (ws.runner_request) |req| { + ws.runner_request = null; + ws.runner_request_empty_cond.signal(io); + return req; + } + try ws.runner_request_ready_cond.wait(io, &ws.runner_request_mutex); + } +} + +const cache_control_header: http.Header = .{ + .name = "Cache-Control", + .value = "max-age=0, must-revalidate", +}; diff --git a/lib/compiler/maker.zig b/lib/compiler/maker.zig deleted file mode 100644 index 78b472530f5e8408bc07a4c5f1c9f0767a98fd66..0000000000000000000000000000000000000000 --- a/lib/compiler/maker.zig +++ /dev/null @@ -1,1848 +0,0 @@ -const Maker = @This(); -const builtin = @import("builtin"); - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const Configuration = std.Build.Configuration; -const File = std.Io.File; -const Io = std.Io; -const Path = std.Build.Cache.Path; -const Writer = std.Io.Writer; -const assert = std.debug.assert; -const fatal = std.process.fatal; -const fmt = std.fmt; -const log = std.log; -const mem = std.mem; -const process = std.process; - -const Fuzz = @import("maker/Fuzz.zig"); -const Graph = @import("maker/Graph.zig"); -const Step = @import("maker/Step.zig"); -const Watch = @import("maker/Watch.zig"); -const WebServer = @import("maker/WebServer.zig"); - -pub const std_options: std.Options = .{ - .side_channels_mitigations = .none, - .http_disable_tls = true, -}; - -gpa: Allocator, -graph: *Graph, -install_paths: InstallPaths, -scanned_config: *const ScannedConfig, -steps: []Step, - -available_rss: usize, -max_rss_is_default: bool, -max_rss_mutex: Io.Mutex, -skip_oom_steps: bool, -unit_test_timeout_ns: ?u64, -watch: bool, -web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, -/// Allocated into `gpa`. -memory_blocked_steps: std.ArrayList(Configuration.Step.Index), -/// Allocated into `gpa`. -step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), - -error_style: ErrorStyle, -multiline_errors: MultilineErrors, -summary: Summary, - -pub fn main(init: process.Init.Minimal) !void { - // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not - // always the case. So, we do need a true gpa for some things. - var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); - defer _ = safe_gpa_state.deinit(); - const gpa = safe_gpa_state.allocator(); - - var threaded: std.Io.Threaded = .init(gpa, .{ - .environ = init.environ, - .argv0 = .init(init.args), - }); - defer threaded.deinit(); - const io = threaded.io(); - - // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. - var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - const args = try init.args.toSlice(arena); - - // skip my own exe name - var arg_idx: usize = 1; - - const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); - const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); - const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); - const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); - const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); - const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); - - const cwd: Io.Dir = .cwd(); - - const zig_lib_directory: Cache.Directory = .{ - .path = zig_lib_dir, - .handle = try cwd.openDir(io, zig_lib_dir, .{}), - }; - - const build_root_directory: Cache.Directory = .{ - .path = build_root, - .handle = try cwd.openDir(io, build_root, .{}), - }; - - const local_cache_directory: Cache.Directory = .{ - .path = local_cache_root, - .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), - }; - - const global_cache_directory: Cache.Directory = .{ - .path = global_cache_root, - .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), - }; - - var graph: Graph = .{ - .io = io, - .arena = arena, - .cache = .{ - .io = io, - .gpa = gpa, - .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), - .cwd = try process.currentPathAlloc(io, arena), - }, - .zig_exe = zig_exe, - .environ_map = try init.environ.createMap(arena), - .global_cache_root = global_cache_directory, - .zig_lib_directory = zig_lib_directory, - }; - - graph.cache.addPrefix(.{ .path = null, .handle = cwd }); - graph.cache.addPrefix(build_root_directory); - graph.cache.addPrefix(local_cache_directory); - graph.cache.addPrefix(global_cache_directory); - graph.cache.hash.addBytes(builtin.zig_version_string); - - var step_names: std.ArrayList([]const u8) = .empty; - var debug_log_scopes: std.ArrayList([]const u8) = .empty; - var help_menu = false; - var steps_menu = false; - var print_configuration = false; - var override_install_prefix: ?[]const u8 = null; - var override_lib_dir: ?[]const u8 = null; - var override_bin_dir: ?[]const u8 = null; - var override_include_dir: ?[]const u8 = null; - var error_style: ErrorStyle = .verbose; - var multiline_errors: MultilineErrors = .indent; - var summary: ?Summary = null; - var max_rss: u64 = 0; - var skip_oom_steps = false; - var test_timeout_ns: ?u64 = null; - var color: Color = .auto; - var watch = false; - var fuzz: ?Fuzz.Mode = null; - var debounce_interval_ms: u16 = 50; - var webui_listen: ?Io.net.IpAddress = null; - var verbose = false; - var sysroot: ?[]const u8 = null; - var search_prefixes: std.ArrayList([]const u8) = .empty; - var libc_file: ?[]const u8 = null; - var debug_pkg_config: bool = false; - // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, - // this will be the directory $glibc-build-dir/install/glibcs - // Given the example of the aarch64 target, this is the directory - // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. - // Also works for dynamic musl. - var libc_runtimes_dir: ?[]const u8 = null; - var enable_wine = false; - var enable_qemu = false; - var enable_wasmtime = false; - var enable_darling = false; - var enable_rosetta = false; - var reference_trace: ?u32 = null; - var run_args: ?[]const []const u8 = null; - - if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(ErrorStyle, str)) |style| { - error_style = style; - } - } - - if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(MultilineErrors, str)) |style| { - multiline_errors = style; - } - } - - while (nextArg(args, &arg_idx)) |arg| { - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - help_menu = true; - } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { - steps_menu = true; - } else if (mem.eql(u8, arg, "--print-configuration")) { - print_configuration = true; - } else if (mem.eql(u8, arg, "--verbose")) { - verbose = true; - } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { - override_install_prefix = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { - override_lib_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { - override_bin_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--prefix-include-dir")) { - override_include_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--sysroot")) { - sysroot = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--maxrss")) { - const max_rss_text = nextArgOrFatal(args, &arg_idx); - max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| - fatal("invalid byte size: '{s}': {t}", .{ max_rss_text, err }); - } else if (mem.eql(u8, arg, "--skip-oom-steps")) { - skip_oom_steps = true; - } else if (mem.eql(u8, arg, "--test-timeout")) { - const units: []const struct { []const u8, u64 } = &.{ - .{ "ns", 1 }, - .{ "nanosecond", 1 }, - .{ "us", std.time.ns_per_us }, - .{ "microsecond", std.time.ns_per_us }, - .{ "ms", std.time.ns_per_ms }, - .{ "millisecond", std.time.ns_per_ms }, - .{ "s", std.time.ns_per_s }, - .{ "second", std.time.ns_per_s }, - .{ "m", std.time.ns_per_min }, - .{ "minute", std.time.ns_per_min }, - .{ "h", std.time.ns_per_hour }, - .{ "hour", std.time.ns_per_hour }, - }; - const timeout_str = nextArgOrFatal(args, &arg_idx); - const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( - "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)", - .{timeout_str}, - ); - const num_str = timeout_str[0 .. num_end_idx + 1]; - const unit_str = timeout_str[num_end_idx + 1 ..]; - const unit_factor: f64 = for (units) |unit_and_factor| { - if (std.mem.eql(u8, unit_str, unit_and_factor[0])) { - break @floatFromInt(unit_and_factor[1]); - } - } else fatal( - "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)", - .{ timeout_str, unit_str }, - ); - const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal( - "invalid timeout '{s}': invalid number '{s}' ({t})", - .{ timeout_str, num_str, err }, - ); - test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); - } else if (mem.eql(u8, arg, "--search-prefix")) { - try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); - } else if (mem.eql(u8, arg, "--libc")) { - libc_file = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); - color = std.meta.stringToEnum(Color, next_arg) orelse { - fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ - arg, next_arg, - }); - }; - } else if (mem.eql(u8, arg, "--error-style")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected style after '{s}'", .{arg}); - error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { - fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); - }; - } else if (mem.eql(u8, arg, "--multiline-errors")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected style after '{s}'", .{arg}); - multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { - fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); - }; - } else if (mem.eql(u8, arg, "--summary")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg}); - summary = std.meta.stringToEnum(Summary, next_arg) orelse { - fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{ - arg, next_arg, - }); - }; - } else if (mem.eql(u8, arg, "--seed")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u32 after '{s}'", .{arg}); - graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {t}", .{ next_arg, err }); - }; - } else if (mem.eql(u8, arg, "--debounce")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u16 after '{s}'", .{arg}); - debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { - fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{ - next_arg, err, - }); - }; - } else if (mem.eql(u8, arg, "--webui")) { - if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; - } else if (mem.startsWith(u8, arg, "--webui=")) { - const addr_str = arg["--webui=".len..]; - if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); - webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { - fatal("invalid web UI address '{s}': {t}", .{ addr_str, err }); - }; - } else if (mem.eql(u8, arg, "--debug-log")) { - const next_arg = nextArgOrFatal(args, &arg_idx); - try debug_log_scopes.append(arena, next_arg); - } else if (mem.eql(u8, arg, "--debug-pkg-config")) { - debug_pkg_config = true; - } else if (mem.eql(u8, arg, "--debug-rt")) { - graph.debug_compiler_runtime_libs = .Debug; - } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { - graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse - fatal("unrecognized optimization mode: {s}", .{rest}); - } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { - // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--watch")) { - watch = true; - } else if (mem.eql(u8, arg, "--time-report")) { - graph.time_report = true; - if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; - } else if (mem.eql(u8, arg, "--fuzz")) { - fuzz = .{ .forever = undefined }; - if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; - } else if (mem.startsWith(u8, arg, "--fuzz=")) { - const value = arg["--fuzz=".len..]; - if (value.len == 0) fatal("missing argument to --fuzz", .{}); - - const unit: u8 = value[value.len - 1]; - const digits = switch (unit) { - '0'...'9' => value, - 'K', 'M', 'G' => value[0 .. value.len - 1], - else => fatal( - "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", - .{}, - ), - }; - - const amount = std.fmt.parseInt(u64, digits, 10) catch { - fatal( - "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", - .{}, - ); - }; - - const normalized_amount = std.math.mul(u64, amount, switch (unit) { - else => unreachable, - '0'...'9' => 1, - 'K' => 1000, - 'M' => 1_000_000, - 'G' => 1_000_000_000, - }) catch fatal("fuzzing limit amount overflows u64", .{}); - - fuzz = .{ - .limit = .{ - .amount = normalized_amount, - }, - }; - } else if (mem.eql(u8, arg, "-fincremental")) { - graph.incremental = true; - } else if (mem.eql(u8, arg, "-fno-incremental")) { - graph.incremental = false; - } else if (mem.eql(u8, arg, "-fwine")) { - enable_wine = true; - } else if (mem.eql(u8, arg, "-fno-wine")) { - enable_wine = false; - } else if (mem.eql(u8, arg, "-fqemu")) { - enable_qemu = true; - } else if (mem.eql(u8, arg, "-fno-qemu")) { - enable_qemu = false; - } else if (mem.eql(u8, arg, "-fwasmtime")) { - enable_wasmtime = true; - } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - enable_wasmtime = false; - } else if (mem.eql(u8, arg, "-frosetta")) { - enable_rosetta = true; - } else if (mem.eql(u8, arg, "-fno-rosetta")) { - enable_rosetta = false; - } else if (mem.eql(u8, arg, "-fdarling")) { - enable_darling = true; - } else if (mem.eql(u8, arg, "-fno-darling")) { - enable_darling = false; - } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { - graph.allow_so_scripts = true; - } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { - graph.allow_so_scripts = false; - } else if (mem.eql(u8, arg, "-freference-trace")) { - reference_trace = 256; - } else if (mem.startsWith(u8, arg, "-freference-trace=")) { - const num = arg["-freference-trace=".len..]; - reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - std.debug.print("unable to parse reference_trace count '{s}': {t}", .{ num, err }); - process.exit(1); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - reference_trace = null; - } else if (mem.cutPrefix(u8, arg, "-j")) |text| { - const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| - fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); - if (n < 1) fatal("number of jobs must be at least 1", .{}); - threaded.setAsyncLimit(.limited(n)); - graph.max_jobs = n; - } else if (mem.eql(u8, arg, "--")) { - run_args = argsRest(args, arg_idx); - break; - } else { - fatalWithHint("unrecognized argument: '{s}'", .{arg}); - } - } else { - try step_names.append(arena, arg); - } - } - - const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); - const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); - - graph.stderr_mode = switch (color) { - .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), - .on => .escape_codes, - .off => .no_color, - }; - - const scanned_config: ScannedConfig = sc: { - const configuration = c: { - var file = cwd.openFile(io, configure_path, .{}) catch |err| - fatal("failed to open configuration file {s}: {t}", .{ configure_path, err }); - defer file.close(io); - break :c Configuration.loadFile(arena, io, file) catch |err| - fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); - }; - var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; - for (configuration.steps, 0..) |*conf_step, step_index_usize| { - const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); - const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]); - if (flags.tag == .top_level) { - const name = step_index.ptr(&configuration).name.slice(&configuration); - try top_level_steps.put(arena, name, step_index); - } - } - break :sc .{ - .configuration = configuration, - .top_level_steps = top_level_steps, - }; - }; - - if (help_menu) { - var w = initStdoutWriter(io); - scanned_config.printUsage(&graph, w) catch |err| switch (err) { - error.WriteFailed => return stdout_writer_allocation.err.?, - else => |e| return e, - }; - w.flush() catch return stdout_writer_allocation.err.?; - return; - } else if (steps_menu) { - var w = initStdoutWriter(io); - scanned_config.printSteps(&graph, w) catch |err| switch (err) { - error.WriteFailed => return stdout_writer_allocation.err.?, - else => |e| return e, - }; - w.flush() catch return stdout_writer_allocation.err.?; - return; - } else if (print_configuration) { - var w = initStdoutWriter(io); - scanned_config.print(w) catch return stdout_writer_allocation.err.?; - w.flush() catch return stdout_writer_allocation.err.?; - return; - } - - if (webui_listen != null) { - if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); - if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); - } - - const main_progress_node = std.Progress.start(io, .{ - .disable_printing = (color == .off), - }); - defer main_progress_node.end(); - - const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ - .root_dir = .cwd(), - .sub_path = try Io.Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), - } else if (override_install_prefix) |cwd_relative| .{ - .root_dir = .cwd(), - .sub_path = cwd_relative, - } else .{ - .root_dir = build_root_directory, - .sub_path = "zig-out", - }; - - const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{ - .root_dir = .cwd(), - .sub_path = cwd_relative, - } else try install_prefix_path.join(arena, "lib"); - - const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{ - .root_dir = .cwd(), - .sub_path = cwd_relative, - } else try install_prefix_path.join(arena, "bin"); - - const install_include_path: Path = if (override_include_dir) |cwd_relative| .{ - .root_dir = .cwd(), - .sub_path = cwd_relative, - } else try install_prefix_path.join(arena, "include"); - - var maker: Maker = .{ - .gpa = gpa, - .graph = &graph, - .scanned_config = &scanned_config, - .install_paths = .{ - .prefix = install_prefix_path, - .lib = install_lib_path, - .bin = install_bin_path, - .include = install_include_path, - }, - .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), - - .available_rss = max_rss, - .max_rss_is_default = false, - .max_rss_mutex = .init, - .skip_oom_steps = skip_oom_steps, - .unit_test_timeout_ns = test_timeout_ns, - - .watch = watch, - .web_server = undefined, // set after `prepare` - .memory_blocked_steps = .empty, - .step_stack = .empty, - - .error_style = error_style, - .multiline_errors = multiline_errors, - .summary = summary orelse if (watch or webui_listen != null) .line else .failures, - }; - defer { - maker.memory_blocked_steps.deinit(gpa); - maker.step_stack.deinit(gpa); - } - - if (maker.available_rss == 0) { - maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); - maker.max_rss_is_default = true; - } - - maker.prepare(step_names.items) catch |err| switch (err) { - error.DependencyLoopDetected, error.InsufficientMemory => { - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(1); - }, - else => |e| return e, - }; - - var w: Watch = w: { - if (!watch) break :w undefined; - if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); - break :w try .init(graph.cache.cwd, &scanned_config.configuration, maker.steps); - }; - - const now = Io.Clock.Timestamp.now(io, .awake); - - maker.web_server = if (webui_listen) |listen_address| ws: { - if (builtin.single_threaded) unreachable; // `fatal` above - break :ws .init(.{ - .gpa = gpa, - .graph = &graph, - .all_steps = maker.step_stack.keys(), - .root_prog_node = main_progress_node, - .watch = watch, - .listen_address = listen_address, - .base_timestamp = now, - .configuration = &scanned_config.configuration, - }); - } else null; - - if (maker.web_server) |*ws| { - ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); - } - - rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H"); - }) { - if (maker.web_server) |*ws| ws.startBuild(); - - try maker.makeStepNames(step_names.items, main_progress_node, fuzz); - - if (maker.web_server) |*web_server| { - if (fuzz) |mode| if (mode != .forever) fatal( - "error: limited fuzzing is not implemented yet for --webui", - .{}, - ); - - web_server.finishBuild(.{ .fuzz = fuzz != null }); - } - - if (maker.web_server) |*ws| { - const c = &scanned_config.configuration; - assert(!watch); // fatal error after CLI parsing - while (true) switch (try ws.wait()) { - .rebuild => { - for (maker.step_stack.keys()) |step_index| { - const step = maker.stepByIndex(step_index); - step.state = .precheck_done; - const deps = step_index.ptr(c).deps.slice(c); - step.pending_deps = @intCast(deps.len); - step.reset(gpa); - } - continue :rebuild; - }, - }; - } - - // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. - if (!Watch.have_impl) unreachable; - - try w.update(gpa, maker.step_stack.keys()); - - // Wait until a file system notification arrives. Read all such events - // until the buffer is empty. Then wait for a debounce interval, resetting - // if any more events come in. After the debounce interval has passed, - // trigger a rebuild on all steps with modified inputs, as well as their - // recursive dependants. - var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; - const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ - w.dir_count, countSubProcesses(maker.steps, maker.step_stack.keys()), - }) catch &caption_buf; - var debouncing_node = main_progress_node.start(caption, 0); - var in_debounce = false; - while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { - .timeout => { - assert(in_debounce); - debouncing_node.end(); - markFailedStepsDirty(gpa, maker.steps, maker.step_stack.keys()); - continue :rebuild; - }, - .dirty => if (!in_debounce) { - in_debounce = true; - debouncing_node.end(); - debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); - }, - .clean => {}, - }; - } -} - -fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void { - for (all_steps) |step_index| { - const step = &make_steps[@intFromEnum(step_index)]; - switch (step.state) { - .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), - else => continue, - } - } - // Now that all dirty steps have been found, the remaining steps that - // succeeded from last run shall be marked "cached". - for (all_steps) |step_index| { - const step = &make_steps[@intFromEnum(step_index)]; - switch (step.state) { - .success => step.result_cached = true, - else => continue, - } - } -} - -fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize { - var count: usize = 0; - for (all_steps) |step_index| { - const s = &make_steps[@intFromEnum(step_index)]; - count += @intFromBool(s.getZigProcess() != null); - } - return count; -} - -const InstallPaths = struct { - prefix: Path, - lib: Path, - bin: Path, - include: Path, -}; - -fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step { - return &maker.steps[@intFromEnum(i)]; -} - -fn prepare(maker: *Maker, step_names: []const []const u8) !void { - const gpa = maker.gpa; - const graph = maker.graph; - const arena = graph.arena; - const seed: u32 = graph.random_seed; - const step_stack = &maker.step_stack; - const c = &maker.scanned_config.configuration; - - @memset(maker.steps, .{}); - - if (step_names.len == 0) { - try step_stack.put(gpa, c.default_step, {}); - } else { - try step_stack.ensureUnusedCapacity(gpa, step_names.len); - for (0..step_names.len) |i| { - const step_name = step_names[step_names.len - i - 1]; - const s = maker.scanned_config.top_level_steps.get(step_name) orelse { - log.info("to list available steps: zig build -l", .{}); - fatal("no such step: {s}", .{step_name}); - }; - step_stack.putAssumeCapacity(s, {}); - } - } - - const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys()); - - var rng = std.Random.DefaultPrng.init(seed); - const rand = rng.random(); - rand.shuffle(Configuration.Step.Index, starting_steps); - - for (starting_steps) |s| { - try constructGraphAndCheckForDependencyLoop(gpa, c, maker.steps, s, &maker.step_stack, rand); - } - - { - // Check that we have enough memory to complete the build. - var any_problems = false; - var max_needed: usize = 0; - for (step_stack.keys()) |step_index| { - const make_step = maker.stepByIndex(step_index); - const conf_step = step_index.ptr(c); - const max_rss = conf_step.max_rss.toBytes(); - if (max_rss == 0) continue; - max_needed = @max(max_needed, max_rss); - if (max_rss > maker.available_rss) { - if (maker.skip_oom_steps) { - make_step.state = .skipped_oom; - for (make_step.dependants.items) |dependant| { - maker.stepByIndex(dependant).pending_deps -= 1; - } - } else { - log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ - conf_step.owner.depPrefixSlice(c), - conf_step.name.slice(c), - max_rss, - maker.available_rss, - }); - any_problems = true; - } - } - } - if (any_problems) { - if (maker.max_rss_is_default) { - std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ - max_needed, - }); - } - return error.InsufficientMemory; - } - } -} - -fn makeStepNames( - maker: *Maker, - step_names: []const []const u8, - parent_prog_node: std.Progress.Node, - fuzz: ?Fuzz.Mode, -) !void { - const graph = maker.graph; - const gpa = maker.gpa; - const io = graph.io; - const step_stack = &maker.step_stack; - const top_level_steps = &maker.scanned_config.top_level_steps; - const c = &maker.scanned_config.configuration; - - { - // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, - // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking - // a step is initial when it actually became ready due to an earlier initial step. - var initial_set: std.ArrayList(Configuration.Step.Index) = .empty; - defer initial_set.deinit(gpa); - try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); - for (step_stack.keys()) |step_index| { - const s = maker.stepByIndex(step_index); - if (s.state == .precheck_done and s.pending_deps == 0) { - initial_set.appendAssumeCapacity(step_index); - } - } - - const step_prog = parent_prog_node.start("steps", step_stack.count()); - defer step_prog.end(); - - var group: Io.Group = .init; - defer group.cancel(io); - // Start working on all of the initial steps... - for (initial_set.items) |step_index| try stepReady(maker, &group, step_index, step_prog); - // ...and `makeStep` will trigger every other step when their last dependency finishes. - try group.await(io); - } - - assert(maker.memory_blocked_steps.items.len == 0); - - var test_pass_count: usize = 0; - var test_skip_count: usize = 0; - var test_fail_count: usize = 0; - var test_crash_count: usize = 0; - var test_timeout_count: usize = 0; - - var test_count: usize = 0; - - var success_count: usize = 0; - var skipped_count: usize = 0; - var failure_count: usize = 0; - var pending_count: usize = 0; - var total_compile_errors: usize = 0; - - var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); - defer cleanup_task.await(io); - - for (step_stack.keys()) |step_index| { - const make_step = maker.stepByIndex(step_index); - test_pass_count += make_step.test_results.passCount(); - test_skip_count += make_step.test_results.skip_count; - test_fail_count += make_step.test_results.fail_count; - test_crash_count += make_step.test_results.crash_count; - test_timeout_count += make_step.test_results.timeout_count; - - test_count += make_step.test_results.test_count; - - switch (make_step.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - .dependency_failure => pending_count += 1, - .success => success_count += 1, - .skipped, .skipped_oom => skipped_count += 1, - .failure => { - failure_count += 1; - const compile_errors_len = make_step.result_error_bundle.errorMessageCount(); - if (compile_errors_len > 0) { - total_compile_errors += compile_errors_len; - } - }, - } - } - - if (fuzz) |mode| blk: { - switch (builtin.os.tag) { - // Current implementation depends on two things that need to be ported to Windows: - // * Memory-mapping to share data between the fuzzer and build runner. - // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving - // many addresses to source locations). - .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), - else => {}, - } - if (@bitSizeOf(usize) != 64) { - // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, - // being compatible with file system's u64 return value. This is not the case - // on 32-bit platforms. - // Affects or affected by issues #5185, #22523, and #22464. - fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); - } - - switch (mode) { - .forever => break :blk, - .limit => {}, - } - - assert(mode == .limit); - var f = Fuzz.init( - gpa, - io, - step_stack.keys(), - parent_prog_node, - mode, - ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); - defer f.deinit(); - - f.start(); - try f.waitAndPrintReport(); - } - - // Every test has a state - assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); - - if (failure_count == 0) { - std.Progress.setStatus(.success); - } else { - std.Progress.setStatus(.failure); - } - - summary: { - switch (maker.summary) { - .all, .new, .line => {}, - .failures => if (failure_count == 0) break :summary, - .none => break :summary, - } - - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - const t = stderr.terminal(); - const w = &stderr.file_writer.interface; - - const total_count = success_count + failure_count + pending_count + skipped_count; - t.setColor(.cyan) catch {}; - t.setColor(.bold) catch {}; - w.writeAll("Build Summary: ") catch {}; - t.setColor(.reset) catch {}; - w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; - { - t.setColor(.dim) catch {}; - var first = true; - if (skipped_count > 0) { - w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; - first = false; - } - if (failure_count > 0) { - w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; - first = false; - } - if (!first) w.writeByte(')') catch {}; - t.setColor(.reset) catch {}; - } - - if (test_count > 0) { - w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; - t.setColor(.dim) catch {}; - var first = true; - if (test_skip_count > 0) { - w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; - first = false; - } - if (test_fail_count > 0) { - w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; - first = false; - } - if (test_crash_count > 0) { - w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; - first = false; - } - if (test_timeout_count > 0) { - w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; - first = false; - } - if (!first) w.writeByte(')') catch {}; - t.setColor(.reset) catch {}; - } - - w.writeAll("\n") catch {}; - - if (maker.summary == .line) break :summary; - - // Print a fancy tree with build results. - var step_stack_copy = try step_stack.clone(gpa); - defer step_stack_copy.deinit(gpa); - - var print_node: PrintNode = .{ .parent = null }; - if (step_names.len == 0) { - print_node.last = true; - printTreeStep(maker, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) { - error.Canceled => |e| return e, - else => {}, - }; - } else { - const last_index = if (maker.summary == .all) top_level_steps.count() else blk: { - var i: usize = step_names.len; - while (i > 0) { - i -= 1; - const step_index = top_level_steps.get(step_names[i]).?; - const step = maker.stepByIndex(step_index); - const found = switch (maker.summary) { - .all, .line, .none => unreachable, - .failures => step.state != .success, - .new => !step.result_cached, - }; - if (found) break :blk i; - } - break :blk top_level_steps.count(); - }; - for (step_names, 0..) |step_name, i| { - const step_index = top_level_steps.get(step_name).?; - print_node.last = i + 1 == last_index; - printTreeStep(maker, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) { - error.Canceled => |e| return e, - else => {}, - }; - } - } - w.writeByte('\n') catch {}; - } - - if (maker.watch or maker.web_server != null) return; - - // Perhaps in the future there could be an Advanced Options flag such as - // --debug-build-runner-leaks which would make this code return instead of - // calling exit. - - const code: u8 = code: { - if (failure_count == 0) break :code 0; // success - if (maker.error_style.verboseContext()) break :code 1; // failure; print build command - break :code 2; // failure; do not print build command - }; - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(code); -} - -fn stepReady( - maker: *Maker, - group: *Io.Group, - step_index: Configuration.Step.Index, - root_prog_node: std.Progress.Node, -) Io.Cancelable!void { - const graph = maker.graph; - const io = graph.io; - const c = &maker.scanned_config.configuration; - const max_rss = step_index.ptr(c).max_rss.toBytes(); - if (max_rss != 0) { - try maker.max_rss_mutex.lock(io); - defer maker.max_rss_mutex.unlock(io); - if (maker.available_rss < max_rss) { - // Running this step right now could possibly exceed the allotted RSS. - maker.memory_blocked_steps.append(maker.gpa, step_index) catch - @panic("TODO eliminate memory allocation here"); - return; - } - maker.available_rss -= max_rss; - } - group.async(io, makeStep, .{ maker, group, step_index, root_prog_node }); -} - -/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready -/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must -/// have already subtracted this value from `maker.available_rss`. This function will release the RSS -/// claim (i.e. add `s.max_rss` back into `maker.available_rss`) and queue any viable memory-blocked -/// steps after "make" completes for `s`. -fn makeStep( - maker: *Maker, - group: *Io.Group, - step_index: Configuration.Step.Index, - root_prog_node: std.Progress.Node, -) Io.Cancelable!void { - const graph = maker.graph; - const io = graph.io; - const gpa = maker.gpa; - const c = &maker.scanned_config.configuration; - const conf_step = step_index.ptr(c); - const step_name = conf_step.name.slice(c); - const deps = conf_step.deps.slice(c); - const make_step = maker.stepByIndex(step_index); - - { - const step_prog_node = root_prog_node.start(step_name, 0); - defer step_prog_node.end(); - - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip); - - const new_state: Step.State = for (deps) |dep_index| { - const dep_make_step = maker.stepByIndex(dep_index); - switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => break .dependency_failure, - - .success, .skipped => {}, - } - } else if (make_step.make(.{ - .progress_node = step_prog_node, - .watch = maker.watch, - .web_server = if (maker.web_server) |*ws| ws else null, - .unit_test_timeout_ns = maker.unit_test_timeout_ns, - .gpa = gpa, - })) state: { - break :state .success; - } else |err| switch (err) { - error.MakeFailed => .failure, - error.MakeSkipped => .skipped, - }; - - @atomicStore(Step.State, &make_step.state, new_state, .monotonic); - - switch (new_state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => { - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure); - std.Progress.setStatus(.failure_working); - }, - - .success, - .skipped, - => { - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success); - }, - } - } - - // No matter the result, we want to display error/warning messages. - if (make_step.result_error_bundle.errorMessageCount() > 0 or - make_step.result_error_msgs.items.len > 0 or - make_step.result_stderr.len > 0) - { - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - printErrorMessages(gpa, c, maker.steps, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) { - error.Canceled => |e| return e, - error.WriteFailed => switch (stderr.file_writer.err.?) { - error.Canceled => |e| return e, - else => {}, - }, - else => {}, - }; - } - - const max_rss = conf_step.max_rss.toBytes(); - if (max_rss != 0) { - var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty; - defer dispatch_set.deinit(gpa); - - // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` - // as a staging buffer to avoid recursing into `makeStep` while `maker.max_rss_mutex` is held. - { - try maker.max_rss_mutex.lock(io); - defer maker.max_rss_mutex.unlock(io); - maker.available_rss += max_rss; - dispatch_set.ensureUnusedCapacity(gpa, maker.memory_blocked_steps.items.len) catch - @panic("TODO eliminate memory allocation here"); - while (maker.memory_blocked_steps.getLast()) |candidate_index| { - const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes(); - if (maker.available_rss < candidate_max_rss) break; - assert(maker.memory_blocked_steps.pop() == candidate_index); - dispatch_set.appendAssumeCapacity(candidate_index); - } - } - for (dispatch_set.items) |candidate| { - group.async(io, makeStep, .{ maker, group, candidate, root_prog_node }); - } - } - - for (make_step.dependants.items) |dependant_index| { - const dependant = maker.stepByIndex(dependant_index); - // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. - if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { - try stepReady(maker, group, dependant_index, root_prog_node); - } - } -} - -fn printTreeStep( - maker: *const Maker, - step_index: Configuration.Step.Index, - stderr: Io.Terminal, - parent_node: *PrintNode, - step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), -) !void { - const writer = stderr.writer; - const first = step_stack.swapRemove(step_index); - const summary = maker.summary; - const c = &maker.scanned_config.configuration; - const conf_step = step_index.ptr(c); - const make_step = maker.stepByIndex(step_index); - const skip = switch (summary) { - .none, .line => unreachable, - .all => false, - .new => make_step.result_cached, - .failures => make_step.state == .success, - }; - if (skip) return; - try printPrefix(parent_node, stderr); - - if (parent_node.parent != null) { - if (parent_node.last) { - try printChildNodePrefix(stderr); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ - else => "+- ", - }); - } - } - - if (!first) try stderr.setColor(.dim); - - // dep_prefix omitted here because it is redundant with the tree. - try writer.writeAll(conf_step.name.slice(c)); - - const deps = conf_step.deps.slice(c); - - if (first) { - try printStepStatus(maker, step_index, stderr); - - const last_index = if (summary == .all) deps.len -| 1 else blk: { - var i: usize = deps.len; - while (i > 0) { - i -= 1; - - const dep_index = deps[i]; - const dep = maker.stepByIndex(dep_index); - const found = switch (summary) { - .all, .line, .none => unreachable, - .failures => dep.state != .success, - .new => !dep.result_cached, - }; - if (found) break :blk i; - } - break :blk deps.len -| 1; - }; - for (deps, 0..) |dep, i| { - var print_node: PrintNode = .{ - .parent = parent_node, - .last = i == last_index, - }; - try printTreeStep(maker, dep, stderr, &print_node, step_stack); - } - } else { - if (deps.len == 0) { - try writer.writeAll(" (reused)\n"); - } else { - try writer.print(" (+{d} more reused dependencies)\n", .{deps.len}); - } - try stderr.setColor(.reset); - } -} - -fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { - const s = maker.stepByIndex(step_index); - const writer = stderr.writer; - switch (s.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .dependency_failure => { - try stderr.setColor(.dim); - try writer.writeAll(" transitive failure\n"); - try stderr.setColor(.reset); - }, - - .success => { - try stderr.setColor(.green); - if (s.result_cached) { - try writer.writeAll(" cached"); - } else if (s.test_results.test_count > 0) { - const pass_count = s.test_results.passCount(); - assert(s.test_results.test_count == pass_count + s.test_results.skip_count); - try writer.print(" {d} pass", .{pass_count}); - if (s.test_results.skip_count > 0) { - try stderr.setColor(.reset); - try writer.writeAll(", "); - try stderr.setColor(.yellow); - try writer.print("{d} skip", .{s.test_results.skip_count}); - } - try stderr.setColor(.reset); - try writer.print(" ({d} total)", .{s.test_results.test_count}); - } else { - try writer.writeAll(" success"); - } - try stderr.setColor(.reset); - if (s.result_duration_ns) |ns| { - try stderr.setColor(.dim); - if (ns >= std.time.ns_per_min) { - try writer.print(" {d}m", .{ns / std.time.ns_per_min}); - } else if (ns >= std.time.ns_per_s) { - try writer.print(" {d}s", .{ns / std.time.ns_per_s}); - } else if (ns >= std.time.ns_per_ms) { - try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); - } else if (ns >= std.time.ns_per_us) { - try writer.print(" {d}us", .{ns / std.time.ns_per_us}); - } else { - try writer.print(" {d}ns", .{ns}); - } - try stderr.setColor(.reset); - } - if (s.result_peak_rss != 0) { - const rss = s.result_peak_rss; - try stderr.setColor(.dim); - if (rss >= 1000_000_000) { - try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); - } else if (rss >= 1000_000) { - try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); - } else if (rss >= 1000) { - try writer.print(" MaxRSS:{d}K", .{rss / 1000}); - } else { - try writer.print(" MaxRSS:{d}B", .{rss}); - } - try stderr.setColor(.reset); - } - try writer.writeAll("\n"); - }, - .skipped => { - try stderr.setColor(.yellow); - try writer.writeAll(" skipped\n"); - try stderr.setColor(.reset); - }, - .skipped_oom => { - const c = &maker.scanned_config.configuration; - const max_rss = step_index.ptr(c).max_rss.toBytes(); - try stderr.setColor(.yellow); - try writer.writeAll(" skipped (not enough memory)"); - try stderr.setColor(.dim); - try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ - max_rss, maker.available_rss, - }); - try stderr.setColor(.reset); - }, - .failure => { - try printStepFailure(maker.steps, step_index, stderr, false); - try stderr.setColor(.reset); - }, - } -} - -fn printStepFailure( - make_steps: []Step, - step_index: Configuration.Step.Index, - stderr: Io.Terminal, - dim: bool, -) !void { - const w = stderr.writer; - const s = &make_steps[@intFromEnum(step_index)]; - if (s.result_error_bundle.errorMessageCount() > 0) { - try stderr.setColor(.red); - try w.print(" {d} errors\n", .{ - s.result_error_bundle.errorMessageCount(), - }); - } else if (!s.test_results.isSuccess()) { - // These first values include all of the test "statuses". Every test is either passsed, - // skipped, failed, crashed, or timed out. - try stderr.setColor(.green); - try w.print(" {d} pass", .{s.test_results.passCount()}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - if (s.test_results.skip_count > 0) { - try w.writeAll(", "); - try stderr.setColor(.yellow); - try w.print("{d} skip", .{s.test_results.skip_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - if (s.test_results.fail_count > 0) { - try w.writeAll(", "); - try stderr.setColor(.red); - try w.print("{d} fail", .{s.test_results.fail_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - if (s.test_results.crash_count > 0) { - try w.writeAll(", "); - try stderr.setColor(.red); - try w.print("{d} crash", .{s.test_results.crash_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - if (s.test_results.timeout_count > 0) { - try w.writeAll(", "); - try stderr.setColor(.red); - try w.print("{d} timeout", .{s.test_results.timeout_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - try w.print(" ({d} total)", .{s.test_results.test_count}); - - // Memory leaks are intentionally written after the total, because is isn't a test *status*, - // but just a flag that any tests -- even passed ones -- can have. We also use a different - // separator, so it looks like: - // 2 pass, 1 skip, 2 fail (5 total); 2 leaks - if (s.test_results.leak_count > 0) { - try w.writeAll("; "); - try stderr.setColor(.red); - try w.print("{d} leaks", .{s.test_results.leak_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - - // It's usually not helpful to know how many error logs there were because they tend to - // just come with other errors (e.g. crashes and leaks print stack traces, and clean - // failures print error traces). So only mention them if they're the only thing causing - // the failure. - const show_err_logs: bool = show: { - var alt_results = s.test_results; - alt_results.log_err_count = 0; - break :show alt_results.isSuccess(); - }; - if (show_err_logs) { - try w.writeAll("; "); - try stderr.setColor(.red); - try w.print("{d} error logs", .{s.test_results.log_err_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - - try w.writeAll("\n"); - } else if (s.result_error_msgs.items.len > 0) { - try stderr.setColor(.red); - try w.writeAll(" failure\n"); - } else { - assert(s.result_stderr.len > 0); - try stderr.setColor(.red); - try w.writeAll(" w\n"); - } -} - -const PrintNode = struct { - parent: ?*PrintNode, - last: bool = false, -}; - -fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { - const parent = node.parent orelse return; - const writer = stderr.writer; - if (parent.parent == null) return; - try printPrefix(parent, stderr); - if (parent.last) { - try writer.writeAll(" "); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ - else => "| ", - }); - } -} - -fn printChildNodePrefix(stderr: Io.Terminal) !void { - try stderr.writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ - else => "+- ", - }); -} - -/// Traverse the dependency graph depth-first and make it undirected by having -/// steps know their dependants (they only know dependencies at start). -/// Along the way, check that there is no dependency loop, and record the steps -/// in traversal order in `step_stack`. -/// Each step has its dependencies traversed in random order, this accomplishes -/// two things: -/// - `step_stack` will be in randomized-depth-first order, so the build runner -/// spawns initial steps in a random order -/// - each step's `dependants` list is also filled in a random order, so that -/// when it finishes executing in `makeStep`, it spawns next steps to run in -/// random order -fn constructGraphAndCheckForDependencyLoop( - gpa: Allocator, - c: *const Configuration, - steps: []Step, - step_index: Configuration.Step.Index, - step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), - rand: std.Random, -) error{ DependencyLoopDetected, OutOfMemory }!void { - const make_step: *Step = &steps[@intFromEnum(step_index)]; - switch (make_step.state) { - .precheck_started => { - log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)}); - return error.DependencyLoopDetected; - }, - .precheck_unstarted => { - make_step.state = .precheck_started; - - const step = step_index.ptr(c); - const dependencies = step.deps.slice(c); - try step_stack.ensureUnusedCapacity(gpa, dependencies.len); - - // We dupe to avoid shuffling the steps in the summary, it depends - // on dependencies' order. - const deps = try gpa.dupe(Configuration.Step.Index, dependencies); - defer gpa.free(deps); - - rand.shuffle(Configuration.Step.Index, deps); - - for (deps) |dep| { - const dep_step: *Step = &steps[@intFromEnum(dep)]; - try step_stack.put(gpa, dep, {}); - try dep_step.dependants.append(gpa, step_index); - constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) { - error.DependencyLoopDetected => { - log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); - return err; - }, - else => return err, - }; - } - - make_step.state = .precheck_done; - make_step.pending_deps = @intCast(dependencies.len); - }, - .precheck_done => {}, - - // These don't happen until we actually run the step graph. - .dependency_failure => unreachable, - .success => unreachable, - .failure => unreachable, - .skipped => unreachable, - .skipped_oom => unreachable, - } -} - -pub fn printErrorMessages( - gpa: Allocator, - c: *const Configuration, - make_steps: []Step, - failing_step_index: Configuration.Step.Index, - options: std.zig.ErrorBundle.RenderOptions, - stderr: Io.Terminal, - error_style: ErrorStyle, - multiline_errors: MultilineErrors, -) !void { - const writer = stderr.writer; - if (error_style.verboseContext()) { - // Provide context for where these error messages are coming from by - // printing the corresponding Step subtree. - var step_stack: std.ArrayList(Configuration.Step.Index) = .empty; - defer step_stack.deinit(gpa); - try step_stack.append(gpa, failing_step_index); - while (true) { - const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])]; - if (last_step.dependants.items.len == 0) break; - try step_stack.append(gpa, last_step.dependants.items[0]); - } - - // Now, `step_stack` has the subtree that we want to print, in reverse order. - try stderr.setColor(.dim); - var indent: usize = 0; - while (step_stack.pop()) |step_index| : (indent += 1) { - if (indent > 0) { - try writer.splatByteAll(' ', (indent - 1) * 3); - try printChildNodePrefix(stderr); - } - - try writer.writeAll(step_index.ptr(c).name.slice(c)); - - if (step_index == failing_step_index) { - try printStepFailure(make_steps, step_index, stderr, true); - } else { - try writer.writeAll("\n"); - } - } - try stderr.setColor(.reset); - } else { - // Just print the failing step itself. - try stderr.setColor(.dim); - try writer.writeAll(failing_step_index.ptr(c).name.slice(c)); - try printStepFailure(make_steps, failing_step_index, stderr, true); - try stderr.setColor(.reset); - } - - const failing_step = &make_steps[@intFromEnum(failing_step_index)]; - - if (failing_step.result_stderr.len > 0) { - try writer.writeAll(failing_step.result_stderr); - if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { - try writer.writeAll("\n"); - } - } - - try failing_step.result_error_bundle.renderToTerminal(options, stderr); - - for (failing_step.result_error_msgs.items) |msg| { - try stderr.setColor(.red); - try writer.writeAll("error:"); - try stderr.setColor(.reset); - if (std.mem.indexOfScalar(u8, msg, '\n') == null) { - try writer.print(" {s}\n", .{msg}); - } else switch (multiline_errors) { - .indent => { - var it = std.mem.splitScalar(u8, msg, '\n'); - try writer.print(" {s}\n", .{it.first()}); - while (it.next()) |line| { - try writer.print(" {s}\n", .{line}); - } - }, - .newline => try writer.print("\n{s}\n", .{msg}), - .none => try writer.print(" {s}\n", .{msg}), - } - } - - if (error_style.verboseContext()) { - if (failing_step.result_failed_command) |cmd_str| { - try stderr.setColor(.red); - try writer.writeAll("failed command: "); - try stderr.setColor(.reset); - try writer.writeAll(cmd_str); - try writer.writeByte('\n'); - } - } - - try writer.writeByte('\n'); -} - -fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { - if (idx.* >= args.len) return null; - defer idx.* += 1; - return args[idx.*]; -} - -fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); - }; -} - -fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { - const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); - if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); - const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); - return arg; -} - -fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { - if (idx >= args.len) return null; - return args[idx..]; -} - -const Color = std.zig.Color; -const ErrorStyle = enum { - verbose, - minimal, - verbose_clear, - minimal_clear, - fn verboseContext(s: ErrorStyle) bool { - return switch (s) { - .verbose, .verbose_clear => true, - .minimal, .minimal_clear => false, - }; - } - fn clearOnUpdate(s: ErrorStyle) bool { - return switch (s) { - .verbose, .minimal => false, - .verbose_clear, .minimal_clear => true, - }; - } -}; -const MultilineErrors = enum { indent, newline, none }; -const Summary = enum { all, new, failures, line, none }; - -fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { - log.info("to access the help menu: zig build -h", .{}); - fatal(f, args); -} - -fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { - for (steps) |step_index| { - if (true) @panic("TODO"); - const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; - if (wf.mode != .tmp) continue; - const path = wf.generated_directory.path orelse continue; - Io.Dir.cwd().deleteTree(io, path) catch |err| { - log.warn("failed to delete {s}: {t}", .{ path, err }); - }; - } -} - -var stdio_buffer_allocation: [256]u8 = undefined; -var stdout_writer_allocation: Io.File.Writer = undefined; - -fn initStdoutWriter(io: Io) *Writer { - stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); - return &stdout_writer_allocation.interface; -} - -const ScannedConfig = struct { - configuration: Configuration, - top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), - - fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { - const c = &sc.configuration; - var serializer: std.zon.Serializer = .{ .writer = w }; - var s = try serializer.beginStruct(.{}); - - try s.field("default_step", @intFromEnum(c.default_step), .{}); - { - var ss = try s.beginStructField("top_level_steps", .{}); - for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| { - try ss.field(name, @intFromEnum(step), .{}); - } - try ss.end(); - } - - try s.end(); - } - - fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { - const arena = graph.arena; - const c = &sc.configuration; - for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| { - const step = step_index.ptr(c); - const decorated_name = if (step_index == c.default_step) - try fmt.allocPrint(arena, "{s} (default)", .{name}) - else - name; - const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); - const description = top_level.description.slice(c); - try w.print(" {s:<28} {s}\n", .{ decorated_name, description }); - } - } - - fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { - const arena = graph.arena; - - try w.print( - \\Usage: {s} build [steps] [options] - \\ - \\Steps: - \\ - , .{graph.zig_exe}); - try printSteps(sc, graph, w); - try w.writeAll( - \\ - \\Project-Specific Options: - \\ - ); - - const available_options = sc.configuration.available_options; - if (available_options.len == 0) { - try w.print(" (none)\n", .{}); - } else { - for (available_options) |option| { - const name = option.name.slice(&sc.configuration); - const description = option.description.slice(&sc.configuration); - const help = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type }); - try w.print("{s:<30} {s}\n", .{ help, description }); - if (option.enum_options.slice(&sc.configuration)) |enum_options| { - const padding: [33]u8 = @splat(' '); - try w.writeAll(padding ++ "Supported Values:\n"); - for (enum_options) |enum_option_index| { - const enum_option = enum_option_index.slice(&sc.configuration); - try w.print(padding ++ " {s}\n", .{enum_option}); - } - } - } - } - - try w.writeAll( - \\ - \\System Integration Options: - \\ --search-prefix [path] Add a path to look for binaries, libraries, headers - \\ --sysroot [path] Set the system root directory (usually /) - \\ --libc [file] Provide a file which specifies libc paths - \\ - \\ --system [pkgdir] Disable package fetching; enable all integrations - \\ -fsys=[name] Enable a system integration - \\ -fno-sys=[name] Disable a system integration - \\ - \\ -fdarling, -fno-darling Integration with system-installed Darling to - \\ execute macOS programs on Linux hosts - \\ (default: no) - \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute - \\ foreign-architecture programs on Linux hosts - \\ (default: no) - \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc - \\ (e.g. glibc or musl) built for multiple foreign - \\ architectures, allowing execution of non-native - \\ programs that link with libc. - \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on - \\ ARM64 macOS hosts. (default: no) - \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to - \\ execute WASI binaries. (default: no) - \\ -fwine, -fno-wine Integration with system-installed Wine to execute - \\ Windows programs on Linux hosts. (default: no) - \\ - \\ Available System Integrations: Enabled: - \\ - ); - if (sc.configuration.system_integrations.len == 0) { - try w.writeAll(" (none) -\n"); - } else { - for (sc.configuration.system_integrations) |system_integration| { - const name = system_integration.name.slice(&sc.configuration); - const status = switch (system_integration.status) { - .disabled => "no", - .enabled => "yes", - }; - try w.print(" {s:<43} {s}\n", .{ name, status }); - } - } - - try w.writeAll( - \\ - \\General Options: - \\ -h, --help Print this help to stdout and exit - \\ -l, --list-steps Print available steps to stdout and exit - \\ - \\ -p, --prefix [path] Where to install files (default: zig-out) - \\ --prefix-lib-dir [path] Where to install libraries - \\ --prefix-exe-dir [path] Where to install executables - \\ --prefix-include-dir [path] Where to install C header files - \\ --release[=mode] Request release mode, optionally specifying a - \\ preferred optimization mode: fast, safe, small - \\ - \\ --verbose Print commands before executing them - \\ --color [auto|off|on] Enable or disable colored error messages - \\ --error-style [style] Control how build errors are printed - \\ verbose (Default) Report errors with full context - \\ minimal Report errors after summary, excluding context like command lines - \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update - \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update - \\ --multiline-errors [style] Control how multi-line error messages are printed - \\ indent (Default) Indent non-initial lines to align with initial line - \\ newline Include a leading newline so that the error message is on its own lines - \\ none Print as usual so the first line is misaligned - \\ --summary [mode] Control the printing of the build summary - \\ all Print the build summary in its entirety - \\ new Omit cached steps - \\ failures (Default if short-lived) Only print failed steps - \\ line (Default if long-lived) Only print the single-line summary - \\ none Do not print the build summary - \\ -j Limit concurrent jobs (default is to use all CPU cores) - \\ --maxrss Limit memory usage (default is to use available memory) - \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss - \\ --test-timeout Limit execution time of unit tests, terminating if exceeded. - \\ The timeout must include a unit: ns, us, ms, s, m, h - \\ --watch Continuously rebuild when source files are modified - \\ --debounce Delay before rebuilding after changed file detected - \\ --webui[=ip] Enable the web interface on the given IP address - \\ --fuzz[=limit] Continuously search for unit test failures with an optional - \\ limit to the max number of iterations. The argument supports - \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies - \\ '--webui' when no limit is specified. - \\ --time-report Force full rebuild and provide detailed information on - \\ compilation time of Zig source code (implies '--webui') - \\ -fincremental Enable incremental compilation - \\ -fno-incremental Disable incremental compilation - \\ - \\Package Management Options: - \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit - \\ needed (Default) Lazy dependencies are fetched as needed - \\ all Lazy dependencies are always fetched - \\ --fork=[path] Override one or more projects from dependency tree - \\ - \\Advanced Options: - \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error - \\ -fno-reference-trace Disable reference trace - \\ -fallow-so-scripts Allows .so files to be GNU ld scripts - \\ -fno-allow-so-scripts (default) .so files must be ELF files - \\ --build-file [file] Override path to build.zig - \\ --cache-dir [path] Override path to local Zig cache directory - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --zig-lib-dir [arg] Override path to Zig lib directory - \\ --build-runner [file] Override path to build runner - \\ --seed [integer] For shuffling dependency traversal order (default: random) - \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries - \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) - \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) - \\ md5 16-byte cryptographic hash (ELF) - \\ uuid 16-byte random UUID (ELF, WASM) - \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) - \\ none (default) No build ID - \\ --debug-log [scope] Enable debugging the compiler - \\ --debug-pkg-config Fail if unknown pkg-config flags encountered - \\ --debug-rt Debug compiler runtime libraries - \\ --verbose-link Enable compiler debug output for linking - \\ --verbose-air Enable compiler debug output for Zig AIR - \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR - \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC - \\ --verbose-cimport Enable compiler debug output for C imports - \\ --verbose-cc Enable compiler debug output for C compilation - \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features - \\ - ); - } -}; diff --git a/lib/compiler/maker/Fuzz.zig b/lib/compiler/maker/Fuzz.zig deleted file mode 100644 index 433439f2ceef355c4663419c67ba51b7f4507b9b..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Fuzz.zig +++ /dev/null @@ -1,606 +0,0 @@ -const Fuzz = @This(); - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Build = std.Build; -const Cache = std.Build.Cache; -const Coverage = std.debug.Coverage; -const Configuration = std.Build.Configuration; -const Io = std.Io; -const abi = std.Build.abi.fuzz; -const assert = std.debug.assert; -const fatal = std.process.fatal; -const log = std.log; - -const maker = @import("../maker.zig"); -const WebServer = @import("WebServer.zig"); - -gpa: Allocator, -io: Io, -mode: Mode, - -/// Allocated into `gpa`. -run_steps: []const Configuration.Step.Index, - -group: Io.Group, -root_prog_node: std.Progress.Node, -prog_node: std.Progress.Node, - -/// Protects `coverage_files`. -coverage_mutex: Io.Mutex, -coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap), - -queue_mutex: Io.Mutex, -queue_cond: Io.Condition, -msg_queue: std.ArrayList(Msg), - -pub const Mode = union(enum) { - forever: struct { ws: *WebServer }, - limit: Limited, - - pub const Limited = struct { - amount: u64, - }; -}; - -const Msg = union(enum) { - coverage: struct { - id: u64, - cumulative: struct { - runs: u64, - unique: u64, - coverage: u64, - }, - run: Configuration.Step.Index, - }, - entry_point: struct { - coverage_id: u64, - addr: u64, - }, -}; - -const CoverageMap = struct { - mapped_memory: []align(std.heap.page_size_min) const u8, - coverage: Coverage, - source_locations: []Coverage.SourceLocation, - /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. - entry_points: std.ArrayList(u32), - start_timestamp: i64, - start_n_runs: u64, - - fn deinit(cm: *CoverageMap, gpa: Allocator) void { - std.posix.munmap(cm.mapped_memory); - cm.coverage.deinit(gpa); - cm.* = undefined; - } -}; - -pub fn init( - gpa: Allocator, - io: Io, - all_steps: []const Configuration.Step.Index, - root_prog_node: std.Progress.Node, - mode: Mode, -) error{ OutOfMemory, Canceled }!Fuzz { - const run_steps: []const Configuration.Step.Index = steps: { - var steps: std.ArrayList(Configuration.Step.Index) = .empty; - defer steps.deinit(gpa); - const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0); - defer rebuild_node.end(); - var rebuild_group: Io.Group = .init; - defer rebuild_group.cancel(io); - - for (all_steps) |step| { - if (true) @panic("TODO"); - const run = step.cast(std.Build.Step.Run) orelse continue; - if (run.producer == null) continue; - if (run.fuzz_tests.items.len == 0) continue; - try steps.append(gpa, run); - rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node }); - } - - if (steps.items.len == 0) fatal("no fuzz tests found", .{}); - rebuild_node.setEstimatedTotalItems(steps.items.len); - const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items); - try rebuild_group.await(io); - break :steps run_steps; - }; - errdefer gpa.free(run_steps); - - for (run_steps) |run_step_index| { - if (true) @panic("TODO"); - assert(run_step_index.fuzz_tests.items.len > 0); - if (run_step_index.rebuilt_executable == null) - fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); - } - - return .{ - .gpa = gpa, - .io = io, - .mode = mode, - .run_steps = run_steps, - .group = .init, - .root_prog_node = root_prog_node, - .prog_node = .none, - .coverage_files = .empty, - .coverage_mutex = .init, - .queue_mutex = .init, - .queue_cond = .init, - .msg_queue = .empty, - }; -} - -pub fn start(fuzz: *Fuzz) void { - const io = fuzz.io; - fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0); - - if (fuzz.mode == .forever) { - // For polling messages and sending updates to subscribers. - fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err| - fatal("unable to spawn coverage task: {t}", .{err}); - } - - if (true) @panic("TODO"); - - for (fuzz.run_steps) |run| { - assert(run.rebuilt_executable != null); - fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run }); - } -} - -pub fn deinit(fuzz: *Fuzz) void { - const io = fuzz.io; - fuzz.group.cancel(io); - fuzz.prog_node.end(); - fuzz.gpa.free(fuzz.run_steps); -} - -fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void { - rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { - const compile = run.producer.?; - log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); - }; -} - -fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { - const graph = run.step.owner.graph; - const io = graph.io; - const compile = run.producer.?; - const prog_node = parent_prog_node.start(compile.step.name, 0); - defer prog_node.end(); - - const result = compile.rebuildInFuzzMode(gpa, prog_node); - - const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; - const show_error_msgs = compile.step.result_error_msgs.items.len > 0; - const show_stderr = compile.step.result_stderr.len > 0; - - if (show_error_msgs or show_compile_errors or show_stderr) { - var buf: [256]u8 = undefined; - const stderr = try io.lockStderr(&buf, graph.stderr_mode); - defer io.unlockStderr(); - maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; - } - - const rebuilt_bin_path = result catch |err| switch (err) { - error.MakeFailed => return, - else => |other| return other, - }; - run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); -} - -fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { - const owner = run.step.owner; - const gpa = owner.allocator; - const graph = owner.graph; - const io = graph.io; - - run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) { - error.MakeFailed => { - var buf: [256]u8 = undefined; - const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) { - error.Canceled => return, - }; - defer io.unlockStderr(); - maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; - return; - }, - else => { - log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err }); - return; - }, - }; -} - -pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { - if (true) @panic("TODO"); - assert(fuzz.mode == .forever); - - var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); - var dedup_table: DedupTable = .empty; - defer dedup_table.deinit(fuzz.gpa); - - for (fuzz.run_steps) |run_step| { - const compile_inputs = run_step.producer.?.step.inputs.table; - for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { - try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len); - for (file_list.items) |sub_path| { - if (!std.mem.endsWith(u8, sub_path, ".zig")) continue; - const joined_path = try dir_path.join(arena, sub_path); - dedup_table.putAssumeCapacity(joined_path, {}); - } - } - } - - const deduped_paths = dedup_table.keys(); - const SortContext = struct { - pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { - _ = this; - return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { - .lt => true, - .gt => false, - .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), - }; - } - }; - std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); - return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths); -} - -pub const Previous = struct { - unique_runs: usize, - entry_points: usize, - sent_source_index: bool, - pub const init: Previous = .{ - .unique_runs = 0, - .entry_points = 0, - .sent_source_index = false, - }; -}; -pub fn sendUpdate( - fuzz: *Fuzz, - socket: *std.http.Server.WebSocket, - prev: *Previous, -) !void { - const io = fuzz.io; - - try fuzz.coverage_mutex.lock(io); - defer fuzz.coverage_mutex.unlock(io); - - const coverage_maps = fuzz.coverage_files.values(); - if (coverage_maps.len == 0) return; - // TODO: handle multiple fuzz steps in the WebSocket packets - const coverage_map = &coverage_maps[0]; - const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the - // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the - // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass - // this data straight to the socket with sendfile... - const seen_pcs = cov_header.seenBits(); - const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); - const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); - { - if (!prev.sent_source_index) { - prev.sent_source_index = true; - // We need to send initial context. - const header: abi.SourceIndexHeader = .{ - .directories_len = @intCast(coverage_map.coverage.directories.entries.len), - .files_len = @intCast(coverage_map.coverage.files.entries.len), - .source_locations_len = @intCast(coverage_map.source_locations.len), - .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), - .start_timestamp = coverage_map.start_timestamp, - .start_n_runs = coverage_map.start_n_runs, - }; - var iovecs: [5][]const u8 = .{ - @ptrCast(&header), - @ptrCast(coverage_map.coverage.directories.keys()), - @ptrCast(coverage_map.coverage.files.keys()), - @ptrCast(coverage_map.source_locations), - coverage_map.coverage.string_bytes.items, - }; - try socket.writeMessageVec(&iovecs, .binary); - } - - const header: abi.CoverageUpdateHeader = .{ - .n_runs = n_runs, - .unique_runs = unique_runs, - }; - var iovecs: [2][]const u8 = .{ - @ptrCast(&header), - @ptrCast(seen_pcs), - }; - try socket.writeMessageVec(&iovecs, .binary); - - prev.unique_runs = unique_runs; - } - - if (prev.entry_points != coverage_map.entry_points.items.len) { - const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len)); - var iovecs: [2][]const u8 = .{ - @ptrCast(&header), - @ptrCast(coverage_map.entry_points.items), - }; - try socket.writeMessageVec(&iovecs, .binary); - - prev.entry_points = coverage_map.entry_points.items.len; - } -} - -fn coverageRun(fuzz: *Fuzz) void { - coverageRunCancelable(fuzz) catch |err| switch (err) { - error.Canceled => return, - }; -} - -fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { - const io = fuzz.io; - - try fuzz.queue_mutex.lock(io); - defer fuzz.queue_mutex.unlock(io); - - while (true) { - try fuzz.queue_cond.wait(io, &fuzz.queue_mutex); - for (fuzz.msg_queue.items) |msg| switch (msg) { - .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) { - error.AlreadyReported => continue, - error.Canceled => return, - else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), - }, - .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) { - error.AlreadyReported => continue, - error.Canceled => return, - else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), - }, - }; - fuzz.msg_queue.clearRetainingCapacity(); - } -} -fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { - if (true) @panic("TODO"); - assert(fuzz.mode == .forever); - const ws = fuzz.mode.forever.ws; - const gpa = fuzz.gpa; - const io = fuzz.io; - - try fuzz.coverage_mutex.lock(io); - defer fuzz.coverage_mutex.unlock(io); - - const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id); - if (gop.found_existing) { - // We are fuzzing the same executable with multiple threads. - // Perhaps the same unit test; perhaps a different one. In any - // case, since the coverage file is the same, we only have to - // notice changes to that one file in order to learn coverage for - // this particular executable. - return; - } - errdefer _ = fuzz.coverage_files.pop(); - - gop.value_ptr.* = .{ - .coverage = std.debug.Coverage.init, - .mapped_memory = undefined, // populated below - .source_locations = undefined, // populated below - .entry_points = .empty, - .start_timestamp = ws.now(), - .start_n_runs = undefined, // populated below - }; - errdefer gop.value_ptr.coverage.deinit(gpa); - - const rebuilt_exe_path = run_step_index.rebuilt_executable.?; - const target = run_step_index.producer.?.rootModuleTarget(); - var debug_info = std.debug.Info.load( - gpa, - io, - rebuilt_exe_path, - &gop.value_ptr.coverage, - target.ofmt, - target.cpu.arch, - ) catch |err| { - log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ - run_step_index.step.name, rebuilt_exe_path, err, - }); - return error.AlreadyReported; - }; - defer debug_info.deinit(gpa); - - const coverage_file_path: Build.Cache.Path = .{ - .root_dir = run_step_index.step.owner.cache_root, - .sub_path = "v/" ++ std.fmt.hex(coverage_id), - }; - var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { - log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ - run_step_index.step.name, coverage_file_path, err, - }); - return error.AlreadyReported; - }; - defer coverage_file.close(io); - - const file_size = coverage_file.length(io) catch |err| { - log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err }); - return error.AlreadyReported; - }; - - const mapped_memory = std.posix.mmap( - null, - file_size, - .{ .READ = true }, - .{ .TYPE = .SHARED }, - coverage_file.handle, - 0, - ) catch |err| { - log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err }); - return error.AlreadyReported; - }; - gop.value_ptr.mapped_memory = mapped_memory; - - const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - const pcs = header.pcAddrs(); - const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len); - errdefer gpa.free(source_locations); - - // Unfortunately the PCs array that LLVM gives us from the 8-bit PC - // counters feature is not sorted. - var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .empty; - defer sorted_pcs.deinit(gpa); - try sorted_pcs.resize(gpa, pcs.len); - @memcpy(sorted_pcs.items(.pc), pcs); - for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i); - sorted_pcs.sortUnstable(struct { - addrs: []const u64, - - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return ctx.addrs[a_index] < ctx.addrs[b_index]; - } - }{ .addrs = sorted_pcs.items(.pc) }); - - debug_info.resolveAddresses(gpa, io, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| { - log.err("failed to resolve addresses to source locations: {t}", .{err}); - return error.AlreadyReported; - }; - - for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl; - gop.value_ptr.source_locations = source_locations; - gop.value_ptr.start_n_runs = header.n_runs; - - ws.notifyUpdate(); -} - -fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void { - const io = fuzz.io; - - try fuzz.coverage_mutex.lock(io); - defer fuzz.coverage_mutex.unlock(io); - - const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?; - const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - const pcs = header.pcAddrs(); - - // Since this pcs list is unsorted, we must linear scan for the best index. - const index = i: { - var best: usize = 0; - for (pcs[1..], 1..) |elem_addr, i| { - if (elem_addr == addr) break :i i; - if (elem_addr > addr) continue; - if (elem_addr > pcs[best]) best = i; - } - break :i best; - }; - if (index >= pcs.len) { - log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{ - addr, pcs[0], pcs[pcs.len - 1], - }); - return error.AlreadyReported; - } - if (false) { - const sl = coverage_map.source_locations[index]; - const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename); - if (pcs.len == 1) { - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{ - addr, file_name, sl.line, sl.column, - }); - } else if (index == 0) { - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{ - addr, file_name, sl.line, sl.column, pcs[index + 1], - }); - } else if (index == pcs.len - 1) { - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{ - addr, file_name, sl.line, sl.column, index, pcs[index - 1], - }); - } else { - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{ - addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1], - }); - } - } - try coverage_map.entry_points.append(fuzz.gpa, @intCast(index)); -} - -pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { - if (true) @panic("TODO"); - assert(fuzz.mode == .limit); - const io = fuzz.io; - - try fuzz.group.await(io); - fuzz.group = .init; - - std.debug.print("======= FUZZING REPORT =======\n", .{}); - for (fuzz.msg_queue.items) |msg| { - if (msg != .coverage) continue; - - const cov = msg.coverage; - const coverage_file_path: std.Build.Cache.Path = .{ - .root_dir = cov.run.step.owner.cache_root, - .sub_path = "v/" ++ std.fmt.hex(cov.id), - }; - var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { - fatal("step '{s}': failed to load coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, - }); - }; - defer coverage_file.close(io); - - const fuzz_abi = std.Build.abi.fuzz; - var rbuf: [0x1000]u8 = undefined; - var r = coverage_file.reader(io, &rbuf); - - var header: fuzz_abi.SeenPcsHeader = undefined; - r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { - fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, - }); - }; - - if (header.pcs_len == 0) { - fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{ - cov.run.step.name, coverage_file_path, - }); - } - - var seen_count: usize = 0; - const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len); - for (0..chunk_count) |_| { - const seen = r.interface.takeInt(usize, .little) catch |err| { - fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, - }); - }; - seen_count += @popCount(seen); - } - - const seen_f: f64 = @floatFromInt(seen_count); - const total_f: f64 = @floatFromInt(header.pcs_len); - const ratio = seen_f / total_f; - std.debug.print( - \\Step: {s} - \\Fuzz test: "{s}" ({x}) - \\Runs: {} -> {} - \\Unique runs: {} -> {} - \\Coverage: {}/{} -> {}/{} ({:.02}%) - \\ - , .{ - cov.run.step.name, - cov.run.fuzz_tests.items[0], - cov.id, - cov.cumulative.runs, - header.n_runs, - cov.cumulative.unique, - header.unique_runs, - cov.cumulative.coverage, - header.pcs_len, - seen_count, - header.pcs_len, - ratio * 100, - }); - - std.debug.print("------------------------------\n", .{}); - } - std.debug.print( - \\Values are accumulated across multiple runs when preserving the cache. - \\============================== - \\ - , .{}); -} diff --git a/lib/compiler/maker/Graph.zig b/lib/compiler/maker/Graph.zig deleted file mode 100644 index ba9cad0ee17b713cd1731aff04e22a132d94afdd..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Graph.zig +++ /dev/null @@ -1,25 +0,0 @@ -//! Shared maker state among all steps. -const Graph = @This(); - -const std = @import("std"); -const Io = std.Io; -const Allocator = std.mem.Allocator; -const Configuration = std.Build.Configuration; - -io: Io, -/// Process lifetime. -arena: Allocator, -cache: std.Build.Cache, -zig_exe: []const u8, -environ_map: std.process.Environ.Map, -global_cache_root: std.Build.Cache.Directory, -zig_lib_directory: std.Build.Cache.Directory, - -debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, -incremental: ?bool = null, -random_seed: u32 = 0, -allow_so_scripts: ?bool = null, -time_report: bool = false, -/// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also -/// respects the '--color' flag. -stderr_mode: ?Io.Terminal.Mode = null, diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/maker/Step.zig deleted file mode 100644 index 845bc1e1f8ecb4191d945a5c3f86b390adc7290e..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Step.zig +++ /dev/null @@ -1,840 +0,0 @@ -//! The state that maker needs in order to process a step. -const Step = @This(); - -const builtin = @import("builtin"); - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const Io = std.Io; -const LazyPath = std.Build.Configuration.LazyPath; -const Package = std.Build.Configuration.Package; -const Path = std.Build.Cache.Path; -const Configuration = std.Build.Configuration; -const assert = std.debug.assert; - -const WebServer = @import("WebServer.zig"); - -pub const Compile = void; // @import("Step/Compile.zig"); -pub const Run = void; // @import("Step/Run.zig"); - -/// Avoid false sharing. -_: void align(std.atomic.cache_line) = {}, - -state: State = .precheck_unstarted, -dependants: std.ArrayList(Configuration.Step.Index) = .empty, -/// Collects the set of files that retrigger this step to run. -/// -/// This is used by the build system's implementation of `--watch` but it can -/// also be potentially useful for IDEs to know what effects editing a -/// particular file has. -/// -/// Populated within `make`. Implementation may choose to clear and repopulate, -/// retain previous value, or update. -inputs: Inputs = .init, -pending_deps: u32 = undefined, - -result_error_msgs: std.ArrayList([]const u8) = .empty, -result_error_bundle: std.zig.ErrorBundle = .empty, -result_stderr: []const u8 = "", -result_cached: bool = false, -result_duration_ns: ?u64 = null, -/// 0 means unavailable or not reported. -result_peak_rss: usize = 0, -/// If the step is failed and this field is populated, this is the command which failed. -/// This field may be populated even if the step succeeded. -result_failed_command: ?[]const u8 = null, -test_results: TestResults = .{}, - -pub const State = enum { - precheck_unstarted, - precheck_started, - /// This is also used to indicate "dirty" steps that have been modified - /// after a previous build completed, in which case, the step may or may - /// not have been completed before. Either way, one or more of its direct - /// file system inputs have been modified, meaning that the step needs to - /// be re-evaluated. - precheck_done, - dependency_failure, - success, - failure, - /// This state indicates that the step did not complete, however, it also did not fail, - /// and it is safe to continue executing its dependencies. - skipped, - /// This step was skipped because it specified a max_rss that exceeded the runner's maximum. - /// It is not safe to run its dependencies. - skipped_oom, -}; - -pub const Inputs = struct { - table: Table, - - pub const init: Inputs = .{ - .table = .{}, - }; - - pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, Files, Cache.Path.TableAdapter, false); - /// The special file name "." means any changes inside the directory. - pub const Files = std.ArrayList([]const u8); - - pub fn populated(inputs: *Inputs) bool { - return inputs.table.count() != 0; - } - - pub fn clear(inputs: *Inputs, gpa: Allocator) void { - for (inputs.table.values()) |*files| files.deinit(gpa); - inputs.table.clearRetainingCapacity(); - } -}; - -pub const TestResults = struct { - /// The total number of tests in the step. Every test has a "status" from the following: - /// * passed - /// * skipped - /// * failed cleanly - /// * crashed - /// * timed out - test_count: u32 = 0, - - /// The number of tests which were skipped (`error.SkipZigTest`). - skip_count: u32 = 0, - /// The number of tests which failed cleanly. - fail_count: u32 = 0, - /// The number of tests which terminated unexpectedly, i.e. crashed. - crash_count: u32 = 0, - /// The number of tests which timed out. - timeout_count: u32 = 0, - - /// The number of detected memory leaks. The associated test may still have passed; indeed, *all* - /// individual tests may have passed. However, the step as a whole fails if any test has leaks. - leak_count: u32 = 0, - /// The number of detected error logs. The associated test may still have passed; indeed, *all* - /// individual tests may have passed. However, the step as a whole fails if any test logs errors. - log_err_count: u32 = 0, - - pub fn isSuccess(tr: TestResults) bool { - // all steps are success or skip - return tr.fail_count == 0 and - tr.crash_count == 0 and - tr.timeout_count == 0 and - // no (otherwise successful) step leaked memory or logged errors - tr.leak_count == 0 and - tr.log_err_count == 0; - } - - /// Computes the number of tests which passed from the other values. - pub fn passCount(tr: TestResults) u32 { - return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count; - } -}; - -pub const MakeOptions = struct { - progress_node: std.Progress.Node, - watch: bool, - web_server: ?*WebServer, - /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. - unit_test_timeout_ns: ?u64, - /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. - gpa: Allocator, -}; - -pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; - -/// If the Step's `make` function reports `error.MakeFailed`, it indicates they -/// have already reported the error. Otherwise, we add a simple error report -/// here. -pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { - if (true) @panic("TODO Step.make"); - const arena = s.owner.allocator; - const graph = s.owner.graph; - const io = graph.io; - - var start_ts: ?Io.Timestamp = t: { - if (!graph.time_report) break :t null; - if (s.id == .compile) break :t null; - if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; - break :t Io.Clock.awake.now(io); - }; - const make_result = s.makeFn(s, options); - if (start_ts) |*ts| { - const duration = ts.untilNow(io, .awake); - options.web_server.?.updateTimeReportGeneric(s, duration); - } - - make_result catch |err| switch (err) { - error.MakeFailed, error.MakeSkipped => |e| return e, - else => { - s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); - return error.MakeFailed; - }, - }; - - if (!s.test_results.isSuccess()) { - return error.MakeFailed; - } - - if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { - const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ - s.result_peak_rss, s.max_rss, - }) catch @panic("OOM"); - s.result_error_msgs.append(arena, msg) catch @panic("OOM"); - } -} - -/// Implementation detail of file watching. Prepares the step for being re-evaluated. -/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. -pub fn invalidateResult(step: *Step, gpa: Allocator) bool { - if (true) @panic("TODO Step.invalidateResult"); - if (step.state == .precheck_done) return false; - assert(step.pending_deps == 0); - step.state = .precheck_done; - step.reset(gpa); - for (step.dependants.items) |dependant| { - _ = dependant.invalidateResult(gpa); - dependant.pending_deps += 1; - } - return true; -} - -/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. -pub fn reset(step: *Step, gpa: Allocator) void { - assert(step.state == .precheck_done); - - if (step.result_failed_command) |cmd| gpa.free(cmd); - - step.result_error_msgs.clearRetainingCapacity(); - step.result_stderr = ""; - step.result_cached = false; - step.result_duration_ns = null; - step.result_peak_rss = 0; - step.result_failed_command = null; - step.test_results = .{}; - step.clearWatchInputs(); - - step.result_error_bundle.deinit(gpa); - step.result_error_bundle = std.zig.ErrorBundle.empty; -} - -/// Populates `s.result_failed_command`. -pub fn captureChildProcess( - s: *Step, - gpa: Allocator, - progress_node: std.Progress.Node, - argv: []const []const u8, -) !std.process.RunResult { - const graph = s.owner.graph; - const arena = graph.arena; - const io = graph.io; - - // If an error occurs, it's happened in this command: - assert(s.result_failed_command == null); - s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); - - try handleChildProcUnsupported(s); - try handleVerbose(s, .inherit, argv); - - const result = std.process.run(arena, io, .{ - .argv = argv, - .environ_map = &graph.environ_map, - .progress_node = progress_node, - }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); - - if (result.stderr.len > 0) { - try s.result_error_msgs.append(arena, result.stderr); - } - - return result; -} - -pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { - try step.addError(fmt, args); - return error.MakeFailed; -} - -pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { - const arena = step.owner.allocator; - const msg = try std.fmt.allocPrint(arena, fmt, args); - try step.result_error_msgs.append(arena, msg); -} - -pub const ZigProcess = struct { - child: std.process.Child, - multi_reader_buffer: Io.File.MultiReader.Buffer(2), - multi_reader: Io.File.MultiReader, - progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn, - - pub const StreamEnum = enum { stdout, stderr }; - - pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void { - zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null; - } - - pub fn deinit(zp: *ZigProcess, io: Io) void { - zp.child.kill(io); - zp.multi_reader.deinit(); - zp.* = undefined; - } -}; - -/// Assumes that argv contains `--listen=-` and that the process being spawned -/// is the zig compiler - the same version that compiled the build runner. -/// Populates `s.result_failed_command`. -pub fn evalZigProcess( - s: *Step, - argv: []const []const u8, - prog_node: std.Progress.Node, - watch: bool, - web_server: ?*WebServer, - gpa: Allocator, -) !?Cache.Path { - const b = s.owner; - const io = b.graph.io; - - // If an error occurs, it's happened in this command: - assert(s.result_failed_command == null); - s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); - - if (s.getZigProcess()) |zp| update: { - assert(watch); - if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index); - zp.progress_ipc_index = null; - var exited = false; - defer if (exited) { - s.cast(Compile).?.zig_process = null; - zp.deinit(io); - gpa.destroy(zp); - } else zp.saveState(prog_node); - const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { - error.BrokenPipe, error.EndOfStream => |reason| { - std.log.info("{s} restart required: {t}", .{ argv[0], reason }); - // Process restart required. - const term = zp.child.wait(io) catch |e| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); - }; - _ = term; - exited = true; - break :update; - }, - else => |e| return e, - }; - - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); - } - - if (s.result_error_msgs.items.len > 0 and result == null) { - // Crash detected. - const term = zp.child.wait(io) catch |e| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); - }; - s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; - exited = true; - try handleChildProcessTerm(s, term); - return error.MakeFailed; - } - - return result; - } - assert(argv.len != 0); - - try handleChildProcUnsupported(s); - try handleVerbose(s, .inherit, argv); - - const zp = try gpa.create(ZigProcess); - defer if (!watch) gpa.destroy(zp); - - zp.child = std.process.spawn(io, .{ - .argv = argv, - .environ_map = &b.graph.environ_map, - .stdin = .pipe, - .stdout = .pipe, - .stderr = .pipe, - .request_resource_usage_statistics = true, - .progress_node = prog_node, - }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); - - zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ - zp.child.stdout.?, zp.child.stderr.?, - }); - if (watch) s.cast(Compile).?.zig_process = zp; - defer if (!watch) zp.deinit(io); - - const result = result: { - defer if (watch) zp.saveState(prog_node); - break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); - }; - - if (!watch) { - // Send EOF to stdin. - zp.child.stdin.?.close(io); - zp.child.stdin = null; - - const term = zp.child.wait(io) catch |err| { - return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); - }; - s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; - - // Special handling for Compile step that is expecting compile errors. - if (s.cast(Compile)) |compile| switch (term) { - .exited => { - // Note that the exit code may be 0 in this case due to the - // compiler server protocol. - if (compile.expect_errors != null) { - return error.NeedCompileErrorCheck; - } - }, - else => {}, - }; - - try handleChildProcessTerm(s, term); - } - - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); - } - - return result; -} - -/// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. -pub fn installFile(s: *Step, src_lazy_path: LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { - const b = s.owner; - const io = b.graph.io; - const src_path = src_lazy_path.getPath3(b, s); - try handleVerbose(s, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); - return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| - return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); -} - -/// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. -pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { - const b = s.owner; - const io = b.graph.io; - try handleVerbose(s, .inherit, &.{ "install", "-d", dest_path }); - return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| - return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); -} - -fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebServer, gpa: Allocator) !?Path { - const b = s.owner; - const arena = b.allocator; - const io = b.graph.io; - - const start_ts = Io.Clock.awake.now(io); - - try sendMessage(io, zp.child.stdin.?, .update); - if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); - - var result: ?Path = null; - var eos_err: error{EndOfStream}!void = {}; - - const stdout = zp.multi_reader.fileReader(0); - - while (true) { - const Header = std.zig.Server.Message.Header; - const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) { - error.EndOfStream => break, - error.ReadFailed => return stdout.err.?, - }; - const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) { - error.EndOfStream => |e| { - // Better to report the crash with stderr below, but we set - // this in case the child exits successfully while violating - // this protocol. - eos_err = e; - break; - }, - error.ReadFailed => return stdout.err.?, - }; - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) { - return s.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - } - }, - .error_bundle => { - s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); - // This message indicates the end of the update. - if (watch) break; - }, - .emit_digest => { - const EmitDigest = std.zig.Server.Message.EmitDigest; - const emit_digest: *align(1) const EmitDigest = @ptrCast(body); - s.result_cached = emit_digest.flags.cache_hit; - const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; - result = .{ - .root_dir = b.cache_root, - .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), - }; - }, - .file_system_inputs => { - s.clearWatchInputs(); - var it = std.mem.splitScalar(u8, body, 0); - while (it.next()) |prefixed_path| { - const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); - const sub_path = try arena.dupe(u8, prefixed_path[1..]); - const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; - switch (prefix_index) { - .cwd => { - const path: Cache.Path = .{ - .root_dir = Cache.Directory.cwd(), - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .zig_lib => zl: { - if (s.cast(Step.Compile)) |compile| { - if (compile.zig_lib_dir) |zig_lib_dir| { - const lp = try zig_lib_dir.join(arena, sub_path); - try addWatchInput(s, lp); - break :zl; - } - } - const path: Cache.Path = .{ - .root_dir = s.owner.graph.zig_lib_directory, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .local_cache => { - const path: Cache.Path = .{ - .root_dir = b.cache_root, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - .global_cache => { - const path: Cache.Path = .{ - .root_dir = s.owner.graph.global_cache_root, - .sub_path = sub_path_dirname, - }; - try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); - }, - } - } - }, - .time_report => if (web_server) |ws| { - const TimeReport = std.zig.Server.Message.TimeReport; - const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); - ws.updateTimeReportCompile(.{ - .compile = s.cast(Step.Compile).?, - .use_llvm = tr.flags.use_llvm, - .stats = tr.stats, - .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), - .llvm_pass_timings_len = tr.llvm_pass_timings_len, - .files_len = tr.files_len, - .decls_len = tr.decls_len, - .trailing = body[@sizeOf(TimeReport)..], - }); - }, - else => {}, // ignore other messages - } - } - - s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()); - - const stderr_contents = zp.multi_reader.reader(1).buffered(); - if (stderr_contents.len > 0) { - try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); - } - - try eos_err; - - return result; -} - -pub fn getZigProcess(s: *Step) ?*ZigProcess { - if (true) @panic("TODO getZigProcess"); - return switch (s.id) { - .compile => s.cast(Compile).?.zig_process, - else => null, - }; -} - -fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 0, - }; - var w = file.writer(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -pub fn handleVerbose( - s: *Step, - arena: Allocator, - cwd: std.process.Child.Cwd, - opt_env: ?*const std.process.Environ.Map, - argv: []const []const u8, -) error{OutOfMemory}!void { - if (!s.verbose) return; - const graph = s.graph; - // Intention of verbose is to print all sub-process command lines to - // stderr before spawning them. - const text = try allocPrintCmd(arena, cwd, if (opt_env) |env| .{ - .child = env, - .parent = &graph.environ_map, - } else null, argv); - std.log.scoped(.verbose).info("{s}", .{text}); -} - -/// Asserts that the caller has already populated `s.result_failed_command`. -pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { - if (!std.process.can_spawn) { - return s.fail("unable to spawn process: host cannot spawn child processes", .{}); - } -} - -/// Asserts that the caller has already populated `s.result_failed_command`. -pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { - assert(s.result_failed_command != null); - return switch (term) { - .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), - .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), - .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), - .unknown => s.fail("process terminated unexpectedly", .{}), - }; -} - -/// Prefer `cacheHitAndWatch` unless you already added watch inputs -/// separately from using the cache system. -pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool { - s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); - return s.result_cached; -} - -/// Clears previous watch inputs, if any, and then populates watch inputs from -/// the full set of files picked up by the cache manifest. -/// -/// Must be accompanied with `writeManifestAndWatch`. -pub fn cacheHitAndWatch(s: *Step, man: *Cache.Manifest) !bool { - const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); - s.result_cached = is_hit; - // The above call to hit() populates the manifest with files, so in case of - // a hit, we need to populate watch inputs. - if (is_hit) try setWatchInputsFromManifest(s, man); - return is_hit; -} - -fn failWithCacheError( - s: *Step, - man: *const Cache.Manifest, - err: Cache.Manifest.HitError, -) error{ OutOfMemory, Canceled, MakeFailed } { - switch (err) { - error.CacheCheckFailed => switch (man.diagnostic) { - .none => unreachable, - .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ - man.diagnostic, e, - }), - .file_open, .file_stat, .file_read, .file_hash => |op| { - const pp = man.files.keys()[op.file_index].prefixed_path; - const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ - prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, - }); - }, - }, - error.OutOfMemory, error.Canceled => |e| return e, - error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), - } -} - -/// Prefer `writeManifestAndWatch` unless you already added watch inputs -/// separately from using the cache system. -pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { - if (s.test_results.isSuccess()) { - man.writeManifest() catch |err| { - try s.addError("unable to write cache manifest: {t}", .{err}); - }; - } -} - -/// Clears previous watch inputs, if any, and then populates watch inputs from -/// the full set of files picked up by the cache manifest. -/// -/// Must be accompanied with `cacheHitAndWatch`. -pub fn writeManifestAndWatch(s: *Step, man: *Cache.Manifest) !void { - try writeManifest(s, man); - try setWatchInputsFromManifest(s, man); -} - -fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void { - const arena = s.owner.allocator; - const prefixes = man.cache.prefixes(); - clearWatchInputs(s); - for (man.files.keys()) |file| { - // The file path data is freed when the cache manifest is cleaned up at the end of `make`. - const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); - try addWatchInputFromPath(s, .{ - .root_dir = prefixes[file.prefixed_path.prefix], - .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, std.fs.path.basename(sub_path)); - } -} - -/// For steps that have a single input that never changes when re-running `make`. -pub fn singleUnchangingWatchInput(step: *Step, lazy_path: LazyPath) Allocator.Error!void { - if (!step.inputs.populated()) try step.addWatchInput(lazy_path); -} - -pub fn clearWatchInputs(step: *Step) void { - const gpa = step.owner.allocator; - step.inputs.clear(gpa); -} - -/// Places a *file* dependency on the path. -pub fn addWatchInput(step: *Step, lazy_file: LazyPath) Allocator.Error!void { - switch (lazy_file) { - .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), - .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), - .cwd_relative => |path_string| { - try addWatchInputFromPath(step, .{ - .root_dir = .{ - .path = null, - .handle = Io.Dir.cwd(), - }, - .sub_path = std.fs.path.dirname(path_string) orelse "", - }, std.fs.path.basename(path_string)); - }, - // Nothing to watch because this dependency edge is modeled instead via `dependants`. - .generated => {}, - } -} - -/// Any changes inside the directory will trigger invalidation. -/// -/// See also `addDirectoryWatchInputFromPath` which takes a `Cache.Path` instead. -/// -/// Paths derived from this directory should also be manually added via -/// `addDirectoryWatchInputFromPath` if and only if this function returns -/// `true`. -pub fn addDirectoryWatchInput(step: *Step, lazy_directory: LazyPath) Allocator.Error!bool { - switch (lazy_directory) { - .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), - .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), - .cwd_relative => |path_string| { - try addDirectoryWatchInputFromPath(step, .{ - .root_dir = .{ - .path = null, - .handle = Io.Dir.cwd(), - }, - .sub_path = path_string, - }); - }, - // Nothing to watch because this dependency edge is modeled instead via `dependants`. - .generated => return false, - } - return true; -} - -/// Any changes inside the directory will trigger invalidation. -/// -/// See also `addDirectoryWatchInput` which takes a `LazyPath` instead. -/// -/// This function should only be called when it has been verified that the -/// dependency on `path` is not already accounted for by a `Step` dependency. -/// In other words, before calling this function, first check that the -/// `LazyPath` which this `path` is derived from is not `generated`. -pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void { - return addWatchInputFromPath(step, path, "."); -} - -fn addWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { - return addWatchInputFromPath(step, .{ - .root_dir = package.build_root, - .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, std.fs.path.basename(sub_path)); -} - -fn addDirectoryWatchInputFromBuilder(step: *Step, package: Package, sub_path: []const u8) !void { - return addDirectoryWatchInputFromPath(step, .{ - .root_dir = package.build_root, - .sub_path = sub_path, - }); -} - -fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !void { - const gpa = step.owner.allocator; - const gop = try step.inputs.table.getOrPut(gpa, path); - if (!gop.found_existing) gop.value_ptr.* = .empty; - try gop.value_ptr.append(gpa, basename); -} - -pub fn allocPrintCmd( - gpa: Allocator, - cwd: std.process.Child.Cwd, - opt_env: ?struct { - child: *const std.process.Environ.Map, - parent: *const std.process.Environ.Map, - }, - argv: []const []const u8, -) Allocator.Error![]u8 { - const shell = struct { - fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { - for (string) |c| { - if (switch (c) { - else => true, - '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, - '=' => is_argv0, - }) break; - } else return writer.writeAll(string); - - try writer.writeByte('"'); - for (string) |c| { - if (switch (c) { - std.ascii.control_code.nul => break, - '!', '"', '$', '\\', '`' => true, - else => !std.ascii.isPrint(c), - }) try writer.writeByte('\\'); - switch (c) { - std.ascii.control_code.nul => unreachable, - std.ascii.control_code.bel => try writer.writeByte('a'), - std.ascii.control_code.bs => try writer.writeByte('b'), - std.ascii.control_code.ht => try writer.writeByte('t'), - std.ascii.control_code.lf => try writer.writeByte('n'), - std.ascii.control_code.vt => try writer.writeByte('v'), - std.ascii.control_code.ff => try writer.writeByte('f'), - std.ascii.control_code.cr => try writer.writeByte('r'), - std.ascii.control_code.esc => try writer.writeByte('E'), - ' '...'~' => try writer.writeByte(c), - else => try writer.print("{o:0>3}", .{c}), - } - } - try writer.writeByte('"'); - } - }; - - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - const writer = &aw.writer; - switch (cwd) { - .inherit => {}, - .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, - .dir => @panic("TODO"), - } - if (opt_env) |env| { - var it = env.child.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; - if (env.parent.get(key)) |process_value| { - if (std.mem.eql(u8, value, process_value)) continue; - } - writer.print("{s}=", .{key}) catch return error.OutOfMemory; - shell.escape(writer, value, false) catch return error.OutOfMemory; - writer.writeByte(' ') catch return error.OutOfMemory; - } - } - shell.escape(writer, argv[0], true) catch return error.OutOfMemory; - for (argv[1..]) |arg| { - writer.writeByte(' ') catch return error.OutOfMemory; - shell.escape(writer, arg, false) catch return error.OutOfMemory; - } - return aw.toOwnedSlice(); -} diff --git a/lib/compiler/maker/Step/Compile.zig b/lib/compiler/maker/Step/Compile.zig deleted file mode 100644 index 5ffbc23cfc15d6d6de5df4f4585585364052da6d..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Step/Compile.zig +++ /dev/null @@ -1,1200 +0,0 @@ -/// Populated during the make phase when there is a long-lived compiler process. -/// Managed by the build runner, not user build script. -zig_process: ?*Step.ZigProcess, - -fn make(step: *Step, options: Step.MakeOptions) !void { - const b = step.owner; - const compile: *Compile = @fieldParentPtr("step", step); - - const zig_args = try getZigArgs(compile, false); - - const maybe_output_dir = step.evalZigProcess( - zig_args, - options.progress_node, - (b.graph.incremental == true) and (options.watch or options.web_server != null), - options.web_server, - options.gpa, - ) catch |err| switch (err) { - error.NeedCompileErrorCheck => { - assert(compile.expect_errors != null); - try checkCompileErrors(compile); - return; - }, - else => |e| return e, - }; - - // Update generated files - if (maybe_output_dir) |output_dir| { - if (compile.emit_directory) |lp| { - lp.path = b.fmt("{f}", .{output_dir}); - } - - // zig fmt: off - if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); - if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); - // hack for stage2_x86_64 + coff - if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); - if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); - if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); - if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); - if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); - if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); - if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); - // zig fmt: on - } - - if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and - compile.version != null and compile.generated_bin != null and - std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) - { - try doAtomicSymLinks( - step, - compile.getEmittedBin().getPath2(b, step), - compile.major_only_filename.?, - compile.name_only_filename.?, - ); - } -} - -fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { - const step = &compile.step; - const b = step.owner; - const arena = b.allocator; - - var zig_args = std.array_list.Managed([]const u8).init(arena); - defer zig_args.deinit(); - - try zig_args.append(b.graph.zig_exe); - - const cmd = switch (compile.kind) { - .lib => "build-lib", - .exe => "build-exe", - .obj => "build-obj", - .@"test" => "test", - .test_obj => "test-obj", - }; - try zig_args.append(cmd); - - if (b.reference_trace) |some| { - try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); - } - try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); - - try addFlag(&zig_args, "llvm", compile.use_llvm); - try addFlag(&zig_args, "lld", compile.use_lld); - try addFlag(&zig_args, "new-linker", compile.use_new_linker); - - if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { - try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); - } - - switch (compile.entry) { - .default => {}, - .disabled => try zig_args.append("-fno-entry"), - .enabled => try zig_args.append("-fentry"), - .symbol_name => |entry_name| { - try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); - }, - } - - { - for (compile.force_undefined_symbols.keys()) |symbol_name| { - try zig_args.append("--force_undefined"); - try zig_args.append(symbol_name.*); - } - } - - if (compile.stack_size) |stack_size| { - try zig_args.append("--stack"); - try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); - } - - if (fuzz) { - try zig_args.append("-ffuzz"); - } - - { - // Stores system libraries that have already been seen for at least one - // module, along with any arguments that need to be passed to the - // compiler for each module individually. - var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; - var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; - - var prev_has_cflags = false; - var prev_has_rcflags = false; - var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; - var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; - // Track the number of positional arguments so that a nice error can be - // emitted if there is nothing to link. - var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); - - // Fully recursive iteration including dynamic libraries to detect - // libc and libc++ linkage. - for (compile.getCompileDependencies(true)) |some_compile| { - for (some_compile.root_module.getGraph().modules) |mod| { - if (mod.link_libc == true) compile.is_linking_libc = true; - if (mod.link_libcpp == true) compile.is_linking_libcpp = true; - } - } - - var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); - - // For this loop, don't chase dynamic libraries because their link - // objects are already linked. - for (compile.getCompileDependencies(false)) |dep_compile| { - for (dep_compile.root_module.getGraph().modules) |mod| { - // While walking transitive dependencies, if a given link object is - // already included in a library, it should not redundantly be - // placed on the linker line of the dependee. - const my_responsibility = dep_compile == compile; - const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); - - // Inherit dependencies on darwin frameworks. - if (!already_linked) { - for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { - try frameworks.put(arena, name, info); - } - } - - // Inherit dependencies on system libraries and static libraries. - for (mod.link_objects.items) |link_object| { - switch (link_object) { - .static_path => |static_path| { - if (my_responsibility) { - try zig_args.append(static_path.getPath2(mod.owner, step)); - total_linker_objects += 1; - } - }, - .system_lib => |system_lib| { - const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); - if (system_lib_gop.found_existing) { - try zig_args.appendSlice(system_lib_gop.value_ptr.*); - continue; - } else { - system_lib_gop.value_ptr.* = &.{}; - } - - if (already_linked) - continue; - - if ((system_lib.search_strategy != prev_search_strategy or - system_lib.preferred_link_mode != prev_preferred_link_mode) and - compile.linkage != .static) - { - switch (system_lib.search_strategy) { - .no_fallback => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_dylibs_only"), - .static => try zig_args.append("-search_static_only"), - }, - .paths_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_paths_first"), - .static => try zig_args.append("-search_paths_first_static"), - }, - .mode_first => switch (system_lib.preferred_link_mode) { - .dynamic => try zig_args.append("-search_dylibs_first"), - .static => try zig_args.append("-search_static_first"), - }, - } - prev_search_strategy = system_lib.search_strategy; - prev_preferred_link_mode = system_lib.preferred_link_mode; - } - - const prefix: []const u8 = prefix: { - if (system_lib.needed) break :prefix "-needed-l"; - if (system_lib.weak) break :prefix "-weak-l"; - break :prefix "-l"; - }; - switch (system_lib.use_pkg_config) { - .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), - .yes, .force => { - if (compile.runPkgConfig(system_lib.name)) |result| { - try zig_args.appendSlice(result.cflags); - try zig_args.appendSlice(result.libs); - try seen_system_libs.put(arena, system_lib.name, result.cflags); - } else |err| switch (err) { - error.PkgConfigInvalidOutput, - error.PkgConfigCrashed, - error.PkgConfigFailed, - error.PkgConfigNotInstalled, - error.PackageNotFound, - => switch (system_lib.use_pkg_config) { - .yes => { - // pkg-config failed, so fall back to linking the library - // by name directly. - try zig_args.append(b.fmt("{s}{s}", .{ - prefix, - system_lib.name, - })); - }, - .force => { - panic("pkg-config failed for library {s}", .{system_lib.name}); - }, - .no => unreachable, - }, - - else => |e| return e, - } - }, - } - }, - .other_step => |other| { - switch (other.kind) { - .exe => return step.fail("cannot link with an executable build artifact", .{}), - .@"test" => return step.fail("cannot link with a test", .{}), - .obj, .test_obj => { - const included_in_lib_or_obj = !my_responsibility and - (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); - if (!already_linked and !included_in_lib_or_obj) { - try zig_args.append(other.getEmittedBin().getPath2(b, step)); - total_linker_objects += 1; - } - }, - .lib => l: { - const other_produces_implib = other.producesImplib(); - const other_is_static = other_produces_implib or other.isStaticLibrary(); - - if (compile.isStaticLibrary() and other_is_static) { - // Avoid putting a static library inside a static library. - break :l; - } - - // For DLLs, we must link against the implib. - // For everything else, we directly link - // against the library file. - const full_path_lib = if (other_produces_implib) - try other.getGeneratedFilePath("generated_implib", &compile.step) - else - try other.getGeneratedFilePath("generated_bin", &compile.step); - - try zig_args.append(full_path_lib); - total_linker_objects += 1; - - if (other.linkage == .dynamic and - compile.rootModuleTarget().os.tag != .windows) - { - if (fs.path.dirname(full_path_lib)) |dirname| { - try zig_args.append("-rpath"); - try zig_args.append(dirname); - } - } - }, - } - }, - .assembly_file => |asm_file| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags) { - try zig_args.append("-cflags"); - try zig_args.append("--"); - prev_has_cflags = false; - } - try zig_args.append(asm_file.getPath2(mod.owner, step)); - total_linker_objects += 1; - }, - - .c_source_file => |c_source_file| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags or c_source_file.flags.len != 0) { - try zig_args.append("-cflags"); - for (c_source_file.flags) |arg| { - try zig_args.append(arg); - } - try zig_args.append("--"); - } - prev_has_cflags = (c_source_file.flags.len != 0); - - if (c_source_file.language) |lang| { - try zig_args.append("-x"); - try zig_args.append(lang.internalIdentifier()); - } - - try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); - - if (c_source_file.language != null) { - try zig_args.append("-x"); - try zig_args.append("none"); - } - total_linker_objects += 1; - }, - - .c_source_files => |c_source_files| l: { - if (!my_responsibility) break :l; - - if (prev_has_cflags or c_source_files.flags.len != 0) { - try zig_args.append("-cflags"); - for (c_source_files.flags) |arg| { - try zig_args.append(arg); - } - try zig_args.append("--"); - } - prev_has_cflags = (c_source_files.flags.len != 0); - - if (c_source_files.language) |lang| { - try zig_args.append("-x"); - try zig_args.append(lang.internalIdentifier()); - } - - const root_path = c_source_files.root.getPath2(mod.owner, step); - for (c_source_files.files) |file| { - try zig_args.append(b.pathJoin(&.{ root_path, file })); - } - - if (c_source_files.language != null) { - try zig_args.append("-x"); - try zig_args.append("none"); - } - - total_linker_objects += c_source_files.files.len; - }, - - .win32_resource_file => |rc_source_file| l: { - if (!my_responsibility) break :l; - - if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { - if (prev_has_rcflags) { - try zig_args.append("-rcflags"); - try zig_args.append("--"); - prev_has_rcflags = false; - } - } else { - try zig_args.append("-rcflags"); - for (rc_source_file.flags) |arg| { - try zig_args.append(arg); - } - for (rc_source_file.include_paths) |include_path| { - try zig_args.append("/I"); - try zig_args.append(include_path.getPath2(mod.owner, step)); - } - try zig_args.append("--"); - prev_has_rcflags = true; - } - try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); - total_linker_objects += 1; - }, - } - } - - // We need to emit the --mod argument here so that the above link objects - // have the correct parent module, but only if the module is part of - // this compilation. - if (!my_responsibility) continue; - if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { - const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; - try mod.appendZigProcessFlags(&zig_args, step); - - // --dep arguments - try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); - for (mod.import_table.keys(), mod.import_table.values()) |name, import| { - const import_index = cli_named_modules.modules.getIndex(import).?; - const import_cli_name = cli_named_modules.names.keys()[import_index]; - zig_args.appendAssumeCapacity("--dep"); - if (std.mem.eql(u8, import_cli_name, name)) { - zig_args.appendAssumeCapacity(import_cli_name); - } else { - zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); - } - } - - // When the CLI sees a -M argument, it determines whether it - // implies the existence of a Zig compilation unit based on - // whether there is a root source file. If there is no root - // source file, then this is not a zig compilation unit - it is - // perhaps a set of linker objects, or C source files instead. - // Linker objects are added to the CLI globally, while C source - // files must have a module parent. - if (mod.root_source_file) |lp| { - const src = lp.getPath2(mod.owner, step); - try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); - } else if (moduleNeedsCliArg(mod)) { - try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); - } - } - } - } - - if (total_linker_objects == 0) { - return step.fail("the linker needs one or more objects to link", .{}); - } - - for (frameworks.keys(), frameworks.values()) |name, info| { - if (info.needed) { - try zig_args.append("-needed_framework"); - } else if (info.weak) { - try zig_args.append("-weak_framework"); - } else { - try zig_args.append("-framework"); - } - try zig_args.append(name); - } - - if (compile.is_linking_libcpp) { - try zig_args.append("-lc++"); - } - - if (compile.is_linking_libc) { - try zig_args.append("-lc"); - } - } - - if (compile.win32_manifest) |manifest_file| { - try zig_args.append(manifest_file.getPath2(b, step)); - } - - if (compile.win32_module_definition) |module_file| { - try zig_args.append(module_file.getPath2(b, step)); - } - - if (compile.image_base) |image_base| { - try zig_args.append("--image-base"); - try zig_args.append(b.fmt("0x{x}", .{image_base})); - } - - for (compile.filters) |filter| { - try zig_args.append("--test-filter"); - try zig_args.append(filter); - } - - if (compile.test_runner) |test_runner| { - try zig_args.append("--test-runner"); - try zig_args.append(test_runner.path.getPath2(b, step)); - } - - for (b.debug_log_scopes) |log_scope| { - try zig_args.append("--debug-log"); - try zig_args.append(log_scope); - } - - if (b.debug_compile_errors) { - try zig_args.append("--debug-compile-errors"); - } - - if (b.debug_incremental) { - try zig_args.append("--debug-incremental"); - } - - if (b.verbose_air) try zig_args.append("--verbose-air"); - if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); - if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); - if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); - if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); - if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); - if (b.graph.time_report) try zig_args.append("--time-report"); - - if (compile.generated_asm != null) try zig_args.append("-femit-asm"); - if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); - if (compile.generated_docs != null) try zig_args.append("-femit-docs"); - if (compile.generated_implib != null) try zig_args.append("-femit-implib"); - if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); - if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); - if (compile.generated_h != null) try zig_args.append("-femit-h"); - - try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); - - switch (compile.compress_debug_sections) { - .none => {}, - .zlib => try zig_args.append("--compress-debug-sections=zlib"), - .zstd => try zig_args.append("--compress-debug-sections=zstd"), - } - - if (compile.link_eh_frame_hdr) { - try zig_args.append("--eh-frame-hdr"); - } - if (compile.link_emit_relocs) { - try zig_args.append("--emit-relocs"); - } - if (compile.link_function_sections) { - try zig_args.append("-ffunction-sections"); - } - if (compile.link_data_sections) { - try zig_args.append("-fdata-sections"); - } - if (compile.link_gc_sections) |x| { - try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); - } - if (!compile.linker_dynamicbase) { - try zig_args.append("--no-dynamicbase"); - } - if (compile.linker_allow_shlib_undefined) |x| { - try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); - } - if (compile.link_z_notext) { - try zig_args.append("-z"); - try zig_args.append("notext"); - } - if (!compile.link_z_relro) { - try zig_args.append("-z"); - try zig_args.append("norelro"); - } - if (compile.link_z_lazy) { - try zig_args.append("-z"); - try zig_args.append("lazy"); - } - if (compile.link_z_common_page_size) |size| { - try zig_args.append("-z"); - try zig_args.append(b.fmt("common-page-size={d}", .{size})); - } - if (compile.link_z_max_page_size) |size| { - try zig_args.append("-z"); - try zig_args.append(b.fmt("max-page-size={d}", .{size})); - } - if (compile.link_z_defs) { - try zig_args.append("-z"); - try zig_args.append("defs"); - } - - if (compile.libc_file) |libc_file| { - try zig_args.append("--libc"); - try zig_args.append(libc_file.getPath2(b, step)); - } else if (b.libc_file) |libc_file| { - try zig_args.append("--libc"); - try zig_args.append(libc_file); - } - - try zig_args.append("--cache-dir"); - try zig_args.append(b.cache_root.path orelse "."); - - try zig_args.append("--global-cache-dir"); - try zig_args.append(b.graph.global_cache_root.path orelse "."); - - if (b.graph.debug_compiler_runtime_libs) |mode| - try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); - - try zig_args.append("--name"); - try zig_args.append(compile.name); - - if (compile.linkage) |some| switch (some) { - .dynamic => try zig_args.append("-dynamic"), - .static => try zig_args.append("-static"), - }; - if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { - if (compile.version) |version| { - try zig_args.append("--version"); - try zig_args.append(b.fmt("{f}", .{version})); - } - - if (compile.rootModuleTarget().os.tag.isDarwin()) { - const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ - compile.rootModuleTarget().libPrefix(), - compile.name, - compile.rootModuleTarget().dynamicLibSuffix(), - }); - try zig_args.append("-install_name"); - try zig_args.append(install_name); - } - } - - if (compile.entitlements) |entitlements| { - try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); - } - if (compile.pagezero_size) |pagezero_size| { - const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); - try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); - } - if (compile.headerpad_size) |headerpad_size| { - const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); - try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); - } - if (compile.headerpad_max_install_names) { - try zig_args.append("-headerpad_max_install_names"); - } - if (compile.dead_strip_dylibs) { - try zig_args.append("-dead_strip_dylibs"); - } - if (compile.force_load_objc) { - try zig_args.append("-ObjC"); - } - if (compile.discard_local_symbols) { - try zig_args.append("--discard-all"); - } - - try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); - try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); - try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); - if (compile.rdynamic) { - try zig_args.append("-rdynamic"); - } - if (compile.import_memory) { - try zig_args.append("--import-memory"); - } - if (compile.export_memory) { - try zig_args.append("--export-memory"); - } - if (compile.import_symbols) { - try zig_args.append("--import-symbols"); - } - if (compile.import_table) { - try zig_args.append("--import-table"); - } - if (compile.export_table) { - try zig_args.append("--export-table"); - } - if (compile.initial_memory) |initial_memory| { - try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); - } - if (compile.max_memory) |max_memory| { - try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); - } - if (compile.shared_memory) { - try zig_args.append("--shared-memory"); - } - if (compile.global_base) |global_base| { - try zig_args.append(b.fmt("--global-base={d}", .{global_base})); - } - - if (compile.wasi_exec_model) |model| { - try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); - } - if (compile.linker_script) |linker_script| { - try zig_args.append("--script"); - try zig_args.append(linker_script.getPath2(b, step)); - } - - if (compile.version_script) |version_script| { - try zig_args.append("--version-script"); - try zig_args.append(version_script.getPath2(b, step)); - } - if (compile.linker_allow_undefined_version) |x| { - try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); - } - - if (compile.linker_enable_new_dtags) |enabled| { - try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); - } - - if (compile.kind == .@"test") { - if (compile.exec_cmd_args) |exec_cmd_args| { - for (exec_cmd_args) |cmd_arg| { - if (cmd_arg) |arg| { - try zig_args.append("--test-cmd"); - try zig_args.append(arg); - } else { - try zig_args.append("--test-cmd-bin"); - } - } - } - } - - if (b.sysroot) |sysroot| { - try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); - } - - // -I and -L arguments that appear after the last --mod argument apply to all modules. - const cwd: Io.Dir = .cwd(); - const io = b.graph.io; - - for (b.search_prefixes.items) |search_prefix| { - var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { - return step.fail("unable to open prefix directory '{s}': {s}", .{ - search_prefix, @errorName(err), - }); - }; - defer prefix_dir.close(io); - - // Avoid passing -L and -I flags for nonexistent directories. - // This prevents a warning, that should probably be upgraded to an error in Zig's - // CLI parsing code, when the linker sees an -L directory that does not exist. - - if (prefix_dir.access(io, "lib", .{})) |_| { - try zig_args.appendSlice(&.{ - "-L", b.pathJoin(&.{ search_prefix, "lib" }), - }); - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ - search_prefix, @errorName(e), - }), - } - - if (prefix_dir.access(io, "include", .{})) |_| { - try zig_args.appendSlice(&.{ - "-I", b.pathJoin(&.{ search_prefix, "include" }), - }); - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ - search_prefix, @errorName(e), - }), - } - } - - if (compile.rc_includes != .any) { - try zig_args.append("-rcincludes"); - try zig_args.append(@tagName(compile.rc_includes)); - } - - try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); - - if (compile.build_id orelse b.build_id) |build_id| { - try zig_args.append(switch (build_id) { - .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), - .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), - }); - } - - const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| - dir.getPath2(b, step) - else if (b.graph.zig_lib_directory.path) |_| - b.fmt("{f}", .{b.graph.zig_lib_directory}) - else - null; - - if (opt_zig_lib_dir) |zig_lib_dir| { - try zig_args.append("--zig-lib-dir"); - try zig_args.append(zig_lib_dir); - } - - try addFlag(&zig_args, "PIE", compile.pie); - - if (compile.lto) |lto| { - try zig_args.append(switch (lto) { - .full => "-flto=full", - .thin => "-flto=thin", - .none => "-fno-lto", - }); - } - - try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); - - if (compile.subsystem) |subsystem| { - try zig_args.append("--subsystem"); - try zig_args.append(@tagName(subsystem)); - } - - if (compile.mingw_unicode_entry_point) { - try zig_args.append("-municode"); - } - - if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ - "--error-limit", b.fmt("{d}", .{err_limit}), - }); - - try addFlag(&zig_args, "incremental", b.graph.incremental); - - try zig_args.append("--listen=-"); - - // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux - // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and - // pass that to zig, e.g. via 'zig build-lib @args.rsp' - // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html - var args_length: usize = 0; - for (zig_args.items) |arg| { - args_length += arg.len + 1; // +1 to account for null terminator - } - if (args_length >= 30 * 1024) { - try b.cache_root.handle.createDirPath(io, "args"); - - const args_to_escape = zig_args.items[2..]; - var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); - arg_blk: for (args_to_escape) |arg| { - for (arg, 0..) |c, arg_idx| { - if (c == '\\' or c == '"') { - // Slow path for arguments that need to be escaped. We'll need to allocate and copy - var escaped: std.ArrayList(u8) = .empty; - try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1); - try escaped.appendSlice(arena, arg[0..arg_idx]); - for (arg[arg_idx..]) |to_escape| { - if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\'); - try escaped.append(arena, to_escape); - } - escaped_args.appendAssumeCapacity(escaped.items); - continue :arg_blk; - } - } - escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument - } - - // Write the args to zig-cache/args/ to avoid conflicts with - // other zig build commands running in parallel. - const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); - const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); - - var args_hash: [Sha256.digest_length]u8 = undefined; - Sha256.hash(args, &args_hash, .{}); - var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; - _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); - - const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; - if (b.cache_root.handle.access(io, args_file, .{})) |_| { - // The args file is already present from a previous run. - } else |err| switch (err) { - error.FileNotFound => { - var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ - .replace = false, - .make_path = true, - }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ - b.cache_root, args_file, e, - }); - defer af.deinit(io); - - af.file.writeStreamingAll(io, args) catch |e| { - return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ - b.cache_root, args_file, e, - }); - }; - // Note we can't clean up this file, not even after build - // success, because that might interfere with another build - // process that needs the same file. - af.link(io) catch |e| switch (e) { - error.PathAlreadyExists => { - // The args file was created by another concurrent build process. - }, - else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ - b.cache_root, args_file, other_err, - }), - }; - }, - else => |other_err| return other_err, - } - - const resolved_args_file = try mem.concat(arena, u8, &.{ - "@", - try b.cache_root.join(arena, &.{args_file}), - }); - - zig_args.shrinkRetainingCapacity(2); - try zig_args.append(resolved_args_file); - } - - return try zig_args.toOwnedSlice(); -} - -pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { - c.step.result_error_msgs.clearRetainingCapacity(); - c.step.result_stderr = ""; - - c.step.result_error_bundle.deinit(gpa); - c.step.result_error_bundle = std.zig.ErrorBundle.empty; - - if (c.step.result_failed_command) |cmd| { - gpa.free(cmd); - c.step.result_failed_command = null; - } - - const zig_args = try getZigArgs(c, true); - const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); - return maybe_output_bin_path.?; -} - -pub fn doAtomicSymLinks( - step: *Step, - output_path: []const u8, - filename_major_only: []const u8, - filename_name_only: []const u8, -) !void { - const b = step.owner; - const io = b.graph.io; - const out_dir = fs.path.dirname(output_path) orelse "."; - const out_basename = fs.path.basename(output_path); - // sym link for libfoo.so.1 to libfoo.so.1.2.3 - const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); - const cwd: Io.Dir = .cwd(); - cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { - return step.fail("unable to symlink {s} -> {s}: {s}", .{ - major_only_path, out_basename, @errorName(err), - }); - }; - // sym link for libfoo.so to libfoo.so.1 - const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); - cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { - return step.fail("Unable to symlink {s} -> {s}: {s}", .{ - name_only_path, filename_major_only, @errorName(err), - }); - }; -} - -fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { - const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); - var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); - errdefer list.deinit(); - var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); - while (line_it.next()) |line| { - if (mem.trim(u8, line, " \t").len == 0) continue; - var tok_it = mem.tokenizeAny(u8, line, " \t"); - try list.append(PkgConfigPkg{ - .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, - .desc = tok_it.rest(), - }); - } - return list.toOwnedSlice(); -} - -fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { - if (b.pkg_config_pkg_list) |res| { - return res; - } - var code: u8 = undefined; - if (execPkgConfigList(b, &code)) |list| { - b.pkg_config_pkg_list = list; - return list; - } else |err| { - const result = switch (err) { - error.ProcessTerminated => error.PkgConfigCrashed, - error.ExecNotSupported => error.PkgConfigFailed, - error.ExitCodeFailure => error.PkgConfigFailed, - error.FileNotFound => error.PkgConfigNotInstalled, - error.InvalidName => error.PkgConfigNotInstalled, - error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, - else => return err, - }; - b.pkg_config_pkg_list = result; - return result; - } -} - -fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { - const cond = opt orelse return; - try args.ensureUnusedCapacity(1); - if (cond) { - args.appendAssumeCapacity("-f" ++ name); - } else { - args.appendAssumeCapacity("-fno-" ++ name); - } -} - -const PkgConfigResult = struct { - cflags: []const []const u8, - libs: []const []const u8, -}; - -/// Run pkg-config for the given library name and parse the output, returning the arguments -/// that should be passed to zig to link the given library. -fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult { - const wl_rpath_prefix = "-Wl,-rpath,"; - - const b = compile.step.owner; - const arena = b.allocator; - const pkg_name = match: { - // First we have to map the library name to pkg config name. Unfortunately, - // there are several examples where this is not straightforward: - // -lSDL2 -> pkg-config sdl2 - // -lgdk-3 -> pkg-config gdk-3.0 - // -latk-1.0 -> pkg-config atk - // -lpulse -> pkg-config libpulse - const pkgs = try getPkgConfigList(b); - - // Exact match means instant winner. - for (pkgs) |pkg| { - if (mem.eql(u8, pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Next we'll try ignoring case. - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { - break :match pkg.name; - } - } - - // Prefixed "lib" or suffixed ".0". - for (pkgs) |pkg| { - if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { - const prefix = pkg.name[0..pos]; - const suffix = pkg.name[pos + lib_name.len ..]; - if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; - if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; - break :match pkg.name; - } - } - - // Trimming "-1.0". - if (mem.endsWith(u8, lib_name, "-1.0")) { - const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; - for (pkgs) |pkg| { - if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { - break :match pkg.name; - } - } - } - - return error.PackageNotFound; - }; - - var code: u8 = undefined; - const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = if (b.runAllowFail(&[_][]const u8{ - pkg_config_exe, - pkg_name, - "--cflags", - "--libs", - }, &code, .ignore)) |stdout| stdout else |err| switch (err) { - error.ProcessTerminated => return error.PkgConfigCrashed, - error.ExecNotSupported => return error.PkgConfigFailed, - error.ExitCodeFailure => return error.PkgConfigFailed, - error.FileNotFound => return error.PkgConfigNotInstalled, - else => return err, - }; - - var zig_cflags: std.ArrayList([]const u8) = .empty; - defer zig_cflags.deinit(arena); - var zig_libs: std.ArrayList([]const u8) = .empty; - defer zig_libs.deinit(arena); - - var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); - while (arg_it.next()) |arg| { - if (mem.eql(u8, arg, "-I")) { - const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_cflags.appendSlice(arena, &.{ "-I", dir }); - } else if (mem.startsWith(u8, arg, "-I")) { - try zig_cflags.append(arena, arg); - } else if (mem.eql(u8, arg, "-L")) { - const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_libs.appendSlice(arena, &.{ "-L", dir }); - } else if (mem.startsWith(u8, arg, "-L")) { - try zig_libs.append(arena, arg); - } else if (mem.eql(u8, arg, "-l")) { - const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_libs.appendSlice(arena, &.{ "-l", lib }); - } else if (mem.startsWith(u8, arg, "-l")) { - try zig_libs.append(arena, arg); - } else if (mem.eql(u8, arg, "-D")) { - const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; - try zig_cflags.appendSlice(arena, &.{ "-D", macro }); - } else if (mem.startsWith(u8, arg, "-D")) { - try zig_cflags.append(arena, arg); - } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { - try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); - } else if (b.debug_pkg_config) { - return compile.step.fail("unknown pkg-config flag '{s}'", .{arg}); - } - } - - try zig_cflags.shrinkToLen(arena); - try zig_libs.shrinkToLen(arena); - - return .{ - .cflags = zig_cflags.toOwnedSliceAssert(), - .libs = zig_libs.toOwnedSliceAssert(), - }; -} - -fn checkCompileErrors(compile: *Compile) !void { - // Clear this field so that it does not get printed by the build runner. - const actual_eb = compile.step.result_error_bundle; - compile.step.result_error_bundle = .empty; - - const arena = compile.step.owner.allocator; - - const actual_errors = ae: { - var aw: std.Io.Writer.Allocating = .init(arena); - defer aw.deinit(); - try actual_eb.renderToWriter(.{ - .include_reference_trace = false, - .include_source_line = false, - }, &aw.writer); - break :ae try aw.toOwnedSlice(); - }; - - // Render the expected lines into a string that we can compare verbatim. - var expected_generated: std.ArrayList(u8) = .empty; - const expect_errors = compile.expect_errors.?; - - var actual_line_it = mem.splitScalar(u8, actual_errors, '\n'); - - // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile - switch (expect_errors) { - .starts_with => |expect_starts_with| { - if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return; - return compile.step.fail( - \\ - \\========= should start with: ============ - \\{s} - \\========= but not found: ================ - \\{s} - \\========================================= - , .{ expect_starts_with, actual_errors }); - }, - .contains => |expect_line| { - while (actual_line_it.next()) |actual_line| { - if (!matchCompileError(actual_line, expect_line)) continue; - return; - } - - return compile.step.fail( - \\ - \\========= should contain: =============== - \\{s} - \\========= but not found: ================ - \\{s} - \\========================================= - , .{ expect_line, actual_errors }); - }, - .stderr_contains => |expect_line| { - const actual_stderr: []const u8 = if (compile.step.result_error_msgs.items.len > 0) - compile.step.result_error_msgs.items[0] - else - &.{}; - compile.step.result_error_msgs.clearRetainingCapacity(); - - var stderr_line_it = mem.splitScalar(u8, actual_stderr, '\n'); - - while (stderr_line_it.next()) |actual_line| { - if (!matchCompileError(actual_line, expect_line)) continue; - return; - } - - return compile.step.fail( - \\ - \\========= should contain: =============== - \\{s} - \\========= but not found: ================ - \\{s} - \\========================================= - , .{ expect_line, actual_stderr }); - }, - .exact => |expect_lines| { - for (expect_lines) |expect_line| { - const actual_line = actual_line_it.next() orelse { - try expected_generated.appendSlice(arena, expect_line); - try expected_generated.append(arena, '\n'); - continue; - }; - if (matchCompileError(actual_line, expect_line)) { - try expected_generated.appendSlice(arena, actual_line); - try expected_generated.append(arena, '\n'); - continue; - } - try expected_generated.appendSlice(arena, expect_line); - try expected_generated.append(arena, '\n'); - } - - if (mem.eql(u8, expected_generated.items, actual_errors)) return; - - return compile.step.fail( - \\ - \\========= expected: ===================== - \\{s} - \\========= but found: ==================== - \\{s} - \\========================================= - , .{ expected_generated.items, actual_errors }); - }, - } -} - -fn matchCompileError(actual: []const u8, expected: []const u8) bool { - if (mem.endsWith(u8, actual, expected)) return true; - if (mem.startsWith(u8, expected, ":?:?: ")) { - if (mem.endsWith(u8, actual, expected[":?:?: ".len..])) return true; - } - // We scan for /?/ in expected line and if there is a match, we match everything - // up to and after /?/. - const expected_trim = mem.trim(u8, expected, " "); - if (mem.find(u8, expected_trim, "/?/")) |index| { - const actual_trim = mem.trim(u8, actual, " "); - const lhs = expected_trim[0..index]; - const rhs = expected_trim[index + "/?/".len ..]; - if (mem.startsWith(u8, actual_trim, lhs) and mem.endsWith(u8, actual_trim, rhs)) return true; - } - return false; -} - -fn moduleNeedsCliArg(mod: *const Module) bool { - return for (mod.link_objects.items) |o| switch (o) { - .c_source_file, .c_source_files, .assembly_file, .win32_resource_file => break true, - else => continue, - } else false; -} - diff --git a/lib/compiler/maker/Step/InstallArtifact.zig b/lib/compiler/maker/Step/InstallArtifact.zig deleted file mode 100644 index ba3846c1a574f9934f4196f03529dac3abcbe275..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Step/InstallArtifact.zig +++ /dev/null @@ -1,96 +0,0 @@ - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); - const b = step.owner; - const io = b.graph.io; - - var all_cached = true; - - if (install_artifact.dest_dir) |dest_dir| { - const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); - const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); - all_cached = all_cached and p == .fresh; - - if (install_artifact.dylib_symlinks) |dls| { - try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); - } - - install_artifact.artifact.installed_path = full_dest_path; - } - - if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { - const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.implib_dir) |implib_dir| { - const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.pdb_dir) |pdb_dir| { - const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); - const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); - all_cached = all_cached and p == .fresh; - } - - if (install_artifact.h_dir) |h_dir| { - if (install_artifact.emitted_h) |emitted_h| { - const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); - const p = try step.installFile(emitted_h, full_h_path); - all_cached = all_cached and p == .fresh; - } - - for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { - .file => |file| { - const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); - const p = try step.installFile(file.source, full_h_path); - all_cached = all_cached and p == .fresh; - }, - .directory => |dir| { - const src_dir_path = dir.source.getPath3(b, step); - const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); - - var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {s}", .{ - src_dir_path, @errorName(err), - }); - }; - defer src_dir.close(io); - - var it = try src_dir.walk(b.allocator); - next_entry: while (try it.next(io)) |entry| { - for (dir.options.exclude_extensions) |ext| { - if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; - } - if (dir.options.include_extensions) |incs| { - for (incs) |inc| { - if (std.mem.endsWith(u8, entry.path, inc)) break; - } else { - continue :next_entry; - } - } - - const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); - switch (entry.kind) { - .directory => { - try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); - const p = try step.installDir(full_dest_path); - all_cached = all_cached and p == .existed; - }, - .file => { - const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); - all_cached = all_cached and p == .fresh; - }, - else => continue, - } - } - }, - }; - } - - step.result_cached = all_cached; -} diff --git a/lib/compiler/maker/Step/Run.zig b/lib/compiler/maker/Step/Run.zig deleted file mode 100644 index 4ba04092e02221fffc55729a64b44aa41a5b9d91..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Step/Run.zig +++ /dev/null @@ -1,2130 +0,0 @@ -const Run = @This(); - -const builtin = @import("builtin"); - -const std = @import("std"); -const Io = std.Io; -const Dir = std.Io.Dir; -const mem = std.mem; -const process = std.process; -const EnvMap = std.process.Environ.Map; -const assert = std.debug.assert; -const Cache = std.Build.Cache; -const Path = std.Build.Cache.Path; - -const Step = @import("../Step.zig"); - -/// If this is a Zig unit test binary, this tracks the names of the unit -/// tests that are also fuzz tests. Indexes cannot be used as they may -/// change between reruns. -fuzz_tests: std.ArrayList([]const u8), -cached_test_metadata: ?CachedTestMetadata = null, - -/// Populated during the fuzz phase if this run step corresponds to a unit test -/// executable that contains fuzz tests. -rebuilt_executable: ?Path, - -fn make(step: *Step, options: Step.MakeOptions) !void { - const b = step.owner; - const io = b.graph.io; - const arena = b.allocator; - const run: *Run = @fieldParentPtr("step", step); - const has_side_effects = run.hasSideEffects(); - - var argv_list = std.array_list.Managed([]const u8).init(arena); - var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); - - var man = b.graph.cache.obtain(); - defer man.deinit(); - - if (run.environ_map) |environ_map| { - for (environ_map.keys(), environ_map.values()) |key, value| { - man.hash.addBytes(key); - man.hash.addBytes(value); - } - } - - man.hash.add(run.color); - man.hash.add(run.disable_zig_progress); - - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(bytes); - man.hash.addBytes(bytes); - }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); - man.hash.addBytes(file.prefix); - _ = try man.addFilePath(file_path, null); - }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); - try argv_list.append(resolved_arg); - man.hash.addBytes(resolved_arg); - }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); - - var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; - - const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { - return step.fail( - "unable to open input file '{f}': {t}", - .{ file_path, err }, - ); - }; - defer file.close(io); - - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); - _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return step.fail( - "failed to read from '{f}': {t}", - .{ file_path, file_reader.err.? }, - ), - error.WriteFailed => return error.OutOfMemory, - }; - - try argv_list.append(result.written()); - man.hash.addBytes(file_plp.prefix); - _ = try man.addFilePath(file_path, null); - }, - .artifact => |pa| { - const artifact = pa.artifact; - - if (artifact.rootModuleTarget().os.tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - run.addPathForDynLibs(artifact); - } - const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; - - try argv_list.append(b.fmt("{s}{s}", .{ - pa.prefix, - run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), - })); - - _ = try man.addFile(file_path, null); - }, - .output_file, .output_directory => |output| { - man.hash.addBytes(output.prefix); - man.hash.addBytes(output.basename); - // Add a placeholder into the argument list because we need the - // manifest hash to be updated with all arguments before the - // object directory is computed. - try output_placeholders.append(.{ - .index = argv_list.items.len, - .tag = arg, - .output = output, - }); - _ = try argv_list.addOne(); - }, - } - } - - switch (run.stdin) { - .bytes => |bytes| { - man.hash.addBytes(bytes); - }, - .lazy_path => |lazy_path| { - const file_path = lazy_path.getPath2(b, step); - _ = try man.addFile(file_path, null); - }, - .none => {}, - } - - if (run.captured_stdout) |captured| { - man.hash.addBytes(captured.output.basename); - man.hash.add(captured.trim_whitespace); - } - - if (run.captured_stderr) |captured| { - man.hash.addBytes(captured.output.basename); - man.hash.add(captured.trim_whitespace); - } - - hashStdIo(&man.hash, run.stdio); - - for (run.file_inputs.items) |lazy_path| { - _ = try man.addFile(lazy_path.getPath2(b, step), null); - } - - if (run.cwd) |cwd| { - const cwd_path = cwd.getPath3(b, step); - _ = man.hash.addBytes(try cwd_path.toString(arena)); - } - - if (!has_side_effects and try step.cacheHitAndWatch(&man)) { - // cache hit, skip running command - const digest = man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); - - step.result_cached = true; - return; - } - - const dep_output_file = run.dep_output_file orelse { - // We already know the final output paths, use them directly. - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); - - const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; - for (output_placeholders.items) |placeholder| { - const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, output_sub_dir_path, @errorName(err), - }); - }; - const arg_output_path = run.convertPathArg(.{ - .root_dir = .cwd(), - .sub_path = placeholder.output.generated_file.getPath(), - }); - argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) - arg_output_path - else - b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); - } - - try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); - if (!has_side_effects) try step.writeManifestAndWatch(&man); - return; - }; - - // We do not know the final output paths yet, use temp paths to run the command. - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - - for (output_placeholders.items) |placeholder| { - const output_components = .{ tmp_dir_path, placeholder.output.basename }; - const output_sub_path = b.pathJoin(&output_components); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, output_sub_dir_path, @errorName(err), - }); - }; - const raw_output_path: Cache.Path = .{ - .root_dir = b.cache_root, - .sub_path = b.pathJoin(&output_components), - }; - placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); - argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ - placeholder.output.prefix, - run.convertPathArg(raw_output_path), - }); - } - - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); - - const dep_file_dir = Dir.cwd(); - const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); - if (has_side_effects) - try man.addDepFile(dep_file_dir, dep_file_basename) - else - try man.addDepFilePost(dep_file_dir, dep_file_basename); - - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - const any_output = output_placeholders.items.len > 0 or - run.captured_stdout != null or run.captured_stderr != null; - - // Rename into place - if (any_output) { - const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; - - b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { - Dir.RenameError.DirNotEmpty => { - b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { - return step.fail("unable to remove dir '{f}'{s}: {t}", .{ - b.cache_root, tmp_dir_path, del_err, - }); - }; - b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { - return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, - }); - }; - }, - else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, - }), - }; - } - - if (!has_side_effects) try step.writeManifestAndWatch(&man); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - run.captured_stdout, - run.captured_stderr, - b.cache_root, - &digest, - ); -} - -/// Reads stdout of a Zig test process until a termination condition is reached: -/// * A write fails, indicating the child unexpectedly closed stdin -/// * A test (or a response from the test runner) times out -/// * The wait fails, indicating the child closed stdout and stderr -fn waitZigTest( - run: *Run, - child: *process.Child, - options: Step.MakeOptions, - multi_reader: *Io.File.MultiReader, - opt_metadata: *?TestMetadata, - results: *Step.TestResults, -) !union(enum) { - write_failed: anyerror, - no_poll: struct { - active_test_index: ?u32, - ns_elapsed: u64, - }, - timeout: struct { - active_test_index: ?u32, - ns_elapsed: u64, - }, -} { - const gpa = run.step.owner.allocator; - const arena = run.step.owner.allocator; - const io = run.step.owner.graph.io; - - var sub_prog_node: ?std.Progress.Node = null; - defer if (sub_prog_node) |n| n.end(); - - if (opt_metadata.*) |*md| { - // Previous unit test process died or was killed; we're continuing where it left off - requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; - } else { - // Running unit tests normally - run.fuzz_tests.clearRetainingCapacity(); - sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err }; - } - - var active_test_index: ?u32 = null; - - var last_update: Io.Clock.Timestamp = .now(io, .awake); - - // This timeout is used when we're waiting on the test runner itself rather than a user-specified - // test. For instance, if the test runner leaves this much time between us requesting a test to - // start and it acknowledging the test starting, we terminate the child and raise an error. This - // *should* never happen, but could in theory be caused by some very unlucky IB in a test. - const response_timeout: Io.Clock.Duration = t: { - if (fuzz_context != null) break :t null; // don't timeout fuzz tests - const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); - break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; - }; - const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ - .clock = .awake, - .raw = .fromNanoseconds(ns), - } else null; - - const stdout = multi_reader.reader(0); - const stderr = multi_reader.reader(1); - const Header = std.zig.Server.Message.Header; - - while (true) { - const timeout: Io.Timeout = t: { - const opt_duration = if (active_test_index == null) response_timeout else test_timeout; - const duration = opt_duration orelse break :t .none; - break :t .{ .deadline = last_update.addDuration(duration) }; - }; - - // This block is exited when `stdout` contains enough bytes for a `Header`. - header_ready: { - if (stdout.buffered().len >= @sizeOf(Header)) { - // We already have one, no need to poll! - break :header_ready; - } - - multi_reader.fill(64, timeout) catch |err| switch (err) { - error.Timeout => return .{ .timeout = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - error.EndOfStream => return .{ .no_poll = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - else => |e| return e, - }; - - continue; - } - // There is definitely a header available now -- read it. - const header = stdout.takeStruct(Header, .little) catch unreachable; - - while (stdout.buffered().len < header.bytes_len) { - multi_reader.fill(64, timeout) catch |err| switch (err) { - error.Timeout => return .{ .timeout = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - error.EndOfStream => return .{ .no_poll = .{ - .active_test_index = active_test_index, - .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), - } }, - else => |e| return e, - }; - } - - const body = stdout.take(header.bytes_len) catch unreachable; - var body_r: std.Io.Reader = .fixed(body); - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - }, - .test_metadata => { - // `metadata` would only be populated if we'd already seen a `test_metadata`, but we - // only request it once (and importantly, we don't re-request it if we kill and - // restart the test runner). - assert(opt_metadata.* == null); - - const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable; - results.test_count = tm_hdr.tests_len; - - const names = try arena.alloc(u32, results.test_count); - for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; - - const expected_panic_msgs = try arena.alloc(u32, results.test_count); - for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; - - const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; - - options.progress_node.setEstimatedTotalItems(names.len); - opt_metadata.* = .{ - .string_bytes = try arena.dupe(u8, string_bytes), - .ns_per_test = try arena.alloc(u64, results.test_count), - .names = names, - .expected_panic_msgs = expected_panic_msgs, - .next_index = 0, - .prog_node = options.progress_node, - }; - @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); - - active_test_index = null; - last_update = .now(io, .awake); - - requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err }; - }, - .test_started => { - active_test_index = opt_metadata.*.?.next_index - 1; - last_update = .now(io, .awake); - }, - .test_results => { - const md = &opt_metadata.*.?; - - const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable; - assert(tr_hdr.index == active_test_index); - - switch (tr_hdr.flags.status) { - .pass => {}, - .skip => results.skip_count +|= 1, - .fail => results.fail_count +|= 1, - } - const leak_count = tr_hdr.flags.leak_count; - const log_err_count = tr_hdr.flags.log_err_count; - results.leak_count +|= leak_count; - results.log_err_count +|= log_err_count; - - if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index)); - - if (tr_hdr.flags.status == .fail) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - if (stderr_bytes.len == 0) { - try run.step.addError("'{s}' failed without output", .{name}); - } else { - try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); - } - } else if (leak_count > 0) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); - } else if (log_err_count > 0) { - const name = md.testName(tr_hdr.index); - const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); - stderr.tossBuffered(); - try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); - } - - active_test_index = null; - - const now: Io.Clock.Timestamp = .now(io, .awake); - md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds); - last_update = now; - - requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; - }, - else => {}, // ignore other messages - } - } -} - -const FuzzTestRunner = struct { - run: *Run, - ctx: FuzzContext, - coverage_id: ?u64, - - instances: []Instance, - /// The indexes of this are layed out such that it is effectively an array - /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr. - batch: Io.Batch, - /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter. - pending_broadcasts: std.ArrayList(u8), - broadcast: std.ArrayList(u8), - broadcast_undelivered: u32, - - const Instance = struct { - child: process.Child, - message: std.ArrayListAligned(u8, .@"4"), - broadcast_written: usize, - stderr: std.ArrayList(u8), - stdin_vec: [1][]u8, - stdout_vec: [1][]u8, - stderr_vec: [1][]u8, - progress_node: std.Progress.Node, - - fn messageHeader(instance: *Instance) InHeader { - assert(instance.message.items.len >= @sizeOf(InHeader)); - const header_ptr: *InHeader = @ptrCast(instance.message.items); - var header = header_ptr.*; - if (std.builtin.Endian.native != .little) { - std.mem.byteSwapAllFields(InHeader, &header); - } - return header; - } - }; - - const PendingBroadcastFooter = struct { - from_id: u32, - body_len: u32, - }; - - const InHeader = std.zig.Server.Message.Header; - const OutHeader = std.zig.Client.Message.Header; - - const stdin_i = 0; - const stdout_i = 1; - const stderr_i = 2; - - fn init( - run: *Run, - ctx: FuzzContext, - progress_node: std.Progress.Node, - spawn_options: process.SpawnOptions, - ) !FuzzTestRunner { - const step_owner = run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - - const n_instances = switch (ctx.fuzz.mode) { - .forever => step_owner.graph.max_jobs orelse @min( - std.Thread.getCpuCount() catch 1, - (std.math.maxInt(u32) - 2) / 3, - ), - .limit => 1, - }; - const instances = try gpa.alloc(Instance, n_instances); - errdefer gpa.free(instances); - const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3); - errdefer gpa.free(batch_storage); - - @memset(instances, .{ - .child = undefined, - .message = .empty, - .broadcast_written = undefined, - .stderr = .empty, - .stdin_vec = undefined, - .stdout_vec = undefined, - .stderr_vec = undefined, - .progress_node = undefined, - }); - for (0.., instances) |id, *instance| { - errdefer for (instances[0..id]) |*spawned| { - spawned.child.kill(io); - spawned.progress_node.end(); - }; - instance.child = try process.spawn(io, spawn_options); - instance.progress_node = progress_node.start("starting fuzzer", 0); - } - - return .{ - .run = run, - .ctx = ctx, - .coverage_id = null, - - .instances = instances, - .batch = .init(batch_storage), - .pending_broadcasts = .empty, - .broadcast = .empty, - .broadcast_undelivered = 0, - }; - } - - fn deinit(f: *FuzzTestRunner) void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - - f.batch.cancel(io); - gpa.free(f.batch.storage); - var total_rss: usize = 0; - for (f.instances) |*instance| { - instance.child.kill(io); - instance.message.deinit(gpa); - instance.stderr.deinit(gpa); - instance.progress_node.end(); - total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; - } - f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); - gpa.free(f.instances); - } - - fn startInstances(f: *FuzzTestRunner) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - - for (0.., f.instances) |id, *instance| { - const id32: u32 = @intCast(id); - (switch (f.ctx.fuzz.mode) { - .forever => sendRunFuzzTestMessage( - io, - instance.child.stdin.?, - f.run.fuzz_tests.items, - .forever, - id32, - ), - .limit => |limit| sendRunFuzzTestMessage( - io, - instance.child.stdin.?, - f.run.fuzz_tests.items, - .iterations, - limit.amount, - ), - }) catch |write_err| { - // The runner unexpectedly closed stdin, which means it crashed during initialization. - // Clean up everything and wait for the child to exit. - instance.child.stdin.?.close(io); - instance.child.stdin = null; - const term = try instance.child.wait(io); - return f.run.step.fail( - "unable to write stdin ({t}); test process unexpectedly {f}", - .{ write_err, fmtTerm(term) }, - ); - }; - - try f.addStdoutRead(id32, @sizeOf(InHeader)); - try f.addStderrRead(id32); - } - } - - fn listen(f: *FuzzTestRunner) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - - while (true) { - try f.batch.awaitConcurrent(io, .none); - while (f.batch.next()) |completion| { - const id = completion.index / 3; - const result = completion.result; - switch (completion.index % 3) { - 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) { - // Avoid calling `instanceEos` until EndOfStream is seen with stderr so - // that all stderr is collected. - error.BrokenPipe => continue, - else => |write_e| return write_e, - }), - 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) { - // Avoid calling `instanceEos` until EndOfStream is seen with stderr so - // that all stderr is collected. - error.EndOfStream => continue, - else => |read_e| return read_e, - }), - 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { - error.EndOfStream => return f.instanceEos(id), - else => |read_e| return read_e, - }), - else => unreachable, - } - } - } - } - - fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const io = step_owner.graph.io; - const instance = &f.instances[id]; - - instance.message.items.len += n; - const total_read = instance.message.items.len; - if (total_read < @sizeOf(InHeader)) { - try f.addStdoutRead(id, @sizeOf(InHeader)); - return; - } - - const header = instance.messageHeader(); - const body = instance.message.items[@sizeOf(InHeader)..]; - if (body.len != header.bytes_len) { - try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len); - return; - } - - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", - .{ builtin.zig_version_string, body }, - ); - }, - .coverage_id => { - var body_r: Io.Reader = .fixed(body); - f.coverage_id = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable; - const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable; - - const fuzz = f.ctx.fuzz; - fuzz.queue_mutex.lockUncancelable(io); - defer fuzz.queue_mutex.unlock(io); - try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ - .id = f.coverage_id.?, - .cumulative = .{ - .runs = cumulative_runs, - .unique = cumulative_unique, - .coverage = cumulative_coverage, - }, - .run = f.run, - } }); - fuzz.queue_cond.signal(io); - }, - .fuzz_start_addr => { - var body_r: Io.Reader = .fixed(body); - const fuzz = f.ctx.fuzz; - const addr = body_r.takeInt(u64, .little) catch unreachable; - - fuzz.queue_mutex.lockUncancelable(io); - defer fuzz.queue_mutex.unlock(io); - try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ - .addr = addr, - .coverage_id = f.coverage_id.?, - } }); - fuzz.queue_cond.signal(io); - }, - .fuzz_test_change => { - const test_i = std.mem.readInt(u32, body[0..4], .little); - instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); - }, - .broadcast_fuzz_input => { - if (f.instances.len == 1) { - // No other processes to broadcast to. - } else if (f.broadcast_undelivered == 0) { - try f.instanceBroadcast(id, body); - } else { - const footer: PendingBroadcastFooter = .{ - .from_id = id, - .body_len = @intCast(body.len), - }; - // There is another broadcast in progress so add this one to the queue. - const size = @sizeOf(PendingBroadcastFooter) + body.len; - try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); - f.pending_broadcasts.appendSliceAssumeCapacity(body); - f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); - } - }, - else => {}, // ignore other messages - } - - instance.message.clearRetainingCapacity(); - try f.addStdoutRead(id, @sizeOf(InHeader)); - } - - fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void { - const instance = &f.instances[id]; - instance.stderr.items.len += n; - try f.addStderrRead(id); - } - - fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void { - const instance = &f.instances[id]; - - instance.broadcast_written += n; - if (instance.broadcast_written == f.broadcast.items.len) { - f.broadcast_undelivered -= 1; - if (f.broadcast_undelivered == 0) { - try f.broadcastComplete(); - } - } else { - f.addStdinWrite(id); - } - } - - fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const instance = &f.instances[id]; - - try instance.message.ensureTotalCapacity(gpa, end); - const start = instance.message.items.len; - instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]}; - f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{ - .file = instance.child.stdout.?, - .data = &instance.stdout_vec, - } }); - } - - fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - const instance = &f.instances[id]; - - try instance.stderr.ensureUnusedCapacity(gpa, 1); - instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()}; - f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{ - .file = instance.child.stderr.?, - .data = &instance.stderr_vec, - } }); - } - - fn addStdinWrite(f: *FuzzTestRunner, id: u32) void { - const instance = &f.instances[id]; - - assert(f.broadcast.items.len != instance.broadcast_written); - instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]}; - f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{ - .file = instance.child.stdin.?, - .data = &instance.stdin_vec, - } }); - } - - fn instanceEos(f: *FuzzTestRunner, id: u32) !void { - const step_owner = f.run.step.owner; - const io = step_owner.graph.io; - const instance = &f.instances[id]; - - instance.child.stdin.?.close(io); - instance.child.stdin = null; - const term = try instance.child.wait(io); - if (!termMatches(.{ .exited = 0 }, term)) { - f.run.step.result_stderr = try f.mergedStderr(); - try f.saveCrash(id, term); - return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); - } - } - - fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { - const step = &f.run.step; - const b = step.owner; - const io = b.graph.io; - - if (f.coverage_id == null) return; - - // Search for the input file corresponding to the instance - const InputHeader = Build.abi.fuzz.MmapInputHeader; - var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; - var in_r: Io.File.Reader = undefined; - var in_f: Io.File = undefined; - var in_name_buf: [12]u8 = undefined; - var in_name: []const u8 = undefined; - var i: u32 = 0; - const header: InputHeader = while (true) : ({ - if (i == std.math.maxInt(u32)) return; - i += 1; - }) { - const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; - in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; - in_f = b.cache_root.handle.openFile(io, in_name, .{ - .lock = .exclusive, - .lock_nonblocking = true, - }) catch |e| switch (e) { - error.FileNotFound => return, - error.WouldBlock => continue, // Can not be from - // the crashed instance since it is still locked. - else => return step.fail("failed to open file '{f}{s}': {t}", .{ - b.cache_root, in_name, e, - }), - }; - - in_r = in_f.readerStreaming(io, &in_r_buf); - const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { - in_f.close(io); - switch (e) { - error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, - }), - error.EndOfStream => continue, - } - }; - - if (header.pc_digest == f.coverage_id.? and - header.instance_id == id and - header.test_i < f.run.fuzz_tests.items.len) - { - break header; - } - - in_f.close(io); - }; - defer in_f.close(io); - - // Save it to a seperate file - const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; - const out = b.cache_root.handle.createFile(io, crash_name, .{ - .lock = .exclusive, // Multiple run steps could have found a crash at the same time - }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ - b.cache_root, crash_name, e, - }); - defer out.close(io); - - var out_w_buf: [512]u8 = undefined; - var out_w = out.writerStreaming(io, &out_w_buf); - _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { - error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, - }), - error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ - b.cache_root, crash_name, out_w.err.?, - }), - }; - - return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ - f.run.fuzz_tests.items[header.test_i], - fmtTerm(term), - b.cache_root, - crash_name, - }); - } - - fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void { - assert(f.instances.len > 1); - assert(f.broadcast_undelivered == 0); // no other broadcast is progress - assert(f.broadcast.items.len == 0); - assert(from_id < f.instances.len); - - const step_owner = f.run.step.owner; - const gpa = step_owner.allocator; - - var out_header: OutHeader = .{ - .tag = .new_fuzz_input, - .bytes_len = @intCast(bytes.len), - }; - if (std.builtin.Endian.native != .little) { - std.mem.byteSwapAllFields(OutHeader, &out_header); - } - try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len); - f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header)); - f.broadcast.appendSliceAssumeCapacity(bytes); - - f.broadcast_undelivered = @intCast(f.instances.len - 1); - for (0.., f.instances) |to_id, *instance| { - if (to_id == from_id) continue; - instance.broadcast_written = 0; - f.addStdinWrite(@intCast(to_id)); - } - } - - fn broadcastComplete(f: *FuzzTestRunner) !void { - assert(f.instances.len > 1); - assert(f.broadcast_undelivered == 0); - f.broadcast.clearRetainingCapacity(); - - const pending = &f.pending_broadcasts; - if (pending.items.len != 0) { - // Another broadcast is pending; copy it over to `broadcast` - - const footer_len = @sizeOf(PendingBroadcastFooter); - const footer_bytes = pending.items[pending.items.len - footer_len ..]; - const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes); - pending.items.len -= footer_len; - - const body = pending.items[pending.items.len - footer.body_len ..]; - try f.instanceBroadcast(footer.from_id, body); - pending.items.len -= body.len; - } - } - - fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { - const step_owner = f.run.step.owner; - const arena = step_owner.allocator; - - // Collect any available stderr - while (f.batch.next()) |completion| { - if (completion.index % 3 != 2) continue; - const len = completion.result.file_read_streaming catch continue; - f.instances[completion.index / 3].stderr.items.len += len; - } - - var stderr_len: usize = 0; - for (f.instances) |*instance| stderr_len += instance.stderr.items.len; - const stderr = try arena.alloc(u8, stderr_len); - - stderr_len = 0; - for (f.instances) |*instance| { - @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items); - stderr_len += instance.stderr.items.len; - } - return stderr; - } -}; - -fn evalFuzzTest( - run: *Run, - spawn_options: process.SpawnOptions, - options: Step.MakeOptions, - fuzz_context: FuzzContext, -) !void { - var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); - defer f.deinit(); - try f.startInstances(); - try f.listen(); -} - -const StdioPollEnum = enum { stdout, stderr }; - -fn evalZigTest( - run: *Run, - spawn_options: process.SpawnOptions, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !void { - if (fuzz_context != null) { - try evalFuzzTest(run, spawn_options, options, fuzz_context.?); - return; - } - - const step_owner = run.step.owner; - const gpa = step_owner.allocator; - const arena = step_owner.allocator; - const io = step_owner.graph.io; - - // We will update this every time a child runs. - run.step.result_peak_rss = 0; - - var test_results: Step.TestResults = .{ - .test_count = 0, - .skip_count = 0, - .fail_count = 0, - .crash_count = 0, - .timeout_count = 0, - .leak_count = 0, - .log_err_count = 0, - }; - var test_metadata: ?TestMetadata = null; - - while (true) { - var child = try process.spawn(io, spawn_options); - var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; - var multi_reader: Io.File.MultiReader = undefined; - multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); - var child_killed = false; - defer if (!child_killed) { - child.kill(io); - multi_reader.deinit(); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - }; - - switch (try waitZigTest( - run, - &child, - options, - &multi_reader, - &test_metadata, - &test_results, - )) { - .write_failed => |err| { - // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured - // all available stderr to make our error output as useful as possible. - const stderr_fr = multi_reader.fileReader(1); - while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) { - error.ReadFailed => return stderr_fr.err.?, - error.EndOfStream => {}, - } - run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); - - // Clean up everything and wait for the child to exit. - child.stdin.?.close(io); - child.stdin = null; - multi_reader.deinit(); - child_killed = true; - const term = try child.wait(io); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - - // The individual unit test results are irrelevant: the test runner itself broke! - // Fail immediately without populating `s.test_results`. - return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); - }, - .no_poll => |no_poll| { - // This might be a success (we requested exit and the child dutifully closed stdout) or - // a crash of some kind. Either way, the child will terminate by itself -- wait for it. - const stderr_reader = multi_reader.reader(1); - const stderr_owned = try arena.dupe(u8, stderr_reader.buffered()); - - // Clean up everything and wait for the child to exit. - child.stdin.?.close(io); - child.stdin = null; - multi_reader.deinit(); - child_killed = true; - const term = try child.wait(io); - run.step.result_peak_rss = @max( - run.step.result_peak_rss, - child.resource_usage_statistics.getMaxRss() orelse 0, - ); - - if (no_poll.active_test_index) |test_index| { - // A test was running, so this is definitely a crash. Report it against that - // test, and continue to the next test. - test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; - test_results.crash_count += 1; - try run.step.addError("'{s}' {f}{s}{s}", .{ - test_metadata.?.testName(test_index), - fmtTerm(term), - if (stderr_owned.len != 0) " with stderr:\n" else "", - std.mem.trim(u8, stderr_owned, "\n"), - }); - continue; - } - - // Report an error if the child terminated uncleanly or if we were still trying to run more tests. - run.step.result_stderr = stderr_owned; - const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); - if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { - // The individual unit test results are irrelevant: the test runner itself broke! - // Fail immediately without populating `s.test_results`. - return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); - } - - // We're done with all of the tests! Commit the test results and return. - run.step.test_results = test_results; - if (test_metadata) |tm| { - run.cached_test_metadata = tm.toCachedTestMetadata(); - if (options.web_server) |ws| { - if (run.step.owner.graph.time_report) { - ws.updateTimeReportRunTest( - run, - &run.cached_test_metadata.?, - tm.ns_per_test, - ); - } - } - } - return; - }, - .timeout => |timeout| { - const stderr_reader = multi_reader.reader(1); - const stderr = stderr_reader.buffered(); - stderr_reader.tossBuffered(); - if (timeout.active_test_index) |test_index| { - // A test was running. Report the timeout against that test, and continue on to - // the next test. - test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; - test_results.timeout_count += 1; - try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ - test_metadata.?.testName(test_index), - Io.Duration{ .nanoseconds = timeout.ns_elapsed }, - if (stderr.len != 0) " with stderr:\n" else "", - std.mem.trim(u8, stderr, "\n"), - }); - continue; - } - // Just log an error and let the child be killed. - run.step.result_stderr = try arena.dupe(u8, stderr); - // The individual unit test results in `results` are irrelevant: the test runner - // is broken! Fail immediately without populating `s.test_results`. - return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); - }, - } - comptime unreachable; - } -} - -const TestMetadata = struct { - names: []const u32, - ns_per_test: []u64, - expected_panic_msgs: []const u32, - string_bytes: []const u8, - next_index: u32, - prog_node: std.Progress.Node, - - fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { - return .{ - .names = tm.names, - .string_bytes = tm.string_bytes, - }; - } - - fn testName(tm: TestMetadata, index: u32) []const u8 { - return tm.toCachedTestMetadata().testName(index); - } -}; - -pub const CachedTestMetadata = struct { - names: []const u32, - string_bytes: []const u8, - - pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { - return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); - } -}; - -fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { - while (metadata.next_index < metadata.names.len) { - const i = metadata.next_index; - metadata.next_index += 1; - - if (metadata.expected_panic_msgs[i] != 0) continue; - - const name = metadata.testName(i); - if (sub_prog_node.*) |n| n.end(); - sub_prog_node.* = metadata.prog_node.start(name, 0); - - try sendRunTestMessage(io, in, .run_test, i); - return; - } else { - metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done - try sendMessage(io, in, .exit); - } -} - -fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 0, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 4, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u32, index, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; -} - -fn sendRunFuzzTestMessage( - io: Io, - file: Io.File, - test_names: []const []const u8, - kind: std.Build.abi.fuzz.LimitKind, - amount_or_instance: u64, -) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = .start_fuzzing, - .bytes_len = 1 + 8 + 4 + count: { - var c: u32 = @intCast(test_names.len * 4); - for (test_names) |name| { - c += @intCast(name.len); - } - break :count c; - }, - }; - var w = file.writerStreaming(io, &.{}); - w.interface.writeStruct(header, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - for (test_names) |test_name| { - w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeAll(test_name) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - } -} - -fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { - const b = run.step.owner; - const io = b.graph.io; - const arena = b.allocator; - const gpa = b.allocator; - - var child = try process.spawn(io, spawn_options); - defer child.kill(io); - - switch (run.stdin) { - .bytes => |bytes| { - child.stdin.?.writeStreamingAll(io, bytes) catch |err| { - return run.step.fail("unable to write stdin: {t}", .{err}); - }; - child.stdin.?.close(io); - child.stdin = null; - }, - .lazy_path => |lazy_path| { - const path = lazy_path.getPath3(b, &run.step); - const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { - return run.step.fail("unable to open stdin file: {t}", .{err}); - }; - defer file.close(io); - // TODO https://github.com/ziglang/zig/issues/23955 - var read_buffer: [1024]u8 = undefined; - var file_reader = file.reader(io, &read_buffer); - var write_buffer: [1024]u8 = undefined; - var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); - _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { - error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ - path, file_reader.err.?, - }), - error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ - stdin_writer.err.?, - }), - }; - stdin_writer.interface.flush() catch |err| switch (err) { - error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ - stdin_writer.err.?, - }), - }; - child.stdin.?.close(io); - child.stdin = null; - }, - .none => {}, - } - - var stdout_bytes: ?[]const u8 = null; - var stderr_bytes: ?[]const u8 = null; - - if (child.stdout) |stdout| { - if (child.stderr) |stderr| { - var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; - var multi_reader: Io.File.MultiReader = undefined; - multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); - defer multi_reader.deinit(); - - const stdout_reader = multi_reader.reader(0); - const stderr_reader = multi_reader.reader(1); - - while (multi_reader.fill(64, .none)) |_| { - if (run.stdio_limit.toInt()) |limit| { - if (stdout_reader.buffered().len > limit) - return error.StdoutStreamTooLong; - if (stderr_reader.buffered().len > limit) - return error.StderrStreamTooLong; - } - } else |err| switch (err) { - error.Timeout => unreachable, - error.EndOfStream => {}, - else => |e| return e, - } - - try multi_reader.checkAnyError(); - - // TODO: this string can leak since alloc below can return error. - stdout_bytes = try multi_reader.toOwnedSlice(0); - // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. - stderr_bytes = try multi_reader.toOwnedSlice(1); - } else { - var stdout_reader = stdout.readerStreaming(io, &.{}); - stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => return stdout_reader.err.?, - error.StreamTooLong => return error.StdoutStreamTooLong, - }; - } - } else if (child.stderr) |stderr| { - var stderr_reader = stderr.readerStreaming(io, &.{}); - stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => return stderr_reader.err.?, - error.StreamTooLong => return error.StderrStreamTooLong, - }; - } - - if (stderr_bytes) |bytes| if (bytes.len > 0) { - // Treat stderr as an error message. - const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { - .check => |checks| !checksContainStderr(checks.items), - else => true, - }; - if (stderr_is_diagnostic) { - run.step.result_stderr = bytes; - } - }; - - run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; - - return .{ - .term = try child.wait(io), - .stdout = stdout_bytes, - .stderr = stderr_bytes, - }; -} - -const IndexedOutput = struct { - index: usize, - tag: @typeInfo(Arg).@"union".tag_type.?, - output: *Output, -}; - -pub fn rerunInFuzzMode( - run: *Run, - fuzz: *std.Build.Fuzz, - prog_node: std.Progress.Node, -) !void { - const step = &run.step; - const b = step.owner; - const io = b.graph.io; - const arena = b.allocator; - var argv_list: std.ArrayList([]const u8) = .empty; - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(arena, bytes); - }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); - }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); - }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); - - var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; - - const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); - defer file.close(io); - - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); - _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - error.WriteFailed => return error.OutOfMemory, - }; - - try argv_list.append(arena, result.written()); - }, - .artifact => |pa| { - const artifact = pa.artifact; - const file_path: []const u8 = p: { - if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); - break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; - }; - try argv_list.append(arena, b.fmt("{s}{s}", .{ - pa.prefix, - run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), - })); - }, - .output_file, .output_directory => unreachable, - } - } - - if (run.step.result_failed_command) |cmd| { - fuzz.gpa.free(cmd); - run.step.result_failed_command = null; - } - - const has_side_effects = false; - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ - .progress_node = prog_node, - .watch = undefined, // not used by `runCommand` - .web_server = null, // only needed for time reports - .unit_test_timeout_ns = null, // don't time out fuzz tests for now - .gpa = fuzz.gpa, - }, .{ - .fuzz = fuzz, - }); -} - -fn populateGeneratedPaths( - arena: std.mem.Allocator, - output_placeholders: []const IndexedOutput, - captured_stdout: ?*CapturedStdIo, - captured_stderr: ?*CapturedStdIo, - cache_root: Cache.Directory, - digest: *const Cache.HexDigest, -) !void { - for (output_placeholders) |placeholder| { - placeholder.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, placeholder.output.basename, - }); - } - - if (captured_stdout) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); - } - - if (captured_stderr) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); - } -} - -fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void { - if (term) |t| switch (t) { - .exited => |code| try w.print("exited with code {d}", .{code}), - .signal => |sig| try w.print("terminated with signal {t}", .{sig}), - .stopped => |sig| try w.print("stopped with signal {t}", .{sig}), - .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}), - } else { - try w.writeAll("exited with any code"); - } -} -fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) { - return .{ .data = term }; -} - -const FuzzContext = struct { - fuzz: *std.Build.Fuzz, -}; - -fn runCommand( - run: *Run, - argv: []const []const u8, - has_side_effects: bool, - output_dir_path: []const u8, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !void { - const step = &run.step; - const b = step.owner; - const arena = b.allocator; - const gpa = options.gpa; - const io = b.graph.io; - - const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; - - try step.handleChildProcUnsupported(); - try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); - - const allow_skip = switch (run.stdio) { - .check, .zig_test => run.skip_foreign_checks, - else => false, - }; - - var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); - defer interp_argv.deinit(); - - var environ_map: EnvMap = env: { - const orig = run.environ_map orelse &b.graph.environ_map; - break :env try orig.clone(gpa); - }; - defer environ_map.deinit(); - - const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { - // InvalidExe: cpu arch mismatch - // FileNotFound: can happen with a wrong dynamic linker path - if (err == error.InvalidExe or err == error.FileNotFound) interpret: { - // TODO: learn the target from the binary directly rather than from - // relying on it being a Compile step. This will make this logic - // work even for the edge case that the binary was produced by a - // third party. - const exe = switch (run.argv.items[0]) { - .artifact => |exe| exe.artifact, - else => break :interpret, - }; - switch (exe.kind) { - .exe, .@"test" => {}, - else => break :interpret, - } - - const root_target = exe.rootModuleTarget(); - const need_cross_libc = exe.is_linking_libc and - (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); - const other_target = exe.root_module.resolved_target.?.result; - switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ - .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, - .link_libc = exe.is_linking_libc, - })) { - .native, .rosetta => { - if (allow_skip) return error.MakeSkipped; - break :interpret; - }, - .wine => |bin_name| { - if (b.enable_wine) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); - - // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but - // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. - if (environ_map.get("WINEDEBUG") == null) { - try environ_map.put("WINEDEBUG", "-all"); - } - } else { - return failForeign(run, "-fwine", argv[0], exe); - } - }, - .qemu => |bin_name| { - if (b.enable_qemu) { - try interp_argv.append(bin_name); - - if (need_cross_libc) { - if (b.libc_runtimes_dir) |dir| { - try interp_argv.append("-L"); - try interp_argv.append(b.pathJoin(&.{ - dir, - try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( - b.allocator, - root_target.cpu.arch, - root_target.os.tag, - root_target.abi, - ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( - b.allocator, - root_target.cpu.arch, - root_target.abi, - ) else unreachable, - })); - } else return failForeign(run, "--libc-runtimes", argv[0], exe); - } - - try interp_argv.appendSlice(argv); - } else return failForeign(run, "-fqemu", argv[0], exe); - }, - .darling => |bin_name| { - if (b.enable_darling) { - try interp_argv.append(bin_name); - try interp_argv.appendSlice(argv); - } else { - return failForeign(run, "-fdarling", argv[0], exe); - } - }, - .wasmtime => |bin_name| { - if (b.enable_wasmtime) { - try interp_argv.append(bin_name); - try interp_argv.append("--dir=."); - // Wasmtime doeesn't inherit environment variables from the parent process - // by default. '-S inherit-env' was added in Wasmtime version 20. - try interp_argv.append("-Sinherit-env"); - try interp_argv.append(argv[0]); - try interp_argv.appendSlice(argv[1..]); - } else { - return failForeign(run, "-fwasmtime", argv[0], exe); - } - }, - .bad_dl => |foreign_dl| { - if (allow_skip) return error.MakeSkipped; - - const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; - - return step.fail( - \\the host system is unable to execute binaries from the target - \\ because the host dynamic linker is '{s}', - \\ while the target dynamic linker is '{s}'. - \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step - , .{ host_dl, foreign_dl }); - }, - .bad_os_or_cpu => { - if (allow_skip) return error.MakeSkipped; - - const host_name = try b.graph.host.result.zigTriple(b.allocator); - const foreign_name = try root_target.zigTriple(b.allocator); - - return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ - host_name, foreign_name, - }); - }, - } - - if (root_target.os.tag == .windows) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - run.addPathForDynLibs(exe); - } - - gpa.free(step.result_failed_command.?); - step.result_failed_command = null; - try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); - - break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { - if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; - if (e == error.MakeFailed) return error.MakeFailed; // error already reported - return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); - }; - } - if (err == error.MakeFailed) return error.MakeFailed; // error already reported - - return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); - }; - - const generic_result = opt_generic_result orelse { - assert(run.stdio == .zig_test); - // Specific errors have already been reported, and test results are populated. All we need - // to do is report step failure if any test failed. - if (!step.test_results.isSuccess()) return error.MakeFailed; - return; - }; - - assert(fuzz_context == null); - assert(run.stdio != .zig_test); - - // Capture stdout and stderr to GeneratedFile objects. - const Stream = struct { - captured: ?*CapturedStdIo, - bytes: ?[]const u8, - }; - for ([_]Stream{ - .{ - .captured = run.captured_stdout, - .bytes = generic_result.stdout, - }, - .{ - .captured = run.captured_stderr, - .bytes = generic_result.stderr, - }, - }) |stream| { - if (stream.captured) |captured| { - const output_components = .{ output_dir_path, captured.output.basename }; - const output_path = try b.cache_root.join(arena, &output_components); - captured.output.generated_file.path = output_path; - - const sub_path = b.pathJoin(&output_components); - const sub_path_dirname = Dir.path.dirname(sub_path).?; - b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.cache_root, sub_path_dirname, @errorName(err), - }); - }; - const data = switch (captured.trim_whitespace) { - .none => stream.bytes.?, - .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), - .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), - .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), - }; - b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { - return step.fail("unable to write file '{f}{s}': {s}", .{ - b.cache_root, sub_path, @errorName(err), - }); - }; - } - } - - switch (run.stdio) { - .zig_test => unreachable, - .check => |checks| for (checks.items) |check| switch (check) { - .expect_stderr_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { - return step.fail( - \\========= expected this stderr: ========= - \\{s} - \\========= but found: ==================== - \\{s} - , .{ - expected_bytes, - generic_result.stderr.?, - }); - } - }, - .expect_stderr_match => |match| { - if (mem.find(u8, generic_result.stderr.?, match) == null) { - return step.fail( - \\========= expected to find in stderr: ========= - \\{s} - \\========= but stderr does not contain it: ===== - \\{s} - , .{ - match, - generic_result.stderr.?, - }); - } - }, - .expect_stdout_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { - return step.fail( - \\========= expected this stdout: ========= - \\{s} - \\========= but found: ==================== - \\{s} - , .{ - expected_bytes, - generic_result.stdout.?, - }); - } - }, - .expect_stdout_match => |match| { - if (mem.find(u8, generic_result.stdout.?, match) == null) { - return step.fail( - \\========= expected to find in stdout: ========= - \\{s} - \\========= but stdout does not contain it: ===== - \\{s} - , .{ - match, - generic_result.stdout.?, - }); - } - }, - .expect_term => |expected_term| { - if (!termMatches(expected_term, generic_result.term)) { - return step.fail("process {f} (expected {f})", .{ - fmtTerm(generic_result.term), - fmtTerm(expected_term), - }); - } - }, - }, - else => { - // On failure, report captured stderr like normal standard error output. - const bad_exit = switch (generic_result.term) { - .exited => |code| code != 0, - .signal, .stopped, .unknown => true, - }; - if (bad_exit) { - if (generic_result.stderr) |bytes| { - run.step.result_stderr = bytes; - } - } - - try step.handleChildProcessTerm(generic_result.term); - }, - } -} - -const EvalGenericResult = struct { - term: process.Child.Term, - stdout: ?[]const u8, - stderr: ?[]const u8, -}; - -fn spawnChildAndCollect( - run: *Run, - argv: []const []const u8, - environ_map: *EnvMap, - has_side_effects: bool, - options: Step.MakeOptions, - fuzz_context: ?FuzzContext, -) !?EvalGenericResult { - const b = run.step.owner; - const graph = b.graph; - const io = graph.io; - - if (fuzz_context != null) { - assert(!has_side_effects); - assert(run.stdio == .zig_test); - } - - const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; - - // If an error occurs, it's caused by this command: - assert(run.step.result_failed_command == null); - run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ - .child = environ_map, - .parent = &graph.environ_map, - }, argv); - - var spawn_options: process.SpawnOptions = .{ - .argv = argv, - .cwd = child_cwd, - .environ_map = environ_map, - .request_resource_usage_statistics = true, - .stdin = if (run.stdin != .none) s: { - assert(run.stdio != .inherit); - break :s .pipe; - } else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .ignore, - .inherit => .inherit, - .check => .ignore, - .zig_test => .pipe, - }, - .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .ignore, - .inherit => .inherit, - .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, - .zig_test => .pipe, - }, - .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { - .infer_from_args => if (has_side_effects) .inherit else .pipe, - .inherit => .inherit, - .check => .pipe, - .zig_test => .pipe, - }, - }; - - if (run.stdio == .zig_test) { - const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| e, - }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); - try result; - return null; - } else { - const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; - if (!run.disable_zig_progress and !inherit) { - spawn_options.progress_node = options.progress_node; - } - const terminal_mode: Io.Terminal.Mode = if (inherit) m: { - const stderr = try io.lockStderr(&.{}, graph.stderr_mode); - break :m stderr.terminal_mode; - } else .no_color; - defer if (inherit) io.unlockStderr(); - try setColorEnvironmentVariables(run, environ_map, terminal_mode); - - const started: Io.Clock.Timestamp = .now(io, .awake); - const result = evalGeneric(run, spawn_options) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| e, - }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); - return try result; - } -} - -fn hashStdIo(hh: *Cache.HashHelper, stdio: StdIo) void { - switch (stdio) { - .infer_from_args, .inherit, .zig_test => {}, - .check => |checks| for (checks.items) |check| { - hh.add(@as(std.meta.Tag(StdIo.Check), check)); - switch (check) { - .expect_stderr_exact, - .expect_stderr_match, - .expect_stdout_exact, - .expect_stdout_match, - => |s| hh.addBytes(s), - - .expect_term => |term| { - hh.add(@as(std.meta.Tag(process.Child.Term), term)); - switch (term) { - inline .exited, .signal, .stopped => |x| hh.add(x), - .unknown => |x| hh.add(x), - } - }, - } - }, - } -} -fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { - return if (expected) |e| switch (e) { - .exited => |expected_code| switch (actual) { - .exited => |actual_code| expected_code == actual_code, - else => false, - }, - .signal => |expected_sig| switch (actual) { - .signal => |actual_sig| expected_sig == actual_sig, - else => false, - }, - .stopped => |expected_sig| switch (actual) { - .stopped => |actual_sig| expected_sig == actual_sig, - else => false, - }, - .unknown => |expected_code| switch (actual) { - .unknown => |actual_code| expected_code == actual_code, - else => false, - }, - } else switch (actual) { - .exited => true, - else => false, - }; -} - -fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { - color: switch (run.color) { - .manual => {}, - .enable => { - try environ_map.put("CLICOLOR_FORCE", "1"); - _ = environ_map.swapRemove("NO_COLOR"); - }, - .disable => { - try environ_map.put("NO_COLOR", "1"); - _ = environ_map.swapRemove("CLICOLOR_FORCE"); - }, - .inherit => switch (terminal_mode) { - .no_color, .windows_api => continue :color .disable, - .escape_codes => continue :color .enable, - }, - .auto => { - const capture_stderr = run.captured_stderr != null or switch (run.stdio) { - .check => |checks| checksContainStderr(checks.items), - .infer_from_args, .inherit, .zig_test => false, - }; - if (capture_stderr) { - continue :color .disable; - } else { - continue :color .inherit; - } - }, - } -} - -fn checksContainStdout(checks: []const StdIo.Check) bool { - for (checks) |check| switch (check) { - .expect_stderr_exact, - .expect_stderr_match, - .expect_term, - => continue, - - .expect_stdout_exact, - .expect_stdout_match, - => return true, - }; - return false; -} - -fn checksContainStderr(checks: []const StdIo.Check) bool { - for (checks) |check| switch (check) { - .expect_stdout_exact, - .expect_stdout_match, - .expect_term, - => continue, - - .expect_stderr_exact, - .expect_stderr_match, - => return true, - }; - return false; -} - -/// Returns whether the Run step has side effects *other than* updating the output arguments. -fn hasSideEffects(run: Run) bool { - if (run.has_side_effects) return true; - return switch (run.stdio) { - .infer_from_args => !run.hasAnyOutputArgs(), - .inherit => true, - .check => false, - .zig_test => false, - }; -} - -fn hasAnyOutputArgs(run: Run) bool { - if (run.captured_stdout != null) return true; - if (run.captured_stderr != null) return true; - for (run.argv.items) |arg| switch (arg) { - .output_file, .output_directory => return true, - else => continue, - }; - return false; -} - -/// If `path` is cwd-relative, make it relative to the cwd of the child instead. -/// -/// Whenever a path is included in the argv of a child, it should be put through this function first -/// to make sure the child doesn't see paths relative to a cwd other than its own. -fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { - const b = run.step.owner; - const graph = b.graph; - const arena = graph.arena; - - const path_str = path.toString(arena) catch @panic("OOM"); - if (Dir.path.isAbsolute(path_str)) { - // Absolute paths don't need changing. - return path_str; - } - const child_cwd_rel: []const u8 = rel: { - const child_lazy_cwd = run.cwd orelse break :rel path_str; - const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); - // Convert it from relative to *our* cwd, to relative to the *child's* cwd. - break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); - }; - // Not every path can be made relative, e.g. if the path and the child cwd are on different - // disk designators on Windows. In that case, `relative` will return an absolute path which we can - // just return. - if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel; - - // We're not done yet. In some cases this path must be prefixed with './': - // * On POSIX, the executable name cannot be a single component like 'foo' - // * Some executables might treat a leading '-' like a flag, which we must avoid - // There's no harm in it, so just *always* apply this prefix. - return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); -} - -fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { - const b = run.step.owner; - const compiles = artifact.getCompileDependencies(true); - for (compiles) |compile| { - if (compile.root_module.resolved_target.?.result.os.tag == .windows and - compile.isDynamicLibrary()) - { - addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); - } - } -} - -fn failForeign( - run: *Run, - suggested_flag: []const u8, - argv0: []const u8, - exe: *Step.Compile, -) error{ MakeFailed, MakeSkipped, OutOfMemory } { - switch (run.stdio) { - .check, .zig_test => { - if (run.skip_foreign_checks) - return error.MakeSkipped; - - const b = run.step.owner; - const host_name = try b.graph.host.result.zigTriple(b.allocator); - const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); - - return run.step.fail( - \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) - \\ consider using {s} or enabling skip_foreign_checks in the Run step - , .{ argv0, foreign_name, host_name, suggested_flag }); - }, - else => { - return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); - }, - } -} diff --git a/lib/compiler/maker/Step/WriteFile.zig b/lib/compiler/maker/Step/WriteFile.zig deleted file mode 100644 index d594f8983fe5ead949a5efdd0ba106c176777d34..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Step/WriteFile.zig +++ /dev/null @@ -1,206 +0,0 @@ - -fn make(step: *Step, options: Step.MakeOptions) !void { - _ = options; - const b = step.owner; - const graph = b.graph; - const io = graph.io; - const arena = b.allocator; - const gpa = graph.cache.gpa; - const write_file: *WriteFile = @fieldParentPtr("step", step); - - const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); - var open_dirs_count: usize = 0; - defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); - - switch (write_file.mode) { - .whole_cached => { - step.clearWatchInputs(); - - // The cache is used here not really as a way to speed things up - because writing - // the data to a file would probably be very fast - but as a way to find a canonical - // location to put build artifacts. - - // If, for example, a hard-coded path was used as the location to put WriteFile - // files, then two WriteFiles executing in parallel might clobber each other. - - var man = b.graph.cache.obtain(); - defer man.deinit(); - - for (write_file.files.items) |file| { - man.hash.addBytes(file.sub_path); - - switch (file.contents) { - .bytes => |bytes| { - man.hash.addBytes(bytes); - }, - .copy => |lazy_path| { - const path = lazy_path.getPath3(b, step); - _ = try man.addFilePath(path, null); - try step.addWatchInput(lazy_path); - }, - } - } - - for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { - man.hash.addBytes(dir.sub_path); - for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); - if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); - - const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); - const src_dir_path = dir.source.getPath3(b, step); - - var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{f}': {s}", .{ - src_dir_path, @errorName(err), - }); - }; - open_dir_cache_elem.* = src_dir; - open_dirs_count += 1; - - var it = try src_dir.walk(gpa); - defer it.deinit(); - while (try it.next(io)) |entry| { - if (!dir.options.pathIncluded(entry.path)) continue; - - switch (entry.kind) { - .directory => { - if (need_derived_inputs) { - const entry_path = try src_dir_path.join(arena, entry.path); - try step.addDirectoryWatchInputFromPath(entry_path); - } - }, - .file => { - const entry_path = try src_dir_path.join(arena, entry.path); - _ = try man.addFilePath(entry_path, null); - }, - else => continue, - } - } - } - - if (try step.cacheHit(&man)) { - const digest = man.final(); - write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); - assert(step.result_cached); - return; - } - - const digest = man.final(); - const cache_path = "o" ++ Dir.path.sep_str ++ digest; - - write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); - - try operate(write_file, open_dir_cache, .{ - .root_dir = b.cache_root, - .sub_path = cache_path, - }); - - try step.writeManifest(&man); - }, - .tmp => { - step.result_cached = false; - - var rand_int: u64 = undefined; - io.random(@ptrCast(&rand_int)); - const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - - write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); - - try operate(write_file, open_dir_cache, .{ - .root_dir = b.cache_root, - .sub_path = tmp_dir_sub_path, - }); - }, - .mutate => |lp| { - step.result_cached = false; - const root_path = try lp.getPath4(b, step); - write_file.generated_directory.path = try root_path.toString(arena); - try operate(write_file, open_dir_cache, root_path); - }, - } -} - -fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { - const step = &write_file.step; - const b = step.owner; - const io = b.graph.io; - const gpa = b.graph.cache.gpa; - const arena = b.allocator; - - var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| - return step.fail("unable to make path {f}: {t}", .{ root_path, err }); - defer cache_dir.close(io); - - for (write_file.files.items) |file| { - if (Dir.path.dirname(file.sub_path)) |dirname| { - cache_dir.createDirPath(io, dirname) catch |err| { - return step.fail("unable to make path '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, dirname, err, - }); - }; - } - switch (file.contents) { - .bytes => |bytes| { - cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, file.sub_path, err, - }); - }; - }, - .copy => |file_source| { - const source_path = file_source.getPath2(b, step); - const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { - return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ - source_path, root_path, Dir.path.sep, file.sub_path, err, - }); - }; - // At this point we already will mark the step as a cache miss. - // But this is kind of a partial cache hit since individual - // file copies may be avoided. Oh well, this information is - // discarded. - _ = prev_status; - }, - } - } - - for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { - const src_dir_path = dir.source.getPath3(b, step); - const dest_dirname = dir.sub_path; - - if (dest_dirname.len != 0) { - cache_dir.createDirPath(io, dest_dirname) catch |err| { - return step.fail("unable to make path '{f}{c}{s}': {t}", .{ - root_path, Dir.path.sep, dest_dirname, err, - }); - }; - } - - var it = try already_open_dir.walk(gpa); - defer it.deinit(); - while (try it.next(io)) |entry| { - if (!dir.options.pathIncluded(entry.path)) continue; - - const src_entry_path = try src_dir_path.join(arena, entry.path); - const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); - switch (entry.kind) { - .directory => try cache_dir.createDirPath(io, dest_path), - .file => { - const prev_status = Io.Dir.updateFile( - src_entry_path.root_dir.handle, - io, - src_entry_path.sub_path, - cache_dir, - dest_path, - .{}, - ) catch |err| { - return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ - src_entry_path, root_path, Dir.path.sep, dest_path, err, - }); - }; - _ = prev_status; - }, - else => continue, - } - } - } -} diff --git a/lib/compiler/maker/Watch.zig b/lib/compiler/maker/Watch.zig deleted file mode 100644 index 907e6536a132863603d70e423041b41c5b677a5c..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Watch.zig +++ /dev/null @@ -1,976 +0,0 @@ -const Watch = @This(); -const builtin = @import("builtin"); - -const std = @import("std"); -const Io = std.Io; -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const fatal = std.process.fatal; -const Configuration = std.Build.Configuration; - -const FsEvents = @import("Watch/FsEvents.zig"); -const Step = @import("Step.zig"); - -os: Os, -/// The number to show as the number of directories being watched. -dir_count: usize, -// These fields are common to most implementations so are kept here for simplicity. -// They are `undefined` on implementations which do not utilize then. -dir_table: DirTable, -generation: Generation, -configuration: *const Configuration, -make_steps: []Step, - -pub const have_impl = Os != void; - -/// Key is the directory to watch which contains one or more files we are -/// interested in noticing changes to. -/// -/// Value is generation. -const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false); - -/// Special key of "." means any changes in this directory trigger the steps. -const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); -const StepSet = std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, Generation); - -const Generation = u8; - -const Hash = std.hash.Wyhash; -const Cache = std.Build.Cache; - -const Os = switch (builtin.os.tag) { - .linux => struct { - const posix = std.posix; - - /// Keyed differently but indexes correspond 1:1 with `dir_table`. - handle_table: HandleTable, - /// fanotify file descriptors are keyed by mount id since marks - /// are limited to a single filesystem. - poll_fds: std.AutoArrayHashMapUnmanaged(MountId, posix.pollfd), - - const MountId = i32; - const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, struct { mount_id: MountId, reaction_set: ReactionSet }, FileHandle.Adapter, false); - - const fan_mask: std.os.linux.fanotify.MarkMask = .{ - .CLOSE_WRITE = true, - .CREATE = true, - .DELETE = true, - .DELETE_SELF = true, - .EVENT_ON_CHILD = true, - .MOVED_FROM = true, - .MOVED_TO = true, - .MOVE_SELF = true, - .ONDIR = true, - }; - - const FileHandle = struct { - handle: *align(1) std.os.linux.file_handle, - - fn clone(lfh: FileHandle, gpa: Allocator) Allocator.Error!FileHandle { - const bytes = lfh.slice(); - const new_ptr = try gpa.alignedAlloc( - u8, - .of(std.os.linux.file_handle), - @sizeOf(std.os.linux.file_handle) + bytes.len, - ); - const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); - new_header.* = lfh.handle.*; - const new: FileHandle = .{ .handle = new_header }; - @memcpy(new.slice(), lfh.slice()); - return new; - } - - fn destroy(lfh: FileHandle, gpa: Allocator) void { - const ptr: [*]u8 = @ptrCast(lfh.handle); - const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; - return gpa.free(allocated_slice); - } - - fn slice(lfh: FileHandle) []u8 { - const ptr: [*]u8 = &lfh.handle.f_handle; - return ptr[0..lfh.handle.handle_bytes]; - } - - const Adapter = struct { - pub fn hash(self: Adapter, a: FileHandle) u32 { - _ = self; - const unsigned_type: u32 = @bitCast(a.handle.handle_type); - return @truncate(Hash.hash(unsigned_type, a.slice())); - } - pub fn eql(self: Adapter, a: FileHandle, b: FileHandle, b_index: usize) bool { - _ = self; - _ = b_index; - return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice()); - } - }; - }; - - fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { - _ = cwd_path; - return .{ - .dir_table = .{}, - .dir_count = 0, - .os = switch (builtin.os.tag) { - .linux => .{ - .handle_table = .{}, - .poll_fds = .{}, - }, - else => {}, - }, - .generation = 0, - .make_steps = make_steps, - .configuration = configuration, - }; - } - - fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path, mount_id: *MountId) !FileHandle { - var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; - var buf: [std.fs.max_path_bytes]u8 = undefined; - const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{ - path.sub_path, - }) catch return error.NameTooLong; - const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); - stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); - try posix.name_to_handle_at(path.root_dir.handle.handle, adjusted_path, stack_ptr, mount_id, std.os.linux.AT.HANDLE_FID); - const stack_lfh: FileHandle = .{ .handle = stack_ptr }; - return stack_lfh.clone(gpa); - } - - fn markDirtySteps(w: *Watch, gpa: Allocator, fan_fd: posix.fd_t) !bool { - const fanotify = std.os.linux.fanotify; - const M = fanotify.event_metadata; - var events_buf: [256 + 4096]u8 = undefined; - var any_dirty = false; - while (true) { - var len = posix.read(fan_fd, &events_buf) catch |err| switch (err) { - error.WouldBlock => return any_dirty, - else => |e| return e, - }; - var meta: [*]align(1) M = @ptrCast(&events_buf); - while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ - len -= meta[0].event_len; - meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); - }) { - assert(meta[0].vers == M.VERSION); - if (meta[0].mask.Q_OVERFLOW) { - any_dirty = true; - std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); - markAllFilesDirty(w, gpa); - return true; - } - const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); - switch (fid.hdr.info_type) { - .DFID_NAME => { - const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); - const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); - const file_name = std.mem.span(file_name_z); - const lfh: FileHandle = .{ .handle = file_handle }; - if (w.os.handle_table.getPtr(lfh)) |value| { - if (value.reaction_set.getPtr(".")) |glob_set| - any_dirty = markStepSetDirty(gpa, w.make_steps, glob_set, any_dirty); - if (value.reaction_set.getPtr(file_name)) |step_set| - any_dirty = markStepSetDirty(gpa, w.make_steps, step_set, any_dirty); - } - }, - else => |t| std.log.warn("unexpected fanotify event '{t}'", .{t}), - } - } - } - } - - fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { - // Add missing marks and note persisted ones. - for (steps) |step_index| { - const step = &w.make_steps[@intFromEnum(step_index)]; - for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { - const reaction_set = rs: { - const gop = try w.dir_table.getOrPut(gpa, path); - if (!gop.found_existing) { - var mount_id: MountId = undefined; - const dir_handle = getDirHandle(gpa, path, &mount_id) catch |err| switch (err) { - error.FileNotFound => { - std.debug.assert(w.dir_table.swapRemove(path)); - continue; - }, - else => return err, - }; - const fan_fd = blk: { - const fd_gop = try w.os.poll_fds.getOrPut(gpa, mount_id); - if (!fd_gop.found_existing) { - const fan_fd = std.posix.fanotify_init(.{ - .CLASS = .NOTIF, - .CLOEXEC = true, - .NONBLOCK = true, - .REPORT_NAME = true, - .REPORT_DIR_FID = true, - .REPORT_FID = true, - .REPORT_TARGET_FID = true, - }, 0) catch |err| switch (err) { - error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}), - else => |e| return e, - }; - fd_gop.value_ptr.* = .{ - .fd = fan_fd, - .events = std.posix.POLL.IN, - .revents = undefined, - }; - } - break :blk fd_gop.value_ptr.*.fd; - }; - // `dir_handle` may already be present in the table in - // the case that we have multiple Cache.Path instances - // that compare inequal but ultimately point to the same - // directory on the file system. - // In such case, we must revert adding this directory, but keep - // the additions to the step set. - const dh_gop = try w.os.handle_table.getOrPut(gpa, dir_handle); - if (dh_gop.found_existing) { - _ = w.dir_table.pop(); - } else { - assert(dh_gop.index == gop.index); - dh_gop.value_ptr.* = .{ .mount_id = mount_id, .reaction_set = .{} }; - posix.fanotify_mark(fan_fd, .{ - .ADD = true, - .ONLYDIR = true, - }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| { - fatal("unable to watch {f}: {s}", .{ path, @errorName(err) }); - }; - } - break :rs &dh_gop.value_ptr.reaction_set; - } - break :rs &w.os.handle_table.values()[gop.index].reaction_set; - }; - for (files.items) |basename| { - const gop = try reaction_set.getOrPut(gpa, basename); - if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step_index, w.generation); - } - } - } - - { - // Remove marks for files that are no longer inputs. - var i: usize = 0; - while (i < w.os.handle_table.entries.len) { - { - const reaction_set = &w.os.handle_table.values()[i].reaction_set; - var step_set_i: usize = 0; - while (step_set_i < reaction_set.entries.len) { - const step_set = &reaction_set.values()[step_set_i]; - var dirent_i: usize = 0; - while (dirent_i < step_set.entries.len) { - const generations = step_set.values(); - if (generations[dirent_i] == w.generation) { - dirent_i += 1; - continue; - } - step_set.swapRemoveAt(dirent_i); - } - if (step_set.entries.len > 0) { - step_set_i += 1; - continue; - } - reaction_set.swapRemoveAt(step_set_i); - } - if (reaction_set.entries.len > 0) { - i += 1; - continue; - } - } - - const path = w.dir_table.keys()[i]; - - const mount_id = w.os.handle_table.values()[i].mount_id; - const fan_fd = w.os.poll_fds.getEntry(mount_id).?.value_ptr.fd; - posix.fanotify_mark(fan_fd, .{ - .REMOVE = true, - .ONLYDIR = true, - }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| switch (err) { - error.FileNotFound => {}, // Expected, harmless. - else => |e| std.log.warn("unable to unwatch '{f}': {s}", .{ path, @errorName(e) }), - }; - - w.dir_table.swapRemoveAt(i); - w.os.handle_table.swapRemoveAt(i); - } - w.generation +%= 1; - } - w.dir_count = w.dir_table.count(); - } - - fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - _ = io; - const events_len = try std.posix.poll(w.os.poll_fds.values(), timeout.to_i32_ms()); - if (events_len == 0) - return .timeout; - for (w.os.poll_fds.values()) |poll_fd| { - if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try markDirtySteps(w, gpa, poll_fd.fd)) - return .dirty; - } - return .clean; - } - }, - .windows => struct { - const windows = std.os.windows; - - /// Keyed differently but indexes correspond 1:1 with `dir_table`. - handle_table: std.ArrayHashMapUnmanaged(*Directory, void, Directory.TableAdapter, false), - ready_dirs: std.DoublyLinkedList, - - const FileId = struct { - volumeSerialNumber: windows.ULONG, - indexNumber: windows.LARGE_INTEGER, - }; - - const Directory = struct { - reaction_set: ReactionSet, - id: FileId, - file: Io.File, - state: enum { idle, listening, ready }, - iosb: windows.IO_STATUS_BLOCK, - // 64 KB is the packet size limit when monitoring over a network. - // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw#remarks - buffer: [64 * 1024]u8 align(@alignOf(windows.FILE.NOTIFY.INFORMATION)), - ready_node: std.DoublyLinkedList.Node, - - /// Start listening for events, buffer field will be overwritten eventually. - fn startListening(dir: *Directory, w: *Watch) !void { - assert(dir.file.flags.nonblocking); - assert(dir.state == .idle); - switch (windows.ntdll.NtNotifyChangeDirectoryFileEx( - dir.file.handle, - null, - ¬ifyApc, - w, - &dir.iosb, - &dir.buffer, - dir.buffer.len, - .{ - .FILE_NAME = true, - .DIR_NAME = true, - .SIZE = true, - .LAST_WRITE = true, - .CREATION = true, - }, - .FALSE, - .Notify, - )) { - .SUCCESS, .PENDING => dir.state = .listening, - .ILLEGAL_FUNCTION => return error.ReadDirectoryChangesUnsupported, - else => |status| return windows.unexpectedStatus(status), - } - } - - fn notifyApc(apc_context: ?*anyopaque, iosb: *windows.IO_STATUS_BLOCK, _: windows.ULONG) align(std.Io.Threaded.apc_align) callconv(.winapi) void { - const w: *Watch = @ptrCast(@alignCast(apc_context)); - const dir: *Directory = @fieldParentPtr("iosb", iosb); - assert(iosb.u.Status != .PENDING); - assert(dir.state == .listening); - w.os.ready_dirs.append(&dir.ready_node); - dir.state = .ready; - } - - fn init(gpa: Allocator, path: Cache.Path) !*Directory { - // The following code is a drawn out NtCreateFile call. (mostly adapted from Io.Dir.makeOpenDirAccessMaskW) - // It's necessary in order to get the specific flags that are required when calling ReadDirectoryChangesW. - var dir_handle: windows.HANDLE = undefined; - const root_fd = path.root_dir.handle.handle; - const sub_path = path.subPathOrDot(); - const sub_path_w = try Io.Threaded.sliceToPrefixedFileW(root_fd, sub_path, .{}); // TODO eliminate this call - var iosb: windows.IO_STATUS_BLOCK = undefined; - switch (windows.ntdll.NtCreateFile( - &dir_handle, - .{ - .SPECIFIC = .{ .FILE_DIRECTORY = .{ - .LIST = true, - } }, - .STANDARD = .{ .SYNCHRONIZE = true }, - .GENERIC = .{ .READ = true }, - }, - &.{ - .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w.span())) null else root_fd, - .ObjectName = @constCast(&sub_path_w.string()), - }, - &iosb, - null, - .{}, - .VALID_FLAGS, - .OPEN, - .{ - .DIRECTORY_FILE = true, - .IO = .ASYNCHRONOUS, - .OPEN_FOR_BACKUP_INTENT = true, - }, - null, - 0, - )) { - .SUCCESS => {}, - .OBJECT_NAME_INVALID => return error.BadPathName, - .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, - .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, - .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, - .NOT_A_DIRECTORY => return error.NotDir, - // This can happen if the directory has 'List folder contents' permission set to 'Deny' - .ACCESS_DENIED => return error.AccessDenied, - .INVALID_PARAMETER => unreachable, - else => |rc| return windows.unexpectedStatus(rc), - } - assert(dir_handle != windows.INVALID_HANDLE_VALUE); - errdefer windows.CloseHandle(dir_handle); - - const dir_id = try getFileId(dir_handle); - - const dir = try gpa.create(Directory); - dir.* = .{ - .reaction_set = .empty, - .id = dir_id, - .file = .{ .handle = dir_handle, .flags = .{ .nonblocking = true } }, - .state = .idle, - .iosb = undefined, - .buffer = undefined, - .ready_node = undefined, - }; - return dir; - } - - fn deinit(dir: *Directory, gpa: Allocator, w: *Watch) void { - state: switch (dir.state) { - .idle => {}, - .listening => { - var cancel_iosb: windows.IO_STATUS_BLOCK = undefined; - _ = windows.ntdll.NtCancelIoFileEx(dir.file.handle, &dir.iosb, &cancel_iosb); - while (switch (dir.state) { - .idle => unreachable, - .listening => true, - .ready => false, - }) Io.Threaded.waitForApcOrAlert(); - continue :state .ready; - }, - .ready => w.os.ready_dirs.remove(&dir.ready_node), - } - windows.CloseHandle(dir.file.handle); - gpa.destroy(dir); - } - - /// Useful to make `*Directory` a key in `std.ArrayHashMap`. - const TableAdapter = struct { - pub fn hash(_: TableAdapter, lhs_dir: *Directory) u32 { - return @truncate(Hash.hash(lhs_dir.id.volumeSerialNumber, @ptrCast(&lhs_dir.id.indexNumber))); - } - pub fn eql(_: TableAdapter, lhs_dir: *Directory, rhs_dir: *Directory, rhs_index: usize) bool { - _ = rhs_index; - return lhs_dir.id.volumeSerialNumber == rhs_dir.id.volumeSerialNumber and - lhs_dir.id.indexNumber == rhs_dir.id.indexNumber; - } - }; - }; - - fn init(cwd_path: []const u8) !Watch { - _ = cwd_path; - return .{ - .dir_table = .{}, - .dir_count = 0, - .os = switch (builtin.os.tag) { - .windows => .{ - .handle_table = .empty, - .ready_dirs = .{}, - }, - else => {}, - }, - .generation = 0, - }; - } - - fn getFileId(handle: windows.HANDLE) !FileId { - var file_id: FileId = undefined; - var io_status: windows.IO_STATUS_BLOCK = undefined; - var volume_info: windows.FILE.FS_VOLUME_INFORMATION = undefined; - switch (windows.ntdll.NtQueryVolumeInformationFile( - handle, - &io_status, - &volume_info, - @sizeOf(windows.FILE.FS_VOLUME_INFORMATION), - .Volume, - )) { - .SUCCESS => {}, - // Buffer overflow here indicates that there is more information available than was able to be stored in the buffer - // size provided. This is treated as success because the type of variable-length information that this would be relevant for - // (name, volume name, etc) we don't care about. - .BUFFER_OVERFLOW => {}, - else => |rc| return windows.unexpectedStatus(rc), - } - file_id.volumeSerialNumber = volume_info.VolumeSerialNumber; - var internal_info: windows.FILE.INTERNAL_INFORMATION = undefined; - switch (windows.ntdll.NtQueryInformationFile( - handle, - &io_status, - &internal_info, - @sizeOf(windows.FILE.INTERNAL_INFORMATION), - .Internal, - )) { - .SUCCESS => {}, - else => |rc| return windows.unexpectedStatus(rc), - } - file_id.indexNumber = internal_info.IndexNumber; - return file_id; - } - - fn markDirtySteps(w: *Watch, gpa: Allocator, dir: *Directory) !bool { - var any_dirty = false; - const bytes_returned = dir.iosb.Information; - if (bytes_returned == 0) { - std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); - markAllFilesDirty(w, gpa); - try dir.startListening(w); - return true; - } - var file_name_buf: [std.fs.max_path_bytes]u8 = undefined; - var offset: usize = 0; - while (true) { - const notify: *windows.FILE.NOTIFY.INFORMATION = @ptrCast(@alignCast(&dir.buffer[offset])); - const file_name = file_name_buf[0..std.unicode.wtf16LeToWtf8(&file_name_buf, notify.fileName())]; - if (dir.reaction_set.getPtr(".")) |glob_set| - any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); - if (dir.reaction_set.getPtr(file_name)) |step_set| - any_dirty = markStepSetDirty(gpa, step_set, any_dirty); - if (notify.NextEntryOffset == 0) - break; - - offset += notify.NextEntryOffset; - } - - // We call this now since at this point we have finished reading dir.buffer. - try dir.startListening(w); - return any_dirty; - } - - fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { - // Add missing marks and note persisted ones. - for (steps) |step| { - for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { - const dir = dir: { - const gop = try w.dir_table.getOrPut(gpa, path); - if (!gop.found_existing) { - const dir: *Directory = try .init(gpa, path); - errdefer dir.deinit(gpa, w); - // `dir.id` may already be present in the table in - // the case that we have multiple Cache.Path instances - // that compare inequal but ultimately point to the same - // directory on the file system. - // In such case, we must revert adding this directory, but keep - // the additions to the step set. - const dh_gop = try w.os.handle_table.getOrPut(gpa, dir); - if (dh_gop.found_existing) { - dir.deinit(gpa, w); - _ = w.dir_table.pop(); - break :dir w.os.handle_table.keys()[dh_gop.index]; - } else { - assert(dh_gop.index == gop.index); - try dir.startListening(w); - break :dir dir; - } - } - break :dir w.os.handle_table.keys()[gop.index]; - }; - for (files.items) |basename| { - const gop = try dir.reaction_set.getOrPut(gpa, basename); - if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step, w.generation); - } - } - } - - { - // Remove marks for files that are no longer inputs. - var i: usize = 0; - while (i < w.os.handle_table.entries.len) { - const dir = w.os.handle_table.keys()[i]; - { - var step_set_i: usize = 0; - while (step_set_i < dir.reaction_set.entries.len) { - const step_set = &dir.reaction_set.values()[step_set_i]; - var dirent_i: usize = 0; - while (dirent_i < step_set.entries.len) { - const generations = step_set.values(); - if (generations[dirent_i] == w.generation) { - dirent_i += 1; - continue; - } - step_set.swapRemoveAt(dirent_i); - } - if (step_set.entries.len > 0) { - step_set_i += 1; - continue; - } - dir.reaction_set.swapRemoveAt(step_set_i); - } - if (dir.reaction_set.entries.len > 0) { - i += 1; - continue; - } - } - - w.dir_table.swapRemoveAt(i); - w.os.handle_table.swapRemoveAt(i); - dir.deinit(gpa, w); - } - w.generation +%= 1; - } - w.dir_count = w.dir_table.count(); - } - - fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - for (0..2) |attempt| { - while (w.os.ready_dirs.popFirst()) |ready_node| { - const dir: *Directory = @fieldParentPtr("ready_node", ready_node); - assert(dir.state == .ready); - dir.state = .idle; - switch (dir.iosb.u.Status) { - .SUCCESS => return if (try markDirtySteps(w, gpa, dir)) .dirty else .clean, - .PENDING => unreachable, - .CANCELLED => {}, - else => |status| return windows.unexpectedStatus(status), - } - try dir.startListening(w); - } - try io.checkCancel(); - if (attempt == 1) return .timeout; - const delay_interval: windows.LARGE_INTEGER = switch (timeout) { - .none => std.math.minInt(windows.LARGE_INTEGER), - .ms => |ms| -@as(windows.LARGE_INTEGER, ms) * (std.time.ns_per_ms / 100), - }; - _ = windows.ntdll.NtDelayExecution(.TRUE, &delay_interval); - } else unreachable; - } - }, - .dragonfly, .freebsd, .netbsd, .openbsd, .ios, .tvos, .visionos, .watchos => struct { - const posix = std.posix; - - kq_fd: i32, - /// Indexes correspond 1:1 with `dir_table`. - handles: std.MultiArrayList(struct { - rs: ReactionSet, - /// If the corresponding dir_table Path has sub_path == "", then it - /// suffices as the open directory handle, and this value will be - /// -1. Otherwise, it needs to be opened in update(), and will be - /// stored here. - dir_fd: i32, - }), - - const dir_open_flags: posix.O = f: { - var f: posix.O = .{ - .ACCMODE = .RDONLY, - .NOFOLLOW = false, - .DIRECTORY = true, - .CLOEXEC = true, - }; - if (@hasField(posix.O, "EVTONLY")) f.EVTONLY = true; - if (@hasField(posix.O, "PATH")) f.PATH = true; - break :f f; - }; - - const EV = std.c.EV; - const NOTE = std.c.NOTE; - - fn init(cwd_path: []const u8) !Watch { - _ = cwd_path; - return .{ - .dir_table = .{}, - .dir_count = 0, - .os = .{ - .kq_fd = try Io.Kqueue.createFileDescriptor(), - .handles = .empty, - }, - .generation = 0, - }; - } - - fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { - const handles = &w.os.handles; - for (steps) |step| { - for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { - const reaction_set = rs: { - const gop = try w.dir_table.getOrPut(gpa, path); - if (!gop.found_existing) { - const skip_open_dir = path.sub_path.len == 0; - const dir_fd = if (skip_open_dir) - path.root_dir.handle.handle - else - posix.openat(path.root_dir.handle.handle, path.sub_path, dir_open_flags, 0) catch |err| { - fatal("failed to open directory {f}: {t}", .{ path, err }); - }; - // Empirically the dir has to stay open or else no events are triggered. - errdefer if (!skip_open_dir) std.Io.Threaded.closeFd(dir_fd); - const changes = [1]posix.Kevent{.{ - .ident = @bitCast(@as(isize, dir_fd)), - .filter = std.c.EVFILT.VNODE, - .flags = EV.ADD | EV.ENABLE | EV.CLEAR, - .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE, - .data = 0, - .udata = gop.index, - }}; - _ = try Io.Kqueue.kevent(w.os.kq_fd, &changes, &.{}, null); - assert(handles.len == gop.index); - try handles.append(gpa, .{ - .rs = .{}, - .dir_fd = if (skip_open_dir) -1 else dir_fd, - }); - } - - break :rs &handles.items(.rs)[gop.index]; - }; - for (files.items) |basename| { - const gop = try reaction_set.getOrPut(gpa, basename); - if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step, w.generation); - } - } - } - - { - // Remove marks for files that are no longer inputs. - var i: usize = 0; - while (i < handles.len) { - { - const reaction_set = &handles.items(.rs)[i]; - var step_set_i: usize = 0; - while (step_set_i < reaction_set.entries.len) { - const step_set = &reaction_set.values()[step_set_i]; - var dirent_i: usize = 0; - while (dirent_i < step_set.entries.len) { - const generations = step_set.values(); - if (generations[dirent_i] == w.generation) { - dirent_i += 1; - continue; - } - step_set.swapRemoveAt(dirent_i); - } - if (step_set.entries.len > 0) { - step_set_i += 1; - continue; - } - reaction_set.swapRemoveAt(step_set_i); - } - if (reaction_set.entries.len > 0) { - i += 1; - continue; - } - } - - // If the sub_path == "" then this patch has already the - // dir fd that we need to use as the ident to remove the - // event. If it was opened above with openat() then we need - // to access that data via the dir_fd field. - const path = w.dir_table.keys()[i]; - const dir_fd = if (path.sub_path.len == 0) - path.root_dir.handle.handle - else - handles.items(.dir_fd)[i]; - assert(dir_fd != -1); - - // The changelist also needs to update the udata field of the last - // event, since we are doing a swap remove, and we store the dir_table - // index in the udata field. - const last_dir_fd = fd: { - const last_path = w.dir_table.keys()[handles.len - 1]; - const last_dir_fd = if (last_path.sub_path.len == 0) - last_path.root_dir.handle.handle - else - handles.items(.dir_fd)[handles.len - 1]; - assert(last_dir_fd != -1); - break :fd last_dir_fd; - }; - const changes = [_]posix.Kevent{ - .{ - .ident = @bitCast(@as(isize, dir_fd)), - .filter = std.c.EVFILT.VNODE, - .flags = EV.DELETE, - .fflags = 0, - .data = 0, - .udata = i, - }, - .{ - .ident = @bitCast(@as(isize, last_dir_fd)), - .filter = std.c.EVFILT.VNODE, - .flags = EV.ADD, - .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE, - .data = 0, - .udata = i, - }, - }; - const filtered_changes = if (i == handles.len - 1) changes[0..1] else &changes; - _ = try Io.Kqueue.kevent(w.os.kq_fd, filtered_changes, &.{}, null); - if (path.sub_path.len != 0) std.Io.Threaded.closeFd(dir_fd); - - w.dir_table.swapRemoveAt(i); - handles.swapRemove(i); - } - w.generation +%= 1; - } - w.dir_count = w.dir_table.count(); - } - - fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - _ = io; - var timespec_buffer: posix.timespec = undefined; - var event_buffer: [100]posix.Kevent = undefined; - var n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, timeout.toTimespec(×pec_buffer)); - if (n == 0) return .timeout; - const reaction_sets = w.os.handles.items(.rs); - var any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], false); - timespec_buffer = .{ .sec = 0, .nsec = 0 }; - while (n == event_buffer.len) { - n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, ×pec_buffer); - if (n == 0) break; - any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], any_dirty); - } - return if (any_dirty) .dirty else .clean; - } - - fn markDirtySteps( - gpa: Allocator, - reaction_sets: []ReactionSet, - events: []const std.c.Kevent, - start_any_dirty: bool, - ) bool { - var any_dirty = start_any_dirty; - for (events) |event| { - const index: usize = @intCast(event.udata); - const reaction_set = &reaction_sets[index]; - // If we knew the basename of the changed file, here we would - // mark only the step set dirty, and possibly the glob set: - //if (reaction_set.getPtr(".")) |glob_set| - // any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); - //if (reaction_set.getPtr(file_name)) |step_set| - // any_dirty = markStepSetDirty(gpa, step_set, any_dirty); - // However we don't know the file name so just mark all the - // sets dirty for this directory. - for (reaction_set.values()) |*step_set| { - any_dirty = markStepSetDirty(gpa, step_set, any_dirty); - } - } - return any_dirty; - } - }, - .macos => struct { - fse: FsEvents, - - fn init(cwd_path: []const u8) !Watch { - return .{ - .os = .{ .fse = try .init(cwd_path) }, - .dir_count = 0, - .dir_table = undefined, - .generation = undefined, - }; - } - fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { - try w.os.fse.setPaths(gpa, steps); - w.dir_count = w.os.fse.watch_roots.len; - } - fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - _ = io; - return w.os.fse.wait(gpa, switch (timeout) { - .none => null, - .ms => |ms| @as(u64, ms) * std.time.ns_per_ms, - }); - } - }, - else => void, -}; - -pub fn init(cwd_path: []const u8, configuration: *const Configuration, make_steps: []Step) !Watch { - return Os.init(cwd_path, configuration, make_steps); -} - -pub const Match = struct { - /// Relative to the watched directory, the file path that triggers this - /// match. - basename: []const u8, - /// The step to re-run when file corresponding to `basename` is changed. - step_index: Configuration.Step.Index, - - pub const Context = struct { - pub fn hash(self: Context, a: Match) u32 { - _ = self; - var hasher = Hash.init(@intFromEnum(a.step_index)); - hasher.update(a.basename); - return @truncate(hasher.final()); - } - pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { - _ = self; - _ = b_index; - return a.step_index == b.step_index and std.mem.eql(u8, a.basename, b.basename); - } - }; -}; - -fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { - for (switch (builtin.os.tag) { - .windows => w.os.handle_table.keys(), - else => w.os.handle_table.values(), - }) |item| { - const reaction_set = switch (builtin.os.tag) { - .linux, .windows => item.reaction_set, - else => item, - }; - for (reaction_set.values()) |step_set| { - for (step_set.keys()) |step_index| { - const step = &w.make_steps[@intFromEnum(step_index)]; - _ = step.invalidateResult(gpa); - } - } - } -} - -fn markStepSetDirty(gpa: Allocator, make_steps: []Step, step_set: *StepSet, any_dirty: bool) bool { - var this_any_dirty = false; - for (step_set.keys()) |step_index| { - const step = &make_steps[@intFromEnum(step_index)]; - if (step.invalidateResult(gpa)) this_any_dirty = true; - } - return any_dirty or this_any_dirty; -} - -pub fn update(w: *Watch, gpa: Allocator, steps: []const Configuration.Step.Index) !void { - return Os.update(w, gpa, steps); -} - -pub const Timeout = union(enum) { - none, - ms: u16, - - pub fn to_i32_ms(t: Timeout) i32 { - return switch (t) { - .none => -1, - .ms => |ms| ms, - }; - } - - pub fn toTimespec(t: Timeout, buf: *std.posix.timespec) ?*std.posix.timespec { - return switch (t) { - .none => null, - .ms => |ms_u16| { - const ms: isize = ms_u16; - buf.* = .{ - .sec = @divTrunc(ms, std.time.ms_per_s), - .nsec = @rem(ms, std.time.ms_per_s) * std.time.ns_per_ms, - }; - return buf; - }, - }; - } -}; - -pub const WaitResult = enum { - timeout, - /// File system watching triggered on files that were marked as inputs to at least one Step. - /// Relevant steps have been marked dirty. - dirty, - /// File system watching triggered but none of the events were relevant to - /// what we are listening to. There is nothing to do. - clean, -}; - -pub fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { - return Os.wait(w, gpa, io, timeout); -} diff --git a/lib/compiler/maker/Watch/FsEvents.zig b/lib/compiler/maker/Watch/FsEvents.zig deleted file mode 100644 index 0a56ce182255176222ef4b9a7a7bbb22abc9350d..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/Watch/FsEvents.zig +++ /dev/null @@ -1,479 +0,0 @@ -//! An implementation of file-system watching based on the `FSEventStream` API in macOS. -//! While macOS supports kqueue, it does not allow detecting changes to files without -//! placing watches on each individual file, meaning FD limits are reached incredibly -//! quickly. The File System Events API works differently: it implements *recursive* -//! directory watches, managed by a system service. Rather than being in libc, the API is -//! exposed by the CoreServices framework. To avoid a compile dependency on the framework -//! bundle, we dynamically load CoreServices with `std.DynLib`. -//! -//! While the logic in this file *is* specialized to `std.Build.Watch`, efforts have been -//! made to keep that specialization to a minimum. Other use cases could be served with -//! relatively minimal modifications to the `watch_paths` field and its usages (in -//! particular the `setPaths` function). We avoid using the global GCD dispatch queue in -//! favour of creating our own and synchronizing with an explicit semaphore, meaning this -//! logic is thread-safe and does not affect process-global state. -//! -//! In theory, this API is quite good at avoiding filesystem race conditions. In practice, -//! the logic that would avoid them is currently disabled, because the build system kind -//! of relies on them at the time of writing to avoid redundant work -- see the comment at -//! the top of `wait` for details. - -const enable_debug_logs = false; - -core_services: std.DynLib, -resolved_symbols: ResolvedSymbols, - -paths_arena: std.heap.ArenaAllocator.State, -/// The roots of the recursive watches. FSEvents has relatively small limits on the number -/// of watched paths, so this slice must not be too long. The paths themselves are allocated -/// into `paths_arena`, but this slice is allocated into the GPA. -watch_roots: [][:0]const u8, -/// All of the paths being watched. Value is the set of steps which depend on the file/directory. -/// Keys and values are in `paths_arena`, but this map is allocated into the GPA. -watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step), - -/// The semaphore we use to block the thread calling `wait` until the callback determines a relevant -/// event has occurred. This is retained across `wait` calls for simplicity and efficiency. -waiting_semaphore: dispatch.semaphore_t, -/// This dispatch queue is created by us and executes serially. It exists exclusively to trigger the -/// callbacks of the FSEventStream we create. This is not in use outside of `wait`, but is retained -/// across `wait` calls for simplicity and efficiency. -dispatch_queue: dispatch.queue_t, -/// In theory, this field avoids race conditions. In practice, it is essentially unused at the time -/// of writing. See the comment at the start of `wait` for details. -since_event: FSEventStreamEventId, - -cwd_path: []const u8, - -/// All of the symbols we pull from the `dlopen`ed CoreServices framework. If any of these symbols -/// is not present, `init` will close the framework and return an error. -const ResolvedSymbols = struct { - FSEventStreamCreate: *const fn ( - allocator: CFAllocatorRef, - callback: FSEventStreamCallback, - ctx: ?*const FSEventStreamContext, - paths_to_watch: CFArrayRef, - since_when: FSEventStreamEventId, - latency: CFTimeInterval, - flags: FSEventStreamCreateFlags, - ) callconv(.c) FSEventStreamRef, - FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch.queue_t) callconv(.c) void, - FSEventStreamStart: *const fn (stream: FSEventStreamRef) callconv(.c) bool, - FSEventStreamStop: *const fn (stream: FSEventStreamRef) callconv(.c) void, - FSEventStreamInvalidate: *const fn (stream: FSEventStreamRef) callconv(.c) void, - FSEventStreamRelease: *const fn (stream: FSEventStreamRef) callconv(.c) void, - FSEventStreamGetLatestEventId: *const fn (stream: ConstFSEventStreamRef) callconv(.c) FSEventStreamEventId, - FSEventsGetCurrentEventId: *const fn () callconv(.c) FSEventStreamEventId, - CFRelease: *const fn (cf: *const anyopaque) callconv(.c) void, - CFArrayCreate: *const fn ( - allocator: CFAllocatorRef, - values: [*]const usize, - num_values: CFIndex, - call_backs: ?*const CFArrayCallBacks, - ) callconv(.c) CFArrayRef, - CFStringCreateWithCString: *const fn ( - alloc: CFAllocatorRef, - c_str: [*:0]const u8, - encoding: CFStringEncoding, - ) callconv(.c) CFStringRef, - CFAllocatorCreate: *const fn (allocator: CFAllocatorRef, context: *const CFAllocatorContext) callconv(.c) CFAllocatorRef, - kCFAllocatorUseContext: *const CFAllocatorRef, -}; - -pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol, SystemResources }!FsEvents { - var core_services = std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/CoreServices") catch - return error.OpenFrameworkFailed; - errdefer core_services.close(); - - var resolved_symbols: ResolvedSymbols = undefined; - inline for (@typeInfo(ResolvedSymbols).@"struct".fields) |f| { - @field(resolved_symbols, f.name) = core_services.lookup(f.type, f.name) orelse return error.MissingCoreServicesSymbol; - } - - return .{ - .core_services = core_services, - .resolved_symbols = resolved_symbols, - .paths_arena = .{}, - .watch_roots = &.{}, - .watch_paths = .empty, - .waiting_semaphore = dispatch.semaphore_create(0) orelse return error.SystemResources, - .dispatch_queue = dispatch.queue_create("zig-watch", .SERIAL()) orelse return error.SystemResources, - // Not `.since_now`, because this means we can init `FsEvents` *before* we do work in order - // to notice any changes which happened during said work. - .since_event = resolved_symbols.FSEventsGetCurrentEventId(), - .cwd_path = cwd_path, - }; -} - -pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void { - fse.waiting_semaphore.as_object().release(); - fse.dispatch_queue.as_object().release(); - fse.core_services.close(io); - - gpa.free(fse.watch_roots); - fse.watch_paths.deinit(gpa); - { - var paths_arena = fse.paths_arena.promote(gpa); - paths_arena.deinit(); - } -} - -pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) !void { - var paths_arena_instance = fse.paths_arena.promote(gpa); - defer fse.paths_arena = paths_arena_instance.state; - const paths_arena = paths_arena_instance.allocator(); - - var need_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; - defer need_dirs.deinit(gpa); - - fse.watch_paths.clearRetainingCapacity(); - - // We take `step` by pointer for a slight memory optimization in a moment. - for (steps) |*step| { - for (step.*.inputs.table.keys(), step.*.inputs.table.values()) |path, *files| { - const resolved_dir = try std.fs.path.resolvePosix(paths_arena, &.{ - fse.cwd_path, path.root_dir.path orelse ".", path.sub_path, - }); - try need_dirs.put(gpa, resolved_dir, {}); - for (files.items) |file_name| { - const watch_path = if (std.mem.eql(u8, file_name, ".")) - resolved_dir - else - try std.fs.path.join(paths_arena, &.{ resolved_dir, file_name }); - const gop = try fse.watch_paths.getOrPut(gpa, watch_path); - if (gop.found_existing) { - const old_steps = gop.value_ptr.*; - const new_steps = try paths_arena.alloc(*std.Build.Step, old_steps.len + 1); - @memcpy(new_steps[0..old_steps.len], old_steps); - new_steps[old_steps.len] = step.*; - gop.value_ptr.* = new_steps; - } else { - // This is why we captured `step` by pointer! We can avoid allocating a slice of one - // step in the arena in the common case where a file is referenced by only one step. - gop.value_ptr.* = step[0..1]; - } - } - } - } - - { - // There's no point looking at directories inside other ones (e.g. "/foo" and "/foo/bar"). - // To eliminate these, we'll re-add directories in order of path length with a redundancy check. - const old_dirs = try gpa.dupe([]const u8, need_dirs.keys()); - defer gpa.free(old_dirs); - std.mem.sort([]const u8, old_dirs, {}, struct { - fn lessThan(ctx: void, a: []const u8, b: []const u8) bool { - ctx; - return std.mem.lessThan(u8, a, b); - } - }.lessThan); - need_dirs.clearRetainingCapacity(); - for (old_dirs) |dir_path| { - var it: std.fs.path.ComponentIterator(.posix, u8) = .init(dir_path); - while (it.next()) |component| { - if (need_dirs.contains(component.path)) { - // this path is '/foo/bar/qux', but '/foo' or '/foo/bar' was already added - break; - } - } else { - need_dirs.putAssumeCapacityNoClobber(dir_path, {}); - } - } - } - - // `need_dirs` is now a set of directories to watch with no redundancy. In practice, this is very - // likely to have reduced it to a quite small set (e.g. it'll typically coalesce a full `src/` - // directory into one entry). However, the FSEventStream API has a fairly low undocumented limit - // on total watches (supposedly 4096), so we should handle the case where we exceed it. To be - // safe, because this API can be a little unpredictable, we'll cap ourselves a little *below* - // that known limit. - if (need_dirs.count() > 2048) { - // Fallback: watch the whole filesystem. This is excessive, but... it *works* :P - if (enable_debug_logs) watch_log.debug("too many dirs; recursively watching root", .{}); - fse.watch_roots = try gpa.realloc(fse.watch_roots, 1); - fse.watch_roots[0] = "/"; - } else { - fse.watch_roots = try gpa.realloc(fse.watch_roots, need_dirs.count()); - for (fse.watch_roots, need_dirs.keys()) |*out, in| { - out.* = try paths_arena.dupeSentinel(u8, in, 0); - } - } - if (enable_debug_logs) { - watch_log.debug("watching {d} paths using {d} recursive watches:", .{ fse.watch_paths.count(), fse.watch_roots.len }); - for (fse.watch_roots) |dir_path| { - watch_log.debug("- '{s}'", .{dir_path}); - } - } -} - -pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory, StartFailed }!std.Build.Watch.WaitResult { - if (fse.watch_roots.len == 0) @panic("nothing to watch"); - - const rs = fse.resolved_symbols; - - // At the time of writing, using `since_event` in the obvious way causes redundant rebuilds - // to occur, because one step modifies a file which is an input to another step. The solution - // to this problem will probably be either: - // - // a) Don't include the output of one step as a watch input of another; only mark external - // files as watch inputs. Or... - // - // b) Note the current event ID when a step begins, and disregard events preceding that ID - // when considering whether to dirty that step in `eventCallback`. - // - // For now, to avoid the redundant rebuilds, we bypass this `since_event` mechanism. This does - // introduce race conditions, but the other `std.Build.Watch` implementations suffer from those - // too at the time of writing, so this is kind of expected. - fse.since_event = .since_now; - - const cf_allocator = rs.CFAllocatorCreate(rs.kCFAllocatorUseContext.*, &.{ - .version = 0, - .info = @constCast(&gpa), - .retain = null, - .release = null, - .copy_description = null, - .allocate = &cf_alloc_callbacks.allocate, - .reallocate = &cf_alloc_callbacks.reallocate, - .deallocate = &cf_alloc_callbacks.deallocate, - .preferred_size = null, - }) orelse return error.OutOfMemory; - defer rs.CFRelease(cf_allocator); - - const cf_paths = try gpa.alloc(?CFStringRef, fse.watch_roots.len); - @memset(cf_paths, null); - defer { - for (cf_paths) |o| if (o) |p| rs.CFRelease(p); - gpa.free(cf_paths); - } - for (fse.watch_roots, cf_paths) |raw_path, *cf_path| { - cf_path.* = rs.CFStringCreateWithCString(cf_allocator, raw_path, .utf8); - } - const cf_paths_array = rs.CFArrayCreate(cf_allocator, @ptrCast(cf_paths), @intCast(cf_paths.len), null); - defer rs.CFRelease(cf_paths_array); - - const callback_ctx: EventCallbackCtx = .{ - .fse = fse, - .gpa = gpa, - }; - const event_stream = rs.FSEventStreamCreate( - null, - &eventCallback, - &.{ - .version = 0, - .info = @constCast(&callback_ctx), - .retain = null, - .release = null, - .copy_description = null, - }, - cf_paths_array, - fse.since_event, - 0.05, // 0.05s latency; higher values increase efficiency by coalescing more events - .{ .watch_root = true, .file_events = true }, - ); - defer rs.FSEventStreamRelease(event_stream); - rs.FSEventStreamSetDispatchQueue(event_stream, fse.dispatch_queue); - defer rs.FSEventStreamInvalidate(event_stream); - if (!rs.FSEventStreamStart(event_stream)) return error.StartFailed; - defer rs.FSEventStreamStop(event_stream); - const result = fse.waiting_semaphore.wait(timeout: { - const ns = timeout_ns orelse break :timeout .FOREVER; - break :timeout .time(.NOW, @intCast(ns)); - }); - return switch (result) { - 0 => .dirty, - else => .timeout, - }; -} - -const cf_alloc_callbacks = struct { - const log = std.log.scoped(.cf_alloc); - fn allocate(size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque { - if (enable_debug_logs) log.debug("allocate {d}", .{size}); - _ = hint; - const gpa: *const Allocator = @ptrCast(@alignCast(info)); - const mem = gpa.alignedAlloc(u8, .of(usize), @intCast(size + @sizeOf(usize))) catch return null; - const metadata: *usize = @ptrCast(mem); - metadata.* = @intCast(size); - return mem[@sizeOf(usize)..].ptr; - } - fn reallocate(ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque { - if (enable_debug_logs) log.debug("reallocate @{*} {d}", .{ ptr, new_size }); - _ = hint; - if (ptr == null or new_size == 0) return null; // not a bug: documentation explicitly states that realloc on NULL should return NULL - const gpa: *const Allocator = @ptrCast(@alignCast(info)); - const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize)); - const old_size = @as(*const usize, @ptrCast(old_base)).*; - const old_mem = old_base[0 .. old_size + @sizeOf(usize)]; - const new_mem = gpa.realloc(old_mem, @intCast(new_size + @sizeOf(usize))) catch return null; - const metadata: *usize = @ptrCast(new_mem); - metadata.* = @intCast(new_size); - return new_mem[@sizeOf(usize)..].ptr; - } - fn deallocate(ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void { - if (enable_debug_logs) log.debug("deallocate @{*}", .{ptr}); - const gpa: *const Allocator = @ptrCast(@alignCast(info)); - const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize)); - const old_size = @as(*const usize, @ptrCast(old_base)).*; - const old_mem = old_base[0 .. old_size + @sizeOf(usize)]; - gpa.free(old_mem); - } -}; - -const EventCallbackCtx = struct { - fse: *FsEvents, - gpa: Allocator, -}; - -fn eventCallback( - stream: ConstFSEventStreamRef, - client_callback_info: ?*anyopaque, - num_events: usize, - events_paths_ptr: *anyopaque, - events_flags_ptr: [*]const FSEventStreamEventFlags, - events_ids_ptr: [*]const FSEventStreamEventId, -) callconv(.c) void { - const ctx: *const EventCallbackCtx = @ptrCast(@alignCast(client_callback_info)); - const fse = ctx.fse; - const gpa = ctx.gpa; - const rs = fse.resolved_symbols; - const events_paths_ptr_casted: [*]const [*:0]const u8 = @ptrCast(@alignCast(events_paths_ptr)); - const events_paths = events_paths_ptr_casted[0..num_events]; - const events_ids = events_ids_ptr[0..num_events]; - const events_flags = events_flags_ptr[0..num_events]; - var any_dirty = false; - for (events_paths, events_ids, events_flags) |event_path_nts, event_id, event_flags| { - _ = event_id; - if (event_flags.history_done) continue; // sentinel - const event_path = std.mem.span(event_path_nts); - switch (event_flags.must_scan_sub_dirs) { - false => { - if (fse.watch_paths.get(event_path)) |steps| { - assert(steps.len > 0); - for (steps) |s| { - if (s.invalidateResult(gpa)) any_dirty = true; - } - } - if (std.fs.path.dirname(event_path)) |event_dirname| { - // Modifying '/foo/bar' triggers the watch on '/foo'. - if (fse.watch_paths.get(event_dirname)) |steps| { - assert(steps.len > 0); - for (steps) |s| { - if (s.invalidateResult(gpa)) any_dirty = true; - } - } - } - }, - true => { - // This is unlikely, but can occasionally happen when bottlenecked: events have been - // coalesced into one. We want to see if any of these events are actually relevant - // to us. The only way we can reasonably do that in this rare edge case is iterate - // the watch paths and see if any is under this directory. That's acceptable because - // we would otherwise kick off a rebuild which would be clearing those paths anyway. - const changed_path = std.fs.path.dirname(event_path) orelse event_path; - for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| { - if (dirStartsWith(watching_path, changed_path)) { - for (steps) |s| { - if (s.invalidateResult(gpa)) any_dirty = true; - } - } - } - }, - } - } - if (any_dirty) { - fse.since_event = rs.FSEventStreamGetLatestEventId(stream); - _ = fse.waiting_semaphore.signal(); - } -} -fn dirStartsWith(path: []const u8, prefix: []const u8) bool { - if (std.mem.eql(u8, path, prefix)) return true; - if (!std.mem.startsWith(u8, path, prefix)) return false; - if (path[prefix.len] != '/') return false; // `path` is `/foo/barx`, `prefix` is `/foo/bar` - return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar` -} - -const CFAllocatorRef = ?*const opaque {}; -const CFArrayRef = *const opaque {}; -const CFStringRef = *const opaque {}; -const CFTimeInterval = f64; -const CFIndex = i32; -const CFOptionFlags = enum(u32) { _ }; -const CFAllocatorRetainCallBack = *const fn (info: ?*const anyopaque) callconv(.c) *const anyopaque; -const CFAllocatorReleaseCallBack = *const fn (info: ?*const anyopaque) callconv(.c) void; -const CFAllocatorCopyDescriptionCallBack = *const fn (info: ?*const anyopaque) callconv(.c) CFStringRef; -const CFAllocatorAllocateCallBack = *const fn (alloc_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque; -const CFAllocatorReallocateCallBack = *const fn (ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque; -const CFAllocatorDeallocateCallBack = *const fn (ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void; -const CFAllocatorPreferredSizeCallBack = *const fn (size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) CFIndex; -const CFAllocatorContext = extern struct { - version: CFIndex, - info: ?*anyopaque, - retain: ?CFAllocatorRetainCallBack, - release: ?CFAllocatorReleaseCallBack, - copy_description: ?CFAllocatorCopyDescriptionCallBack, - allocate: CFAllocatorAllocateCallBack, - reallocate: ?CFAllocatorReallocateCallBack, - deallocate: ?CFAllocatorDeallocateCallBack, - preferred_size: ?CFAllocatorPreferredSizeCallBack, -}; -const CFArrayCallBacks = opaque {}; -const CFStringEncoding = enum(u32) { - invalid_id = std.math.maxInt(u32), - mac_roman = 0, - windows_latin_1 = 0x500, - iso_latin_1 = 0x201, - next_step_latin = 0xB01, - ascii = 0x600, - unicode = 0x100, - utf8 = 0x8000100, - non_lossy_ascii = 0xBFF, -}; - -const FSEventStreamRef = *opaque {}; -const ConstFSEventStreamRef = *const @typeInfo(FSEventStreamRef).pointer.child; -const FSEventStreamCallback = *const fn ( - stream: ConstFSEventStreamRef, - client_callback_info: ?*anyopaque, - num_events: usize, - event_paths: *anyopaque, - event_flags: [*]const FSEventStreamEventFlags, - event_ids: [*]const FSEventStreamEventId, -) callconv(.c) void; -const FSEventStreamContext = extern struct { - version: CFIndex, - info: ?*anyopaque, - retain: ?CFAllocatorRetainCallBack, - release: ?CFAllocatorReleaseCallBack, - copy_description: ?CFAllocatorCopyDescriptionCallBack, -}; -const FSEventStreamEventId = enum(u64) { - since_now = std.math.maxInt(u64), - _, -}; -const FSEventStreamCreateFlags = packed struct(u32) { - use_cf_types: bool = false, - no_defer: bool = false, - watch_root: bool = false, - ignore_self: bool = false, - file_events: bool = false, - _: u27 = 0, -}; -const FSEventStreamEventFlags = packed struct(u32) { - must_scan_sub_dirs: bool, - user_dropped: bool, - kernel_dropped: bool, - event_ids_wrapped: bool, - history_done: bool, - root_changed: bool, - mount: bool, - unmount: bool, - _: u24 = 0, -}; - -const dispatch = std.c.dispatch; -const std = @import("std"); -const Io = std.Io; -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; -const watch_log = std.log.scoped(.watch); -const FsEvents = @This(); diff --git a/lib/compiler/maker/WebServer.zig b/lib/compiler/maker/WebServer.zig deleted file mode 100644 index fd3806e8b2b925f6badad2830ddd8d780abab2b4..0000000000000000000000000000000000000000 --- a/lib/compiler/maker/WebServer.zig +++ /dev/null @@ -1,940 +0,0 @@ -const WebServer = @This(); - -const builtin = @import("builtin"); - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const Configuration = std.Build.Configuration; -const Io = std.Io; -const abi = std.Build.abi; -const assert = std.debug.assert; -const http = std.http; -const log = std.log.scoped(.web_server); -const mem = std.mem; -const net = std.Io.net; - -const Fuzz = @import("Fuzz.zig"); -const Graph = @import("Graph.zig"); -const Step = @import("Step.zig"); - -gpa: Allocator, -graph: *const Graph, -all_steps: []const Configuration.Step.Index, -listen_address: net.IpAddress, -root_prog_node: std.Progress.Node, -watch: bool, - -tcp_server: ?net.Server, -serve_task: ?Io.Future(Io.Cancelable!void), - -/// Uses `Io.Clock.awake`. -base_timestamp: Io.Timestamp, -/// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`. -step_names_trailing: []u8, - -/// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps. -/// Accessed atomically. -step_status_bits: []u8, - -fuzz: ?Fuzz, -time_report_mutex: Io.Mutex, -time_report_msgs: [][]u8, -time_report_update_times: []i64, - -build_status: std.atomic.Value(abi.BuildStatus), -/// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate` -/// to increment this value. Each client thread waits for this increment with `Io.futexWaitTimeout`, so -/// `notifyUpdate` will wake those threads. Updates are sent on a short interval regardless, so it -/// is recommended to only use `notifyUpdate` for changes which the user should see immediately. For -/// instance, we do not call `notifyUpdate` when the number of "unique runs" in the fuzzer changes, -/// because this value changes quickly so this would result in constantly spamming all clients with -/// an unreasonable number of packets. -update_id: std.atomic.Value(u32), - -runner_request_mutex: Io.Mutex, -runner_request_ready_cond: Io.Condition, -runner_request_empty_cond: Io.Condition, -runner_request: ?RunnerRequest, - -/// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates -/// on a fixed interval of this many milliseconds. -const default_update_interval_ms = 500; - -pub const base_clock: Io.Clock = .awake; - -/// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`. -pub fn notifyUpdate(ws: *WebServer) void { - _ = ws.update_id.rmw(.Add, 1, .release); - ws.graph.io.futexWake(u32, &ws.update_id.raw, 16); -} - -pub const Options = struct { - gpa: Allocator, - graph: *const Graph, - all_steps: []const Configuration.Step.Index, - root_prog_node: std.Progress.Node, - watch: bool, - listen_address: net.IpAddress, - base_timestamp: Io.Clock.Timestamp, - configuration: *const Configuration, -}; -pub fn init(opts: Options) WebServer { - // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` - // instead of threads, so that the web server can function in single-threaded builds. - comptime assert(!builtin.single_threaded); - assert(opts.base_timestamp.clock == base_clock); - - const all_steps = opts.all_steps; - const c = opts.configuration; - - const step_names_trailing = opts.gpa.alloc(u8, len: { - var name_bytes: usize = 0; - for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len; - break :len name_bytes + all_steps.len * 4; - }) catch @panic("out of memory"); - { - const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]); - var idx: usize = all_steps.len * 4; - for (all_steps, step_name_lens) |step_index, *name_len| { - const step_name = step_index.ptr(c).name.slice(c); - name_len.* = @intCast(step_name.len); - @memcpy(step_names_trailing[idx..][0..step_name.len], step_name); - idx += step_name.len; - } - assert(idx == step_names_trailing.len); - } - - const step_status_bits = opts.gpa.alloc( - u8, - std.math.divCeil(usize, all_steps.len, 4) catch unreachable, - ) catch @panic("out of memory"); - @memset(step_status_bits, 0); - - const time_reports_len: usize = if (opts.graph.time_report) all_steps.len else 0; - const time_report_msgs = opts.gpa.alloc([]u8, time_reports_len) catch @panic("out of memory"); - const time_report_update_times = opts.gpa.alloc(i64, time_reports_len) catch @panic("out of memory"); - @memset(time_report_msgs, &.{}); - @memset(time_report_update_times, std.math.minInt(i64)); - - return .{ - .gpa = opts.gpa, - .graph = opts.graph, - .all_steps = all_steps, - .listen_address = opts.listen_address, - .root_prog_node = opts.root_prog_node, - .watch = opts.watch, - - .tcp_server = null, - .serve_task = null, - - .base_timestamp = opts.base_timestamp.raw, - .step_names_trailing = step_names_trailing, - - .step_status_bits = step_status_bits, - - .fuzz = null, - .time_report_mutex = .init, - .time_report_msgs = time_report_msgs, - .time_report_update_times = time_report_update_times, - - .build_status = .init(.idle), - .update_id = .init(0), - - .runner_request_mutex = .init, - .runner_request_ready_cond = .init, - .runner_request_empty_cond = .init, - .runner_request = null, - }; -} -pub fn deinit(ws: *WebServer) void { - const gpa = ws.gpa; - const io = ws.graph.io; - - gpa.free(ws.step_names_trailing); - gpa.free(ws.step_status_bits); - - if (ws.fuzz) |*f| f.deinit(); - for (ws.time_report_msgs) |msg| gpa.free(msg); - gpa.free(ws.time_report_msgs); - gpa.free(ws.time_report_update_times); - - if (ws.serve_task) |t| { - if (ws.tcp_server) |*s| s.stream.close(io); - t.await(); - } - if (ws.tcp_server) |*s| s.deinit(); - - gpa.free(ws.step_names_trailing); -} -pub fn start(ws: *WebServer) error{AlreadyReported}!void { - assert(ws.tcp_server == null); - assert(ws.serve_task == null); - const io = ws.graph.io; - - ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| { - log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err }); - return error.AlreadyReported; - }; - ws.serve_task = io.concurrent(serve, .{ws}) catch |err| { - log.err("unable to spawn web server thread: {t}", .{err}); - ws.tcp_server.?.deinit(io); - ws.tcp_server = null; - return error.AlreadyReported; - }; - - log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.socket.address}); - if (ws.listen_address.getPort() == 0) { - log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.socket.address}); - } -} -fn serve(ws: *WebServer) Io.Cancelable!void { - const io = ws.graph.io; - var group: Io.Group = .init; - defer group.cancel(io); - while (true) { - var stream = ws.tcp_server.?.accept(io) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| { - log.err("failed to accept connection: {t}", .{e}); - return; - }, - }; - group.concurrent(io, accept, .{ ws, stream }) catch |err| { - log.err("unable to spawn connection thread: {t}", .{err}); - stream.close(io); - continue; - }; - } -} - -pub fn startBuild(ws: *WebServer) void { - if (ws.fuzz) |*fuzz| { - fuzz.deinit(); - ws.fuzz = null; - } - for (ws.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic); - ws.build_status.store(.running, .monotonic); - ws.notifyUpdate(); -} - -pub fn updateStepStatus( - ws: *WebServer, - step_index: Configuration.Step.Index, - new_status: abi.StepUpdate.Status, -) void { - // TODO don't do linear search, especially in a hot loop like this - const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == step_index) break @intCast(i); - } else unreachable; - const ptr = &ws.step_status_bits[step_idx / 4]; - const bit_offset: u3 = @intCast((step_idx % 4) * 2); - const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset); - const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset; - _ = @atomicRmw(u8, ptr, .Xor, mask, .monotonic); - ws.notifyUpdate(); -} - -pub fn finishBuild(ws: *WebServer, opts: struct { - fuzz: bool, -}) void { - if (opts.fuzz) { - switch (builtin.os.tag) { - // Current implementation depends on two things that need to be ported to Windows: - // * Memory-mapping to share data between the fuzzer and build runner. - // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving - // many addresses to source locations). - .windows => std.process.fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), - else => {}, - } - if (@bitSizeOf(usize) != 64) { - // Current implementation depends on posix.mmap()'s second - // parameter, `length: usize`, being compatible with file system's - // u64 return value. This is not the case on 32-bit platforms. - // Affects or affected by issues #5185, #22523, and #22464. - std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); - } - - assert(ws.fuzz == null); - - ws.build_status.store(.fuzz_init, .monotonic); - ws.notifyUpdate(); - - ws.fuzz = Fuzz.init( - ws.gpa, - ws.graph.io, - ws.all_steps, - ws.root_prog_node, - .{ .forever = .{ .ws = ws } }, - ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)}); - ws.fuzz.?.start(); - } - - ws.build_status.store(if (ws.watch) .watching else .idle, .monotonic); - ws.notifyUpdate(); -} - -pub fn now(s: *const WebServer) i64 { - const io = s.graph.io; - const ts = base_clock.now(io); - return @intCast(s.base_timestamp.durationTo(ts).toNanoseconds()); -} - -fn accept(ws: *WebServer, stream: net.Stream) void { - const io = ws.graph.io; - defer { - // `net.Stream.close` wants to helpfully overwrite `stream` with - // `undefined`, but it cannot do so since it is an immutable parameter. - var copy = stream; - copy.close(io); - } - var send_buffer: [4096]u8 = undefined; - var recv_buffer: [4096]u8 = undefined; - var connection_reader = stream.reader(io, &recv_buffer); - var connection_writer = stream.writer(io, &send_buffer); - var server: http.Server = .init(&connection_reader.interface, &connection_writer.interface); - - while (true) { - var request = server.receiveHead() catch |err| switch (err) { - error.HttpConnectionClosing => return, - else => return log.err("failed to receive http request: {t}", .{err}), - }; - switch (request.upgradeRequested()) { - .websocket => |opt_key| { - const key = opt_key orelse return log.err("missing websocket key", .{}); - var web_socket = request.respondWebSocket(.{ .key = key }) catch { - return log.err("failed to respond web socket: {t}", .{connection_writer.err.?}); - }; - ws.serveWebSocket(&web_socket) catch |err| { - log.err("failed to serve websocket: {t}", .{err}); - return; - }; - comptime unreachable; - }, - .other => |name| return log.err("unknown upgrade request: {s}", .{name}), - .none => { - ws.serveRequest(&request) catch |err| switch (err) { - error.AlreadyReported => return, - else => { - log.err("failed to serve '{s}': {t}", .{ request.head.target, err }); - return; - }, - }; - }, - } - } -} - -fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { - const io = ws.graph.io; - - var prev_build_status = ws.build_status.load(.monotonic); - - const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len); - defer ws.gpa.free(prev_step_status_bits); - for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| { - copy.* = @atomicLoad(u8, shared, .monotonic); - } - - var recv_thread = try io.concurrent(recvWebSocketMessages, .{ ws, sock }); - defer recv_thread.cancel(io); - - { - const hello_header: abi.Hello = .{ - .status = prev_build_status, - .flags = .{ - .time_report = ws.graph.time_report, - }, - .timestamp = ws.now(), - .steps_len = @intCast(ws.all_steps.len), - }; - var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits }; - try sock.writeMessageVec(&bufs, .binary); - } - - var prev_fuzz: Fuzz.Previous = .init; - var prev_time: i64 = std.math.minInt(i64); - while (true) { - const start_time = ws.now(); - const start_update_id = ws.update_id.load(.acquire); - - if (ws.fuzz) |*fuzz| { - try fuzz.sendUpdate(sock, &prev_fuzz); - } - - { - try ws.time_report_mutex.lock(io); - defer ws.time_report_mutex.unlock(io); - for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| { - if (update_time <= prev_time) continue; - // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so - // that we don't hold up the build system on the client accepting this packet. - const owned_msg = try ws.gpa.dupe(u8, msg); - defer ws.gpa.free(owned_msg); - // Temporarily unlock, then re-lock after the message is sent. - ws.time_report_mutex.unlock(io); - defer ws.time_report_mutex.lockUncancelable(io); - try sock.writeMessage(owned_msg, .binary); - } - } - - { - const build_status = ws.build_status.load(.monotonic); - if (build_status != prev_build_status) { - prev_build_status = build_status; - const msg: abi.StatusUpdate = .{ .new = build_status }; - try sock.writeMessage(@ptrCast(&msg), .binary); - } - } - - for (prev_step_status_bits, ws.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| { - const cur_byte = @atomicLoad(u8, shared, .monotonic); - if (prev_byte.* == cur_byte) continue; - const cur: [4]abi.StepUpdate.Status = .{ - @enumFromInt(@as(u2, @truncate(cur_byte >> 0))), - @enumFromInt(@as(u2, @truncate(cur_byte >> 2))), - @enumFromInt(@as(u2, @truncate(cur_byte >> 4))), - @enumFromInt(@as(u2, @truncate(cur_byte >> 6))), - }; - const prev: [4]abi.StepUpdate.Status = .{ - @enumFromInt(@as(u2, @truncate(prev_byte.* >> 0))), - @enumFromInt(@as(u2, @truncate(prev_byte.* >> 2))), - @enumFromInt(@as(u2, @truncate(prev_byte.* >> 4))), - @enumFromInt(@as(u2, @truncate(prev_byte.* >> 6))), - }; - for (cur, prev, byte_idx * 4..) |cur_status, prev_status, step_idx| { - const msg: abi.StepUpdate = .{ .step_idx = @intCast(step_idx), .bits = .{ .status = cur_status } }; - if (cur_status != prev_status) try sock.writeMessage(@ptrCast(&msg), .binary); - } - prev_byte.* = cur_byte; - } - - prev_time = start_time; - - const old_cp = io.swapCancelProtection(.blocked); - defer _ = io.swapCancelProtection(old_cp); - io.futexWaitTimeout( - u32, - &ws.update_id.raw, - start_update_id, - .{ .duration = .{ - .clock = .awake, - .raw = .fromMilliseconds(default_update_interval_ms), - } }, - ) catch |err| switch (err) { - error.Canceled => unreachable, - }; - } -} -fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void { - const io = ws.graph.io; - - while (true) { - const msg = sock.readSmallMessage() catch return; - if (msg.opcode != .binary) continue; - if (msg.data.len == 0) continue; - const tag: abi.ToServerTag = @enumFromInt(msg.data[0]); - switch (tag) { - _ => continue, - .rebuild => while (true) { - ws.runner_request_mutex.lock(io) catch |err| switch (err) { - error.Canceled => return, - }; - defer ws.runner_request_mutex.unlock(io); - if (ws.runner_request == null) { - ws.runner_request = .rebuild; - ws.runner_request_ready_cond.signal(io); - break; - } - ws.runner_request_empty_cond.wait(io, &ws.runner_request_mutex) catch return; - }, - } - } -} - -fn serveRequest(ws: *WebServer, req: *http.Server.Request) !void { - // Strip an optional leading '/debug' component from the request. - const target: []const u8, const debug: bool = target: { - if (mem.eql(u8, req.head.target, "/debug")) break :target .{ "/", true }; - if (mem.eql(u8, req.head.target, "/debug/")) break :target .{ "/", true }; - if (mem.startsWith(u8, req.head.target, "/debug/")) break :target .{ req.head.target["/debug".len..], true }; - break :target .{ req.head.target, false }; - }; - - if (mem.eql(u8, target, "/")) return serveLibFile(ws, req, "build-web/index.html", "text/html"); - if (mem.eql(u8, target, "/main.js")) return serveLibFile(ws, req, "build-web/main.js", "application/javascript"); - if (mem.eql(u8, target, "/style.css")) return serveLibFile(ws, req, "build-web/style.css", "text/css"); - if (mem.eql(u8, target, "/time_report.css")) return serveLibFile(ws, req, "build-web/time_report.css", "text/css"); - if (mem.eql(u8, target, "/main.wasm")) return serveClientWasm(ws, req, if (debug) .Debug else .ReleaseFast); - - if (ws.fuzz) |*fuzz| { - if (mem.eql(u8, target, "/sources.tar")) return fuzz.serveSourcesTar(req); - } - - try req.respond("not found", .{ - .status = .not_found, - .extra_headers = &.{ - .{ .name = "Content-Type", .value = "text/plain" }, - }, - }); -} - -fn serveLibFile( - ws: *WebServer, - request: *http.Server.Request, - sub_path: []const u8, - content_type: []const u8, -) !void { - return serveFile(ws, request, .{ - .root_dir = ws.graph.zig_lib_directory, - .sub_path = sub_path, - }, content_type); -} -fn serveClientWasm( - ws: *WebServer, - req: *http.Server.Request, - optimize_mode: std.builtin.OptimizeMode, -) !void { - var arena_state: std.heap.ArenaAllocator = .init(ws.gpa); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // We always rebuild the wasm on-the-fly, so that if it is edited the user can just refresh the page. - const bin_path = try buildClientWasm(ws, arena, optimize_mode); - return serveFile(ws, req, bin_path, "application/wasm"); -} - -pub fn serveFile( - ws: *WebServer, - request: *http.Server.Request, - path: Cache.Path, - content_type: []const u8, -) !void { - const gpa = ws.gpa; - const io = ws.graph.io; - // The desired API is actually sendfile, which will require enhancing http.Server. - // We load the file with every request so that the user can make changes to the file - // and refresh the HTML page without restarting this server. - const file_contents = path.root_dir.handle.readFileAlloc(io, path.sub_path, gpa, .limited(10 * 1024 * 1024)) catch |err| { - log.err("failed to read '{f}': {t}", .{ path, err }); - return error.AlreadyReported; - }; - defer gpa.free(file_contents); - try request.respond(file_contents, .{ - .extra_headers = &.{ - .{ .name = "Content-Type", .value = content_type }, - cache_control_header, - }, - }); -} -pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void { - const graph = ws.graph; - const io = graph.io; - - var send_buffer: [0x4000]u8 = undefined; - var response = try request.respondStreaming(&send_buffer, .{ - .respond_options = .{ - .extra_headers = &.{ - .{ .name = "Content-Type", .value = "application/x-tar" }, - cache_control_header, - }, - }, - }); - - var archiver: std.tar.Writer = .{ .underlying_writer = &response.writer }; - - for (paths) |path| { - var file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| { - log.err("failed to open '{f}': {s}", .{ path, @errorName(err) }); - continue; - }; - defer file.close(io); - const stat = try file.stat(io); - var read_buffer: [1024]u8 = undefined; - var file_reader: Io.File.Reader = .initSize(file, io, &read_buffer, stat.size); - - // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can - // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI: - // it turns out the WASM treats the first path component as the module name, typically - // resulting in modules named "" and "src". The compiler needs to tell the build system - // about the module graph so that the build system can correctly encode this information in - // the tar file. - // - // Additionally, this needs to ensure that all path separators for both prefix and - // sub_path are using the POSIX-style `/` on platforms that don't use it as their native - // path separator. - archiver.prefix = path.root_dir.path orelse graph.cache.cwd; - try archiver.writeFile(path.sub_path, &file_reader, @intCast(stat.mtime.toSeconds())); - } - - // intentionally not calling `archiver.finishPedantically` - try response.end(); -} - -fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.OptimizeMode) !Cache.Path { - const root_name = "build-web"; - const arch_os_abi = "wasm32-freestanding"; - const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; - - const gpa = ws.gpa; - const graph = ws.graph; - const io = graph.io; - - const main_src_path: Cache.Path = .{ - .root_dir = graph.zig_lib_directory, - .sub_path = "build-web/main.zig", - }; - const walk_src_path: Cache.Path = .{ - .root_dir = graph.zig_lib_directory, - .sub_path = "docs/wasm/Walk.zig", - }; - const html_render_src_path: Cache.Path = .{ - .root_dir = graph.zig_lib_directory, - .sub_path = "docs/wasm/html_render.zig", - }; - - var argv: std.ArrayList([]const u8) = .empty; - - try argv.appendSlice(arena, &.{ - graph.zig_exe, "build-exe", // - "-fno-entry", // - "-O", @tagName(optimize), // - "-target", arch_os_abi, // - "-mcpu", cpu_features, // - "--cache-dir", graph.global_cache_root.path orelse ".", // - "--global-cache-dir", graph.global_cache_root.path orelse ".", // - "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // - "--name", root_name, // - "-rdynamic", // - "-fsingle-threaded", // - "--dep", "Walk", // - "--dep", "html_render", // - try std.fmt.allocPrint(arena, "-Mroot={f}", .{main_src_path}), // - try std.fmt.allocPrint(arena, "-MWalk={f}", .{walk_src_path}), // - "--dep", "Walk", // - try std.fmt.allocPrint(arena, "-Mhtml_render={f}", .{html_render_src_path}), // - "--listen=-", - }); - - var child = try std.process.spawn(io, .{ - .argv = argv.items, - .environ_map = &graph.environ_map, - .stdin = .pipe, - .stdout = .pipe, - .stderr = .pipe, - }); - defer child.kill(io); - - var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited }); - defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {}; - - var stdout_buffer: [512]u8 = undefined; - var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer); - const stdout = &stdout_reader.interface; - - { - var w = child.stdin.?.writer(io, &.{}); - w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - } - - const Header = std.zig.Server.Message.Header; - - var result: ?Cache.Path = null; - var result_error_bundle = std.zig.ErrorBundle.empty; - var body_buffer: std.ArrayList(u8) = .empty; - defer body_buffer.deinit(gpa); - - while (true) { - const header = stdout.takeStruct(Header, .little) catch |err| switch (err) { - error.ReadFailed => |e| return e, - error.EndOfStream => break, - }; - body_buffer.clearRetainingCapacity(); - try stdout.appendExact(gpa, &body_buffer, header.bytes_len); - const body = body_buffer.items; - - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) { - return error.ZigProtocolVersionMismatch; - } - }, - .error_bundle => { - result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body); - }, - .emit_digest => { - const EmitDigest = std.zig.Server.Message.EmitDigest; - const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body); - if (!ebp_hdr.flags.cache_hit) { - log.info("source changes detected; rebuilt wasm component", .{}); - } - const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; - result = .{ - .root_dir = graph.global_cache_root, - .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), - }; - }, - else => {}, // ignore other messages - } - } - - const stderr_contents = try stderr_task.await(io); - if (stderr_contents.len > 0) { - std.debug.print("{s}", .{stderr_contents}); - } - - // Send EOF to stdin. - child.stdin.?.close(io); - child.stdin = null; - - switch (try child.wait(io)) { - .exited => |code| { - if (code != 0) { - log.err( - "the following command exited with error code {d}:\n{s}", - .{ code, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, - ); - return error.WasmCompilationFailed; - } - }, - .signal => |sig| { - log.err( - "the following command terminated with signal {t}:\n{s}", - .{ sig, try Step.allocPrintCmd(arena, .inherit, null, argv.items) }, - ); - return error.WasmCompilationFailed; - }, - .stopped => |sig| { - log.err( - "the following command stopped unexpectedly with signal {t}:\n{s}", - .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, - ); - return error.WasmCompilationFailed; - }, - .unknown => { - log.err( - "the following command terminated unexpectedly:\n{s}", - .{try Step.allocPrintCmd(arena, .inherit, null, argv.items)}, - ); - return error.WasmCompilationFailed; - }, - } - - if (result_error_bundle.errorMessageCount() > 0) { - try result_error_bundle.renderToStderr(io, .{}, .auto); - log.err("the following command failed with {d} compilation errors:\n{s}", .{ - result_error_bundle.errorMessageCount(), - try Step.allocPrintCmd(arena, .inherit, null, argv.items), - }); - return error.WasmCompilationFailed; - } - - const base_path = result orelse { - log.err("child process failed to report result\n{s}", .{ - try Step.allocPrintCmd(arena, .inherit, null, argv.items), - }); - return error.WasmCompilationFailed; - }; - const bin_name = try std.zig.binNameAlloc(arena, .{ - .root_name = root_name, - .target = &(std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ - .arch_os_abi = arch_os_abi, - .cpu_features = cpu_features, - }) catch unreachable) catch unreachable), - .output_mode = .Exe, - }); - return base_path.join(arena, bin_name); -} - -fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 { - var file_reader: Io.File.Reader = .initStreaming(file, io, &.{}); - return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - else => |e| return e, - }; -} - -pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { - compile_step: Configuration.Step.Index, - - use_llvm: bool, - stats: abi.time_report.CompileResult.Stats, - ns_total: u64, - - llvm_pass_timings_len: u32, - files_len: u32, - decls_len: u32, - - /// The trailing data of `abi.time_report.CompileResult`, except the step name. - trailing: []const u8, -}) void { - const gpa = ws.gpa; - const io = ws.graph.io; - - // TODO don't do linear search - const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == opts.compile_step) break @intCast(i); - } else unreachable; - - const old_buf = old: { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - const old = ws.time_report_msgs[step_idx]; - ws.time_report_msgs[step_idx] = &.{}; - break :old old; - }; - const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory"); - - const out_header: *align(1) abi.time_report.CompileResult = @ptrCast(buf[0..@sizeOf(abi.time_report.CompileResult)]); - out_header.* = .{ - .step_idx = step_idx, - .flags = .{ - .use_llvm = opts.use_llvm, - }, - .stats = opts.stats, - .ns_total = opts.ns_total, - .llvm_pass_timings_len = opts.llvm_pass_timings_len, - .files_len = opts.files_len, - .decls_len = opts.decls_len, - }; - @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing); - - { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - assert(ws.time_report_msgs[step_idx].len == 0); - ws.time_report_msgs[step_idx] = buf; - ws.time_report_update_times[step_idx] = ws.now(); - } - ws.notifyUpdate(); -} - -pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void { - const gpa = ws.gpa; - const io = ws.graph.io; - - // TODO don't do linear search - const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == step_index) break @intCast(i); - } else unreachable; - - const old_buf = old: { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - const old = ws.time_report_msgs[step_idx]; - ws.time_report_msgs[step_idx] = &.{}; - break :old old; - }; - const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory"); - const out: *align(1) abi.time_report.GenericResult = @ptrCast(buf); - out.* = .{ - .step_idx = step_idx, - .ns_total = @intCast(duration.toNanoseconds()), - }; - { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - assert(ws.time_report_msgs[step_idx].len == 0); - ws.time_report_msgs[step_idx] = buf; - ws.time_report_update_times[step_idx] = ws.now(); - } - ws.notifyUpdate(); -} - -pub fn updateTimeReportRunTest( - ws: *WebServer, - run_step_index: Configuration.Step.Index, - tests: *const Step.Run.CachedTestMetadata, - ns_per_test: []const u64, -) void { - const gpa = ws.gpa; - const io = ws.graph.io; - - // TODO don't do linear search - const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { - if (s == run_step_index) break @intCast(i); - } else unreachable; - - assert(tests.names.len == ns_per_test.len); - const tests_len: u32 = @intCast(tests.names.len); - - const new_len: u64 = len: { - var names_len: u64 = 0; - for (0..tests_len) |i| { - names_len += tests.testName(@intCast(i)).len + 1; - } - break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len; - }; - const old_buf = old: { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - const old = ws.time_report_msgs[step_idx]; - ws.time_report_msgs[step_idx] = &.{}; - break :old old; - }; - const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory"); - - const out_header: *align(1) abi.time_report.RunTestResult = @ptrCast(buf[0..@sizeOf(abi.time_report.RunTestResult)]); - out_header.* = .{ - .step_idx = step_idx, - .tests_len = tests_len, - }; - var offset: usize = @sizeOf(abi.time_report.RunTestResult); - const ns_per_test_out: []align(1) u64 = @ptrCast(buf[offset..][0 .. tests_len * 8]); - @memcpy(ns_per_test_out, ns_per_test); - offset += tests_len * 8; - for (0..tests_len) |i| { - const name = tests.testName(@intCast(i)); - @memcpy(buf[offset..][0..name.len], name); - buf[offset..][name.len] = 0; - offset += name.len + 1; - } - assert(offset == buf.len); - - { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - assert(ws.time_report_msgs[step_idx].len == 0); - ws.time_report_msgs[step_idx] = buf; - ws.time_report_update_times[step_idx] = ws.now(); - } - ws.notifyUpdate(); -} - -const RunnerRequest = union(enum) { - rebuild, -}; -pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { - const io = ws.graph.io; - ws.runner_request_mutex.lock(io) catch return; - defer ws.runner_request_mutex.unlock(io); - if (ws.runner_request) |req| { - ws.runner_request = null; - ws.runner_request_empty_cond.signal(); - return req; - } - return null; -} -pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest { - const io = ws.graph.io; - try ws.runner_request_mutex.lock(io); - defer ws.runner_request_mutex.unlock(io); - while (true) { - if (ws.runner_request) |req| { - ws.runner_request = null; - ws.runner_request_empty_cond.signal(io); - return req; - } - try ws.runner_request_ready_cond.wait(io, &ws.runner_request_mutex); - } -} - -const cache_control_header: http.Header = .{ - .name = "Cache-Control", - .value = "max-age=0, must-revalidate", -}; diff --git a/src/main.zig b/src/main.zig index a24acdc14ec48af7be44fe003704912c12533649..1366deef2bc3646540643588915af21a1c571008 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5799,7 +5799,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn const main_mod_paths: Package.Module.CreateOptions.Paths = .{ .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"), - .root_src_path = "maker.zig", + .root_src_path = "Maker.zig", }; const config = try Compilation.Config.resolve(.{