| author |  Andrew Kelley <andrew@ziglang.org> 2026-02-15 16:03:37-08:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2026-05-25 18:54:33-07:00 |
| log | 2e88ac8842a6a5bbd9fe654292ab416cadfaf8bf |
| tree | 55716dcecb4e6c749261dbbcf19b0f1ea8f3ac7a |
| parent | e10cbf08eeed53561b5efe87a8cae0d7827f7301 |
zig build: configure runner basics implemented
37 files changed, 9743 insertions(+), 9540 deletions(-)
lib- src
lib/compiler/build_runner.zig deleted-1857
| ... | ... | @@ -1,1857 +0,0 @@ |
| 1 | const runner = @This(); | |
| 2 | const builtin = @import("builtin"); | |
| 3 | ||
| 4 | const std = @import("std"); | |
| 5 | const Io = std.Io; | |
| 6 | const assert = std.debug.assert; | |
| 7 | const fmt = std.fmt; | |
| 8 | const mem = std.mem; | |
| 9 | const process = std.process; | |
| 10 | const File = std.Io.File; | |
| 11 | const Step = std.Build.Step; | |
| 12 | const Watch = std.Build.Watch; | |
| 13 | const WebServer = std.Build.WebServer; | |
| 14 | const Allocator = std.mem.Allocator; | |
| 15 | const fatal = std.process.fatal; | |
| 16 | const Writer = std.Io.Writer; | |
| 17 | ||
| 18 | pub const root = @import("@build"); | |
| 19 | pub const dependencies = @import("@dependencies"); | |
| 20 | ||
| 21 | pub const std_options: std.Options = .{ | |
| 22 | .side_channels_mitigations = .none, | |
| 23 | .http_disable_tls = true, | |
| 24 | }; | |
| 25 | ||
| 26 | pub fn main(init: process.Init.Minimal) !void { | |
| 27 | // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not | |
| 28 | // always the case. So, we do need a true gpa for some things. | |
| 29 | var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); | |
| 30 | defer _ = safe_gpa_state.deinit(); | |
| 31 | const gpa = safe_gpa_state.allocator(); | |
| 32 | ||
| 33 | var threaded: std.Io.Threaded = .init(gpa, .{ | |
| 34 | .environ = init.environ, | |
| 35 | .argv0 = .init(init.args), | |
| 36 | }); | |
| 37 | defer threaded.deinit(); | |
| 38 | const io = threaded.io(); | |
| 39 | ||
| 40 | // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. | |
| 41 | var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); | |
| 42 | defer arena_instance.deinit(); | |
| 43 | const arena = arena_instance.allocator(); | |
| 44 | ||
| 45 | const args = try init.args.toSlice(arena); | |
| 46 | ||
| 47 | // skip my own exe name | |
| 48 | var arg_idx: usize = 1; | |
| 49 | ||
| 50 | const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); | |
| 51 | const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); | |
| 52 | const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); | |
| 53 | const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); | |
| 54 | const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); | |
| 55 | ||
| 56 | const cwd: Io.Dir = .cwd(); | |
| 57 | ||
| 58 | const zig_lib_directory: std.Build.Cache.Directory = .{ | |
| 59 | .path = zig_lib_dir, | |
| 60 | .handle = try cwd.openDir(io, zig_lib_dir, .{}), | |
| 61 | }; | |
| 62 | ||
| 63 | const build_root_directory: std.Build.Cache.Directory = .{ | |
| 64 | .path = build_root, | |
| 65 | .handle = try cwd.openDir(io, build_root, .{}), | |
| 66 | }; | |
| 67 | ||
| 68 | const local_cache_directory: std.Build.Cache.Directory = .{ | |
| 69 | .path = cache_root, | |
| 70 | .handle = try cwd.createDirPathOpen(io, cache_root, .{}), | |
| 71 | }; | |
| 72 | ||
| 73 | const global_cache_directory: std.Build.Cache.Directory = .{ | |
| 74 | .path = global_cache_root, | |
| 75 | .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), | |
| 76 | }; | |
| 77 | ||
| 78 | var graph: std.Build.Graph = .{ | |
| 79 | .io = io, | |
| 80 | .arena = arena, | |
| 81 | .cache = .{ | |
| 82 | .io = io, | |
| 83 | .gpa = gpa, | |
| 84 | .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), | |
| 85 | .cwd = try process.currentPathAlloc(io, arena), | |
| 86 | }, | |
| 87 | .zig_exe = zig_exe, | |
| 88 | .environ_map = try init.environ.createMap(arena), | |
| 89 | .global_cache_root = global_cache_directory, | |
| 90 | .zig_lib_directory = zig_lib_directory, | |
| 91 | .host = .{ | |
| 92 | .query = .{}, | |
| 93 | .result = try std.zig.system.resolveTargetQuery(io, .{}), | |
| 94 | }, | |
| 95 | .time_report = false, | |
| 96 | }; | |
| 97 | ||
| 98 | graph.cache.addPrefix(.{ .path = null, .handle = cwd }); | |
| 99 | graph.cache.addPrefix(build_root_directory); | |
| 100 | graph.cache.addPrefix(local_cache_directory); | |
| 101 | graph.cache.addPrefix(global_cache_directory); | |
| 102 | graph.cache.hash.addBytes(builtin.zig_version_string); | |
| 103 | ||
| 104 | const builder = try std.Build.create( | |
| 105 | &graph, | |
| 106 | build_root_directory, | |
| 107 | local_cache_directory, | |
| 108 | dependencies.root_deps, | |
| 109 | ); | |
| 110 | ||
| 111 | var targets = std.array_list.Managed([]const u8).init(arena); | |
| 112 | var debug_log_scopes = std.array_list.Managed([]const u8).init(arena); | |
| 113 | ||
| 114 | var install_prefix: ?[]const u8 = null; | |
| 115 | var dir_list = std.Build.DirList{}; | |
| 116 | var error_style: ErrorStyle = .verbose; | |
| 117 | var multiline_errors: MultilineErrors = .indent; | |
| 118 | var summary: ?Summary = null; | |
| 119 | var max_rss: u64 = 0; | |
| 120 | var skip_oom_steps = false; | |
| 121 | var test_timeout_ns: ?u64 = null; | |
| 122 | var color: Color = .auto; | |
| 123 | var help_menu = false; | |
| 124 | var steps_menu = false; | |
| 125 | var output_tmp_nonce: ?[16]u8 = null; | |
| 126 | var watch = false; | |
| 127 | var fuzz: ?std.Build.Fuzz.Mode = null; | |
| 128 | var debounce_interval_ms: u16 = 50; | |
| 129 | var webui_listen: ?Io.net.IpAddress = null; | |
| 130 | ||
| 131 | if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { | |
| 132 | if (std.meta.stringToEnum(ErrorStyle, str)) |style| { | |
| 133 | error_style = style; | |
| 134 | } | |
| 135 | } | |
| 136 | ||
| 137 | if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { | |
| 138 | if (std.meta.stringToEnum(MultilineErrors, str)) |style| { | |
| 139 | multiline_errors = style; | |
| 140 | } | |
| 141 | } | |
| 142 | ||
| 143 | while (nextArg(args, &arg_idx)) |arg| { | |
| 144 | if (mem.startsWith(u8, arg, "-Z")) { | |
| 145 | if (arg.len != 18) fatalWithHint("bad argument: '{s}'", .{arg}); | |
| 146 | output_tmp_nonce = arg[2..18].*; | |
| 147 | } else if (mem.startsWith(u8, arg, "-D")) { | |
| 148 | const option_contents = arg[2..]; | |
| 149 | if (option_contents.len == 0) | |
| 150 | fatalWithHint("expected option name after '-D'", .{}); | |
| 151 | if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { | |
| 152 | const option_name = option_contents[0..name_end]; | |
| 153 | const option_value = option_contents[name_end + 1 ..]; | |
| 154 | if (try builder.addUserInputOption(option_name, option_value)) | |
| 155 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 156 | } else { | |
| 157 | if (try builder.addUserInputFlag(option_contents)) | |
| 158 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 159 | } | |
| 160 | } else if (mem.startsWith(u8, arg, "-")) { | |
| 161 | if (mem.eql(u8, arg, "--verbose")) { | |
| 162 | builder.verbose = true; | |
| 163 | } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { | |
| 164 | help_menu = true; | |
| 165 | } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { | |
| 166 | install_prefix = nextArgOrFatal(args, &arg_idx); | |
| 167 | } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { | |
| 168 | steps_menu = true; | |
| 169 | } else if (mem.startsWith(u8, arg, "-fsys=")) { | |
| 170 | const name = arg["-fsys=".len..]; | |
| 171 | graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); | |
| 172 | } else if (mem.startsWith(u8, arg, "-fno-sys=")) { | |
| 173 | const name = arg["-fno-sys=".len..]; | |
| 174 | graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); | |
| 175 | } else if (mem.eql(u8, arg, "--release")) { | |
| 176 | builder.release_mode = .any; | |
| 177 | } else if (mem.startsWith(u8, arg, "--release=")) { | |
| 178 | const text = arg["--release=".len..]; | |
| 179 | builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { | |
| 180 | fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ | |
| 181 | arg, text, | |
| 182 | }); | |
| 183 | }; | |
| 184 | } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { | |
| 185 | dir_list.lib_dir = nextArgOrFatal(args, &arg_idx); | |
| 186 | } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { | |
| 187 | dir_list.exe_dir = nextArgOrFatal(args, &arg_idx); | |
| 188 | } else if (mem.eql(u8, arg, "--prefix-include-dir")) { | |
| 189 | dir_list.include_dir = nextArgOrFatal(args, &arg_idx); | |
| 190 | } else if (mem.eql(u8, arg, "--sysroot")) { | |
| 191 | builder.sysroot = nextArgOrFatal(args, &arg_idx); | |
| 192 | } else if (mem.eql(u8, arg, "--maxrss")) { | |
| 193 | const max_rss_text = nextArgOrFatal(args, &arg_idx); | |
| 194 | max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { | |
| 195 | std.debug.print("invalid byte size: '{s}': {s}\n", .{ | |
| 196 | max_rss_text, @errorName(err), | |
| 197 | }); | |
| 198 | process.exit(1); | |
| 199 | }; | |
| 200 | } else if (mem.eql(u8, arg, "--skip-oom-steps")) { | |
| 201 | skip_oom_steps = true; | |
| 202 | } else if (mem.eql(u8, arg, "--test-timeout")) { | |
| 203 | const units: []const struct { []const u8, u64 } = &.{ | |
| 204 | .{ "ns", 1 }, | |
| 205 | .{ "nanosecond", 1 }, | |
| 206 | .{ "us", std.time.ns_per_us }, | |
| 207 | .{ "microsecond", std.time.ns_per_us }, | |
| 208 | .{ "ms", std.time.ns_per_ms }, | |
| 209 | .{ "millisecond", std.time.ns_per_ms }, | |
| 210 | .{ "s", std.time.ns_per_s }, | |
| 211 | .{ "second", std.time.ns_per_s }, | |
| 212 | .{ "m", std.time.ns_per_min }, | |
| 213 | .{ "minute", std.time.ns_per_min }, | |
| 214 | .{ "h", std.time.ns_per_hour }, | |
| 215 | .{ "hour", std.time.ns_per_hour }, | |
| 216 | }; | |
| 217 | const timeout_str = nextArgOrFatal(args, &arg_idx); | |
| 218 | const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( | |
| 219 | "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)", | |
| 220 | .{timeout_str}, | |
| 221 | ); | |
| 222 | const num_str = timeout_str[0 .. num_end_idx + 1]; | |
| 223 | const unit_str = timeout_str[num_end_idx + 1 ..]; | |
| 224 | const unit_factor: f64 = for (units) |unit_and_factor| { | |
| 225 | if (std.mem.eql(u8, unit_str, unit_and_factor[0])) { | |
| 226 | break @floatFromInt(unit_and_factor[1]); | |
| 227 | } | |
| 228 | } else fatal( | |
| 229 | "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)", | |
| 230 | .{ timeout_str, unit_str }, | |
| 231 | ); | |
| 232 | const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal( | |
| 233 | "invalid timeout '{s}': invalid number '{s}' ({t})", | |
| 234 | .{ timeout_str, num_str, err }, | |
| 235 | ); | |
| 236 | test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); | |
| 237 | } else if (mem.eql(u8, arg, "--search-prefix")) { | |
| 238 | const search_prefix = nextArgOrFatal(args, &arg_idx); | |
| 239 | builder.addSearchPrefix(search_prefix); | |
| 240 | } else if (mem.eql(u8, arg, "--libc")) { | |
| 241 | builder.libc_file = nextArgOrFatal(args, &arg_idx); | |
| 242 | } else if (mem.eql(u8, arg, "--color")) { | |
| 243 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 244 | fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); | |
| 245 | color = std.meta.stringToEnum(Color, next_arg) orelse { | |
| 246 | fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ | |
| 247 | arg, next_arg, | |
| 248 | }); | |
| 249 | }; | |
| 250 | } else if (mem.eql(u8, arg, "--error-style")) { | |
| 251 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 252 | fatalWithHint("expected style after '{s}'", .{arg}); | |
| 253 | error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { | |
| 254 | fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); | |
| 255 | }; | |
| 256 | } else if (mem.eql(u8, arg, "--multiline-errors")) { | |
| 257 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 258 | fatalWithHint("expected style after '{s}'", .{arg}); | |
| 259 | multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { | |
| 260 | fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); | |
| 261 | }; | |
| 262 | } else if (mem.eql(u8, arg, "--summary")) { | |
| 263 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 264 | fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg}); | |
| 265 | summary = std.meta.stringToEnum(Summary, next_arg) orelse { | |
| 266 | fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{ | |
| 267 | arg, next_arg, | |
| 268 | }); | |
| 269 | }; | |
| 270 | } else if (mem.eql(u8, arg, "--seed")) { | |
| 271 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 272 | fatalWithHint("expected u32 after '{s}'", .{arg}); | |
| 273 | graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { | |
| 274 | fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ | |
| 275 | next_arg, @errorName(err), | |
| 276 | }); | |
| 277 | }; | |
| 278 | } else if (mem.eql(u8, arg, "--build-id")) { | |
| 279 | builder.build_id = .fast; | |
| 280 | } else if (mem.startsWith(u8, arg, "--build-id=")) { | |
| 281 | const style = arg["--build-id=".len..]; | |
| 282 | builder.build_id = std.zig.BuildId.parse(style) catch |err| { | |
| 283 | fatal("unable to parse --build-id style '{s}': {s}", .{ | |
| 284 | style, @errorName(err), | |
| 285 | }); | |
| 286 | }; | |
| 287 | } else if (mem.eql(u8, arg, "--debounce")) { | |
| 288 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 289 | fatalWithHint("expected u16 after '{s}'", .{arg}); | |
| 290 | debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { | |
| 291 | fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{ | |
| 292 | next_arg, err, | |
| 293 | }); | |
| 294 | }; | |
| 295 | } else if (mem.eql(u8, arg, "--webui")) { | |
| 296 | if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; | |
| 297 | } else if (mem.startsWith(u8, arg, "--webui=")) { | |
| 298 | const addr_str = arg["--webui=".len..]; | |
| 299 | if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); | |
| 300 | webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { | |
| 301 | fatal("invalid web UI address '{s}': {s}", .{ addr_str, @errorName(err) }); | |
| 302 | }; | |
| 303 | } else if (mem.eql(u8, arg, "--debug-log")) { | |
| 304 | const next_arg = nextArgOrFatal(args, &arg_idx); | |
| 305 | try debug_log_scopes.append(next_arg); | |
| 306 | } else if (mem.eql(u8, arg, "--debug-pkg-config")) { | |
| 307 | builder.debug_pkg_config = true; | |
| 308 | } else if (mem.eql(u8, arg, "--debug-rt")) { | |
| 309 | graph.debug_compiler_runtime_libs = .Debug; | |
| 310 | } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { | |
| 311 | graph.debug_compiler_runtime_libs = | |
| 312 | std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse | |
| 313 | fatal("unrecognized optimization mode: '{s}'", .{rest}); | |
| 314 | } else if (mem.eql(u8, arg, "--debug-compile-errors")) { | |
| 315 | builder.debug_compile_errors = true; | |
| 316 | } else if (mem.eql(u8, arg, "--debug-incremental")) { | |
| 317 | builder.debug_incremental = true; | |
| 318 | } else if (mem.eql(u8, arg, "--system")) { | |
| 319 | // The usage text shows another argument after this parameter | |
| 320 | // but it is handled by the parent process. The build runner | |
| 321 | // only sees this flag. | |
| 322 | graph.system_package_mode = true; | |
| 323 | } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { | |
| 324 | // --glibc-runtimes was the old name of the flag; kept for compatibility for now. | |
| 325 | builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); | |
| 326 | } else if (mem.eql(u8, arg, "--verbose-link")) { | |
| 327 | builder.verbose_link = true; | |
| 328 | } else if (mem.eql(u8, arg, "--verbose-air")) { | |
| 329 | builder.verbose_air = true; | |
| 330 | } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { | |
| 331 | builder.verbose_llvm_ir = "-"; | |
| 332 | } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { | |
| 333 | builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; | |
| 334 | } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { | |
| 335 | builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; | |
| 336 | } else if (mem.eql(u8, arg, "--verbose-cc")) { | |
| 337 | builder.verbose_cc = true; | |
| 338 | } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { | |
| 339 | builder.verbose_llvm_cpu_features = true; | |
| 340 | } else if (mem.eql(u8, arg, "--watch")) { | |
| 341 | watch = true; | |
| 342 | } else if (mem.eql(u8, arg, "--time-report")) { | |
| 343 | graph.time_report = true; | |
| 344 | if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; | |
| 345 | } else if (mem.eql(u8, arg, "--fuzz")) { | |
| 346 | fuzz = .{ .forever = undefined }; | |
| 347 | if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; | |
| 348 | } else if (mem.startsWith(u8, arg, "--fuzz=")) { | |
| 349 | const value = arg["--fuzz=".len..]; | |
| 350 | if (value.len == 0) fatal("missing argument to --fuzz", .{}); | |
| 351 | ||
| 352 | const unit: u8 = value[value.len - 1]; | |
| 353 | const digits = switch (unit) { | |
| 354 | '0'...'9' => value, | |
| 355 | 'K', 'M', 'G' => value[0 .. value.len - 1], | |
| 356 | else => fatal( | |
| 357 | "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", | |
| 358 | .{}, | |
| 359 | ), | |
| 360 | }; | |
| 361 | ||
| 362 | const amount = std.fmt.parseInt(u64, digits, 10) catch { | |
| 363 | fatal( | |
| 364 | "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", | |
| 365 | .{}, | |
| 366 | ); | |
| 367 | }; | |
| 368 | ||
| 369 | const normalized_amount = std.math.mul(u64, amount, switch (unit) { | |
| 370 | else => unreachable, | |
| 371 | '0'...'9' => 1, | |
| 372 | 'K' => 1000, | |
| 373 | 'M' => 1_000_000, | |
| 374 | 'G' => 1_000_000_000, | |
| 375 | }) catch fatal("fuzzing limit amount overflows u64", .{}); | |
| 376 | ||
| 377 | fuzz = .{ | |
| 378 | .limit = .{ | |
| 379 | .amount = normalized_amount, | |
| 380 | }, | |
| 381 | }; | |
| 382 | } else if (mem.eql(u8, arg, "-fincremental")) { | |
| 383 | graph.incremental = true; | |
| 384 | } else if (mem.eql(u8, arg, "-fno-incremental")) { | |
| 385 | graph.incremental = false; | |
| 386 | } else if (mem.eql(u8, arg, "-fwine")) { | |
| 387 | builder.enable_wine = true; | |
| 388 | } else if (mem.eql(u8, arg, "-fno-wine")) { | |
| 389 | builder.enable_wine = false; | |
| 390 | } else if (mem.eql(u8, arg, "-fqemu")) { | |
| 391 | builder.enable_qemu = true; | |
| 392 | } else if (mem.eql(u8, arg, "-fno-qemu")) { | |
| 393 | builder.enable_qemu = false; | |
| 394 | } else if (mem.eql(u8, arg, "-fwasmtime")) { | |
| 395 | builder.enable_wasmtime = true; | |
| 396 | } else if (mem.eql(u8, arg, "-fno-wasmtime")) { | |
| 397 | builder.enable_wasmtime = false; | |
| 398 | } else if (mem.eql(u8, arg, "-frosetta")) { | |
| 399 | builder.enable_rosetta = true; | |
| 400 | } else if (mem.eql(u8, arg, "-fno-rosetta")) { | |
| 401 | builder.enable_rosetta = false; | |
| 402 | } else if (mem.eql(u8, arg, "-fdarling")) { | |
| 403 | builder.enable_darling = true; | |
| 404 | } else if (mem.eql(u8, arg, "-fno-darling")) { | |
| 405 | builder.enable_darling = false; | |
| 406 | } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { | |
| 407 | graph.allow_so_scripts = true; | |
| 408 | } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { | |
| 409 | graph.allow_so_scripts = false; | |
| 410 | } else if (mem.eql(u8, arg, "-freference-trace")) { | |
| 411 | builder.reference_trace = 256; | |
| 412 | } else if (mem.startsWith(u8, arg, "-freference-trace=")) { | |
| 413 | const num = arg["-freference-trace=".len..]; | |
| 414 | builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { | |
| 415 | std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); | |
| 416 | process.exit(1); | |
| 417 | }; | |
| 418 | } else if (mem.eql(u8, arg, "-fno-reference-trace")) { | |
| 419 | builder.reference_trace = null; | |
| 420 | } else if (mem.cutPrefix(u8, arg, "-j")) |text| { | |
| 421 | const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| | |
| 422 | fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); | |
| 423 | if (n < 1) fatal("number of jobs must be at least 1", .{}); | |
| 424 | threaded.setAsyncLimit(.limited(n)); | |
| 425 | graph.max_jobs = n; | |
| 426 | } else if (mem.eql(u8, arg, "--")) { | |
| 427 | builder.args = argsRest(args, arg_idx); | |
| 428 | break; | |
| 429 | } else { | |
| 430 | fatalWithHint("unrecognized argument: '{s}'", .{arg}); | |
| 431 | } | |
| 432 | } else { | |
| 433 | try targets.append(arg); | |
| 434 | } | |
| 435 | } | |
| 436 | ||
| 437 | const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); | |
| 438 | const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); | |
| 439 | ||
| 440 | graph.stderr_mode = switch (color) { | |
| 441 | .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), | |
| 442 | .on => .escape_codes, | |
| 443 | .off => .no_color, | |
| 444 | }; | |
| 445 | ||
| 446 | if (webui_listen != null) { | |
| 447 | if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); | |
| 448 | if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); | |
| 449 | } | |
| 450 | ||
| 451 | const main_progress_node = std.Progress.start(io, .{ | |
| 452 | .disable_printing = (color == .off), | |
| 453 | }); | |
| 454 | defer main_progress_node.end(); | |
| 455 | ||
| 456 | builder.debug_log_scopes = debug_log_scopes.items; | |
| 457 | builder.resolveInstallPrefix(install_prefix, dir_list); | |
| 458 | { | |
| 459 | var prog_node = main_progress_node.start("Configure", 0); | |
| 460 | defer prog_node.end(); | |
| 461 | try builder.runBuild(root); | |
| 462 | createModuleDependencies(builder) catch @panic("OOM"); | |
| 463 | } | |
| 464 | ||
| 465 | if (graph.needed_lazy_dependencies.entries.len != 0) { | |
| 466 | var buffer: std.ArrayList(u8) = .empty; | |
| 467 | for (graph.needed_lazy_dependencies.keys()) |k| { | |
| 468 | try buffer.appendSlice(arena, k); | |
| 469 | try buffer.append(arena, '\n'); | |
| 470 | } | |
| 471 | const s = std.fs.path.sep_str; | |
| 472 | const tmp_sub_path = "tmp" ++ s ++ (output_tmp_nonce orelse fatal("missing -Z arg", .{})); | |
| 473 | local_cache_directory.handle.writeFile(io, .{ | |
| 474 | .sub_path = tmp_sub_path, | |
| 475 | .data = buffer.items, | |
| 476 | .flags = .{ .exclusive = true }, | |
| 477 | }) catch |err| { | |
| 478 | fatal("unable to write configuration results to '{f}{s}': {s}", .{ | |
| 479 | local_cache_directory, tmp_sub_path, @errorName(err), | |
| 480 | }); | |
| 481 | }; | |
| 482 | process.exit(3); // Indicate configure phase failed with meaningful stdout. | |
| 483 | } | |
| 484 | ||
| 485 | if (builder.validateUserInputDidItFail()) { | |
| 486 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 487 | } | |
| 488 | ||
| 489 | validateSystemLibraryOptions(builder); | |
| 490 | ||
| 491 | if (help_menu) { | |
| 492 | var w = initStdoutWriter(io); | |
| 493 | printUsage(builder, w) catch return stdout_writer_allocation.err.?; | |
| 494 | w.flush() catch return stdout_writer_allocation.err.?; | |
| 495 | return; | |
| 496 | } | |
| 497 | ||
| 498 | if (steps_menu) { | |
| 499 | var w = initStdoutWriter(io); | |
| 500 | printSteps(builder, w) catch return stdout_writer_allocation.err.?; | |
| 501 | w.flush() catch return stdout_writer_allocation.err.?; | |
| 502 | return; | |
| 503 | } | |
| 504 | ||
| 505 | var run: Run = .{ | |
| 506 | .gpa = gpa, | |
| 507 | ||
| 508 | .available_rss = max_rss, | |
| 509 | .max_rss_is_default = false, | |
| 510 | .max_rss_mutex = .init, | |
| 511 | .skip_oom_steps = skip_oom_steps, | |
| 512 | .unit_test_timeout_ns = test_timeout_ns, | |
| 513 | ||
| 514 | .watch = watch, | |
| 515 | .web_server = undefined, // set after `prepare` | |
| 516 | .memory_blocked_steps = .empty, | |
| 517 | .step_stack = .empty, | |
| 518 | ||
| 519 | .error_style = error_style, | |
| 520 | .multiline_errors = multiline_errors, | |
| 521 | .summary = summary orelse if (watch or webui_listen != null) .line else .failures, | |
| 522 | }; | |
| 523 | defer { | |
| 524 | run.memory_blocked_steps.deinit(gpa); | |
| 525 | run.step_stack.deinit(gpa); | |
| 526 | } | |
| 527 | ||
| 528 | if (run.available_rss == 0) { | |
| 529 | run.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); | |
| 530 | run.max_rss_is_default = true; | |
| 531 | } | |
| 532 | ||
| 533 | prepare(arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { | |
| 534 | error.DependencyLoopDetected, error.InsufficientMemory => { | |
| 535 | // Perhaps in the future there could be an Advanced Options flag | |
| 536 | // such as --debug-build-runner-leaks which would make this code | |
| 537 | // return instead of calling exit. | |
| 538 | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; | |
| 539 | process.exit(1); | |
| 540 | }, | |
| 541 | else => |e| return e, | |
| 542 | }; | |
| 543 | ||
| 544 | var w: Watch = w: { | |
| 545 | if (!watch) break :w undefined; | |
| 546 | if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); | |
| 547 | break :w try .init(graph.cache.cwd); | |
| 548 | }; | |
| 549 | ||
| 550 | const now = Io.Clock.Timestamp.now(io, .awake); | |
| 551 | ||
| 552 | run.web_server = if (webui_listen) |listen_address| ws: { | |
| 553 | if (builtin.single_threaded) unreachable; // `fatal` above | |
| 554 | break :ws .init(.{ | |
| 555 | .gpa = gpa, | |
| 556 | .graph = &graph, | |
| 557 | .all_steps = run.step_stack.keys(), | |
| 558 | .root_prog_node = main_progress_node, | |
| 559 | .watch = watch, | |
| 560 | .listen_address = listen_address, | |
| 561 | .base_timestamp = now, | |
| 562 | }); | |
| 563 | } else null; | |
| 564 | ||
| 565 | if (run.web_server) |*ws| { | |
| 566 | ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); | |
| 567 | } | |
| 568 | ||
| 569 | rebuild: while (true) : (if (run.error_style.clearOnUpdate()) { | |
| 570 | const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); | |
| 571 | defer io.unlockStderr(); | |
| 572 | try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H"); | |
| 573 | }) { | |
| 574 | if (run.web_server) |*ws| ws.startBuild(); | |
| 575 | ||
| 576 | try runStepNames( | |
| 577 | builder, | |
| 578 | targets.items, | |
| 579 | main_progress_node, | |
| 580 | &run, | |
| 581 | fuzz, | |
| 582 | ); | |
| 583 | ||
| 584 | if (run.web_server) |*web_server| { | |
| 585 | if (fuzz) |mode| if (mode != .forever) fatal( | |
| 586 | "error: limited fuzzing is not implemented yet for --webui", | |
| 587 | .{}, | |
| 588 | ); | |
| 589 | ||
| 590 | web_server.finishBuild(.{ .fuzz = fuzz != null }); | |
| 591 | } | |
| 592 | ||
| 593 | if (run.web_server) |*ws| { | |
| 594 | assert(!watch); // fatal error after CLI parsing | |
| 595 | while (true) switch (try ws.wait()) { | |
| 596 | .rebuild => { | |
| 597 | for (run.step_stack.keys()) |step| { | |
| 598 | step.state = .precheck_done; | |
| 599 | step.pending_deps = @intCast(step.dependencies.items.len); | |
| 600 | step.reset(gpa); | |
| 601 | } | |
| 602 | continue :rebuild; | |
| 603 | }, | |
| 604 | }; | |
| 605 | } | |
| 606 | ||
| 607 | // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. | |
| 608 | if (!Watch.have_impl) unreachable; | |
| 609 | ||
| 610 | try w.update(gpa, run.step_stack.keys()); | |
| 611 | ||
| 612 | // Wait until a file system notification arrives. Read all such events | |
| 613 | // until the buffer is empty. Then wait for a debounce interval, resetting | |
| 614 | // if any more events come in. After the debounce interval has passed, | |
| 615 | // trigger a rebuild on all steps with modified inputs, as well as their | |
| 616 | // recursive dependants. | |
| 617 | var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; | |
| 618 | const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ | |
| 619 | w.dir_count, countSubProcesses(run.step_stack.keys()), | |
| 620 | }) catch &caption_buf; | |
| 621 | var debouncing_node = main_progress_node.start(caption, 0); | |
| 622 | var in_debounce = false; | |
| 623 | while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { | |
| 624 | .timeout => { | |
| 625 | assert(in_debounce); | |
| 626 | debouncing_node.end(); | |
| 627 | markFailedStepsDirty(gpa, run.step_stack.keys()); | |
| 628 | continue :rebuild; | |
| 629 | }, | |
| 630 | .dirty => if (!in_debounce) { | |
| 631 | in_debounce = true; | |
| 632 | debouncing_node.end(); | |
| 633 | debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); | |
| 634 | }, | |
| 635 | .clean => {}, | |
| 636 | }; | |
| 637 | } | |
| 638 | } | |
| 639 | ||
| 640 | fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { | |
| 641 | for (all_steps) |step| switch (step.state) { | |
| 642 | .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), | |
| 643 | else => continue, | |
| 644 | }; | |
| 645 | // Now that all dirty steps have been found, the remaining steps that | |
| 646 | // succeeded from last run shall be marked "cached". | |
| 647 | for (all_steps) |step| switch (step.state) { | |
| 648 | .success => step.result_cached = true, | |
| 649 | else => continue, | |
| 650 | }; | |
| 651 | } | |
| 652 | ||
| 653 | fn countSubProcesses(all_steps: []const *Step) usize { | |
| 654 | var count: usize = 0; | |
| 655 | for (all_steps) |s| { | |
| 656 | count += @intFromBool(s.getZigProcess() != null); | |
| 657 | } | |
| 658 | return count; | |
| 659 | } | |
| 660 | ||
| 661 | const Run = struct { | |
| 662 | gpa: Allocator, | |
| 663 | ||
| 664 | available_rss: usize, | |
| 665 | max_rss_is_default: bool, | |
| 666 | max_rss_mutex: Io.Mutex, | |
| 667 | skip_oom_steps: bool, | |
| 668 | unit_test_timeout_ns: ?u64, | |
| 669 | watch: bool, | |
| 670 | web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, | |
| 671 | /// Allocated into `gpa`. | |
| 672 | memory_blocked_steps: std.ArrayList(*Step), | |
| 673 | /// Allocated into `gpa`. | |
| 674 | step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 675 | ||
| 676 | error_style: ErrorStyle, | |
| 677 | multiline_errors: MultilineErrors, | |
| 678 | summary: Summary, | |
| 679 | }; | |
| 680 | ||
| 681 | fn prepare( | |
| 682 | arena: Allocator, | |
| 683 | b: *std.Build, | |
| 684 | step_names: []const []const u8, | |
| 685 | run: *Run, | |
| 686 | seed: u32, | |
| 687 | ) !void { | |
| 688 | const gpa = run.gpa; | |
| 689 | const step_stack = &run.step_stack; | |
| 690 | ||
| 691 | if (step_names.len == 0) { | |
| 692 | try step_stack.put(gpa, b.default_step, {}); | |
| 693 | } else { | |
| 694 | try step_stack.ensureUnusedCapacity(gpa, step_names.len); | |
| 695 | for (0..step_names.len) |i| { | |
| 696 | const step_name = step_names[step_names.len - i - 1]; | |
| 697 | const s = b.top_level_steps.get(step_name) orelse { | |
| 698 | std.log.info("access the help menu with \"zig build -h\"", .{}); | |
| 699 | fatal("no step named '{s}'", .{step_name}); | |
| 700 | }; | |
| 701 | step_stack.putAssumeCapacity(&s.step, {}); | |
| 702 | } | |
| 703 | } | |
| 704 | ||
| 705 | const starting_steps = try arena.dupe(*Step, step_stack.keys()); | |
| 706 | ||
| 707 | var rng = std.Random.DefaultPrng.init(seed); | |
| 708 | const rand = rng.random(); | |
| 709 | rand.shuffle(*Step, starting_steps); | |
| 710 | ||
| 711 | for (starting_steps) |s| { | |
| 712 | try constructGraphAndCheckForDependencyLoop(gpa, b, s, &run.step_stack, rand); | |
| 713 | } | |
| 714 | ||
| 715 | { | |
| 716 | // Check that we have enough memory to complete the build. | |
| 717 | var any_problems = false; | |
| 718 | var max_needed: usize = 0; | |
| 719 | for (step_stack.keys()) |s| { | |
| 720 | if (s.max_rss == 0) continue; | |
| 721 | max_needed = @max(max_needed, s.max_rss); | |
| 722 | if (s.max_rss > run.available_rss) { | |
| 723 | if (run.skip_oom_steps) { | |
| 724 | s.state = .skipped_oom; | |
| 725 | for (s.dependants.items) |dependant| { | |
| 726 | dependant.pending_deps -= 1; | |
| 727 | } | |
| 728 | } else { | |
| 729 | std.log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ | |
| 730 | s.owner.dep_prefix, s.name, s.max_rss, run.available_rss, | |
| 731 | }); | |
| 732 | any_problems = true; | |
| 733 | } | |
| 734 | } | |
| 735 | } | |
| 736 | if (any_problems) { | |
| 737 | if (run.max_rss_is_default) { | |
| 738 | std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ | |
| 739 | max_needed, | |
| 740 | }); | |
| 741 | } | |
| 742 | return error.InsufficientMemory; | |
| 743 | } | |
| 744 | } | |
| 745 | } | |
| 746 | ||
| 747 | fn runStepNames( | |
| 748 | b: *std.Build, | |
| 749 | step_names: []const []const u8, | |
| 750 | parent_prog_node: std.Progress.Node, | |
| 751 | run: *Run, | |
| 752 | fuzz: ?std.Build.Fuzz.Mode, | |
| 753 | ) !void { | |
| 754 | const gpa = run.gpa; | |
| 755 | const graph = b.graph; | |
| 756 | const io = graph.io; | |
| 757 | const step_stack = &run.step_stack; | |
| 758 | ||
| 759 | { | |
| 760 | // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, | |
| 761 | // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking | |
| 762 | // a step is initial when it actually became ready due to an earlier initial step. | |
| 763 | var initial_set: std.ArrayList(*Step) = .empty; | |
| 764 | defer initial_set.deinit(gpa); | |
| 765 | try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); | |
| 766 | for (step_stack.keys()) |s| { | |
| 767 | if (s.state == .precheck_done and s.pending_deps == 0) { | |
| 768 | initial_set.appendAssumeCapacity(s); | |
| 769 | } | |
| 770 | } | |
| 771 | ||
| 772 | const step_prog = parent_prog_node.start("steps", step_stack.count()); | |
| 773 | defer step_prog.end(); | |
| 774 | ||
| 775 | var group: Io.Group = .init; | |
| 776 | defer group.cancel(io); | |
| 777 | // Start working on all of the initial steps... | |
| 778 | for (initial_set.items) |s| try stepReady(&group, b, s, step_prog, run); | |
| 779 | // ...and `makeStep` will trigger every other step when their last dependency finishes. | |
| 780 | try group.await(io); | |
| 781 | } | |
| 782 | ||
| 783 | assert(run.memory_blocked_steps.items.len == 0); | |
| 784 | ||
| 785 | var test_pass_count: usize = 0; | |
| 786 | var test_skip_count: usize = 0; | |
| 787 | var test_fail_count: usize = 0; | |
| 788 | var test_crash_count: usize = 0; | |
| 789 | var test_timeout_count: usize = 0; | |
| 790 | ||
| 791 | var test_count: usize = 0; | |
| 792 | ||
| 793 | var success_count: usize = 0; | |
| 794 | var skipped_count: usize = 0; | |
| 795 | var failure_count: usize = 0; | |
| 796 | var pending_count: usize = 0; | |
| 797 | var total_compile_errors: usize = 0; | |
| 798 | ||
| 799 | var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); | |
| 800 | defer cleanup_task.await(io); | |
| 801 | ||
| 802 | for (step_stack.keys()) |s| { | |
| 803 | test_pass_count += s.test_results.passCount(); | |
| 804 | test_skip_count += s.test_results.skip_count; | |
| 805 | test_fail_count += s.test_results.fail_count; | |
| 806 | test_crash_count += s.test_results.crash_count; | |
| 807 | test_timeout_count += s.test_results.timeout_count; | |
| 808 | ||
| 809 | test_count += s.test_results.test_count; | |
| 810 | ||
| 811 | switch (s.state) { | |
| 812 | .precheck_unstarted => unreachable, | |
| 813 | .precheck_started => unreachable, | |
| 814 | .precheck_done => unreachable, | |
| 815 | .dependency_failure => pending_count += 1, | |
| 816 | .success => success_count += 1, | |
| 817 | .skipped, .skipped_oom => skipped_count += 1, | |
| 818 | .failure => { | |
| 819 | failure_count += 1; | |
| 820 | const compile_errors_len = s.result_error_bundle.errorMessageCount(); | |
| 821 | if (compile_errors_len > 0) { | |
| 822 | total_compile_errors += compile_errors_len; | |
| 823 | } | |
| 824 | }, | |
| 825 | } | |
| 826 | } | |
| 827 | ||
| 828 | if (fuzz) |mode| blk: { | |
| 829 | switch (builtin.os.tag) { | |
| 830 | // Current implementation depends on two things that need to be ported to Windows: | |
| 831 | // * Memory-mapping to share data between the fuzzer and build runner. | |
| 832 | // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving | |
| 833 | // many addresses to source locations). | |
| 834 | .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), | |
| 835 | else => {}, | |
| 836 | } | |
| 837 | if (@bitSizeOf(usize) != 64) { | |
| 838 | // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, | |
| 839 | // being compatible with file system's u64 return value. This is not the case | |
| 840 | // on 32-bit platforms. | |
| 841 | // Affects or affected by issues #5185, #22523, and #22464. | |
| 842 | fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); | |
| 843 | } | |
| 844 | ||
| 845 | switch (mode) { | |
| 846 | .forever => break :blk, | |
| 847 | .limit => {}, | |
| 848 | } | |
| 849 | ||
| 850 | assert(mode == .limit); | |
| 851 | var f = std.Build.Fuzz.init( | |
| 852 | gpa, | |
| 853 | io, | |
| 854 | step_stack.keys(), | |
| 855 | parent_prog_node, | |
| 856 | mode, | |
| 857 | ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); | |
| 858 | defer f.deinit(); | |
| 859 | ||
| 860 | f.start(); | |
| 861 | try f.waitAndPrintReport(); | |
| 862 | } | |
| 863 | ||
| 864 | // Every test has a state | |
| 865 | assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); | |
| 866 | ||
| 867 | if (failure_count == 0) { | |
| 868 | std.Progress.setStatus(.success); | |
| 869 | } else { | |
| 870 | std.Progress.setStatus(.failure); | |
| 871 | } | |
| 872 | ||
| 873 | summary: { | |
| 874 | switch (run.summary) { | |
| 875 | .all, .new, .line => {}, | |
| 876 | .failures => if (failure_count == 0) break :summary, | |
| 877 | .none => break :summary, | |
| 878 | } | |
| 879 | ||
| 880 | const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); | |
| 881 | defer io.unlockStderr(); | |
| 882 | const t = stderr.terminal(); | |
| 883 | const w = &stderr.file_writer.interface; | |
| 884 | ||
| 885 | const total_count = success_count + failure_count + pending_count + skipped_count; | |
| 886 | t.setColor(.cyan) catch {}; | |
| 887 | t.setColor(.bold) catch {}; | |
| 888 | w.writeAll("Build Summary: ") catch {}; | |
| 889 | t.setColor(.reset) catch {}; | |
| 890 | w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; | |
| 891 | { | |
| 892 | t.setColor(.dim) catch {}; | |
| 893 | var first = true; | |
| 894 | if (skipped_count > 0) { | |
| 895 | w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; | |
| 896 | first = false; | |
| 897 | } | |
| 898 | if (failure_count > 0) { | |
| 899 | w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; | |
| 900 | first = false; | |
| 901 | } | |
| 902 | if (!first) w.writeByte(')') catch {}; | |
| 903 | t.setColor(.reset) catch {}; | |
| 904 | } | |
| 905 | ||
| 906 | if (test_count > 0) { | |
| 907 | w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; | |
| 908 | t.setColor(.dim) catch {}; | |
| 909 | var first = true; | |
| 910 | if (test_skip_count > 0) { | |
| 911 | w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; | |
| 912 | first = false; | |
| 913 | } | |
| 914 | if (test_fail_count > 0) { | |
| 915 | w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; | |
| 916 | first = false; | |
| 917 | } | |
| 918 | if (test_crash_count > 0) { | |
| 919 | w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; | |
| 920 | first = false; | |
| 921 | } | |
| 922 | if (test_timeout_count > 0) { | |
| 923 | w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; | |
| 924 | first = false; | |
| 925 | } | |
| 926 | if (!first) w.writeByte(')') catch {}; | |
| 927 | t.setColor(.reset) catch {}; | |
| 928 | } | |
| 929 | ||
| 930 | w.writeAll("\n") catch {}; | |
| 931 | ||
| 932 | if (run.summary == .line) break :summary; | |
| 933 | ||
| 934 | // Print a fancy tree with build results. | |
| 935 | var step_stack_copy = try step_stack.clone(gpa); | |
| 936 | defer step_stack_copy.deinit(gpa); | |
| 937 | ||
| 938 | var print_node: PrintNode = .{ .parent = null }; | |
| 939 | if (step_names.len == 0) { | |
| 940 | print_node.last = true; | |
| 941 | printTreeStep(b, b.default_step, run, t, &print_node, &step_stack_copy) catch {}; | |
| 942 | } else { | |
| 943 | const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: { | |
| 944 | var i: usize = step_names.len; | |
| 945 | while (i > 0) { | |
| 946 | i -= 1; | |
| 947 | const step = b.top_level_steps.get(step_names[i]).?.step; | |
| 948 | const found = switch (run.summary) { | |
| 949 | .all, .line, .none => unreachable, | |
| 950 | .failures => step.state != .success, | |
| 951 | .new => !step.result_cached, | |
| 952 | }; | |
| 953 | if (found) break :blk i; | |
| 954 | } | |
| 955 | break :blk b.top_level_steps.count(); | |
| 956 | }; | |
| 957 | for (step_names, 0..) |step_name, i| { | |
| 958 | const tls = b.top_level_steps.get(step_name).?; | |
| 959 | print_node.last = i + 1 == last_index; | |
| 960 | printTreeStep(b, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; | |
| 961 | } | |
| 962 | } | |
| 963 | w.writeByte('\n') catch {}; | |
| 964 | } | |
| 965 | ||
| 966 | if (run.watch or run.web_server != null) return; | |
| 967 | ||
| 968 | // Perhaps in the future there could be an Advanced Options flag such as | |
| 969 | // --debug-build-runner-leaks which would make this code return instead of | |
| 970 | // calling exit. | |
| 971 | ||
| 972 | const code: u8 = code: { | |
| 973 | if (failure_count == 0) break :code 0; // success | |
| 974 | if (run.error_style.verboseContext()) break :code 1; // failure; print build command | |
| 975 | break :code 2; // failure; do not print build command | |
| 976 | }; | |
| 977 | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; | |
| 978 | process.exit(code); | |
| 979 | } | |
| 980 | ||
| 981 | const PrintNode = struct { | |
| 982 | parent: ?*PrintNode, | |
| 983 | last: bool = false, | |
| 984 | }; | |
| 985 | ||
| 986 | fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { | |
| 987 | const parent = node.parent orelse return; | |
| 988 | const writer = stderr.writer; | |
| 989 | if (parent.parent == null) return; | |
| 990 | try printPrefix(parent, stderr); | |
| 991 | if (parent.last) { | |
| 992 | try writer.writeAll(" "); | |
| 993 | } else { | |
| 994 | try writer.writeAll(switch (stderr.mode) { | |
| 995 | .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ | |
| 996 | else => "| ", | |
| 997 | }); | |
| 998 | } | |
| 999 | } | |
| 1000 | ||
| 1001 | fn printChildNodePrefix(stderr: Io.Terminal) !void { | |
| 1002 | try stderr.writer.writeAll(switch (stderr.mode) { | |
| 1003 | .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ | |
| 1004 | else => "+- ", | |
| 1005 | }); | |
| 1006 | } | |
| 1007 | ||
| 1008 | fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void { | |
| 1009 | const writer = stderr.writer; | |
| 1010 | switch (s.state) { | |
| 1011 | .precheck_unstarted => unreachable, | |
| 1012 | .precheck_started => unreachable, | |
| 1013 | .precheck_done => unreachable, | |
| 1014 | ||
| 1015 | .dependency_failure => { | |
| 1016 | try stderr.setColor(.dim); | |
| 1017 | try writer.writeAll(" transitive failure\n"); | |
| 1018 | try stderr.setColor(.reset); | |
| 1019 | }, | |
| 1020 | ||
| 1021 | .success => { | |
| 1022 | try stderr.setColor(.green); | |
| 1023 | if (s.result_cached) { | |
| 1024 | try writer.writeAll(" cached"); | |
| 1025 | } else if (s.test_results.test_count > 0) { | |
| 1026 | const pass_count = s.test_results.passCount(); | |
| 1027 | assert(s.test_results.test_count == pass_count + s.test_results.skip_count); | |
| 1028 | try writer.print(" {d} pass", .{pass_count}); | |
| 1029 | if (s.test_results.skip_count > 0) { | |
| 1030 | try stderr.setColor(.reset); | |
| 1031 | try writer.writeAll(", "); | |
| 1032 | try stderr.setColor(.yellow); | |
| 1033 | try writer.print("{d} skip", .{s.test_results.skip_count}); | |
| 1034 | } | |
| 1035 | try stderr.setColor(.reset); | |
| 1036 | try writer.print(" ({d} total)", .{s.test_results.test_count}); | |
| 1037 | } else { | |
| 1038 | try writer.writeAll(" success"); | |
| 1039 | } | |
| 1040 | try stderr.setColor(.reset); | |
| 1041 | if (s.result_duration_ns) |ns| { | |
| 1042 | try stderr.setColor(.dim); | |
| 1043 | if (ns >= std.time.ns_per_min) { | |
| 1044 | try writer.print(" {d}m", .{ns / std.time.ns_per_min}); | |
| 1045 | } else if (ns >= std.time.ns_per_s) { | |
| 1046 | try writer.print(" {d}s", .{ns / std.time.ns_per_s}); | |
| 1047 | } else if (ns >= std.time.ns_per_ms) { | |
| 1048 | try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); | |
| 1049 | } else if (ns >= std.time.ns_per_us) { | |
| 1050 | try writer.print(" {d}us", .{ns / std.time.ns_per_us}); | |
| 1051 | } else { | |
| 1052 | try writer.print(" {d}ns", .{ns}); | |
| 1053 | } | |
| 1054 | try stderr.setColor(.reset); | |
| 1055 | } | |
| 1056 | if (s.result_peak_rss != 0) { | |
| 1057 | const rss = s.result_peak_rss; | |
| 1058 | try stderr.setColor(.dim); | |
| 1059 | if (rss >= 1000_000_000) { | |
| 1060 | try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); | |
| 1061 | } else if (rss >= 1000_000) { | |
| 1062 | try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); | |
| 1063 | } else if (rss >= 1000) { | |
| 1064 | try writer.print(" MaxRSS:{d}K", .{rss / 1000}); | |
| 1065 | } else { | |
| 1066 | try writer.print(" MaxRSS:{d}B", .{rss}); | |
| 1067 | } | |
| 1068 | try stderr.setColor(.reset); | |
| 1069 | } | |
| 1070 | try writer.writeAll("\n"); | |
| 1071 | }, | |
| 1072 | .skipped => { | |
| 1073 | try stderr.setColor(.yellow); | |
| 1074 | try writer.writeAll(" skipped\n"); | |
| 1075 | try stderr.setColor(.reset); | |
| 1076 | }, | |
| 1077 | .skipped_oom => { | |
| 1078 | try stderr.setColor(.yellow); | |
| 1079 | try writer.writeAll(" skipped (not enough memory)"); | |
| 1080 | try stderr.setColor(.dim); | |
| 1081 | try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ s.max_rss, run.available_rss }); | |
| 1082 | try stderr.setColor(.reset); | |
| 1083 | }, | |
| 1084 | .failure => { | |
| 1085 | try printStepFailure(s, stderr, false); | |
| 1086 | try stderr.setColor(.reset); | |
| 1087 | }, | |
| 1088 | } | |
| 1089 | } | |
| 1090 | ||
| 1091 | fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void { | |
| 1092 | const w = stderr.writer; | |
| 1093 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 1094 | try stderr.setColor(.red); | |
| 1095 | try w.print(" {d} errors\n", .{ | |
| 1096 | s.result_error_bundle.errorMessageCount(), | |
| 1097 | }); | |
| 1098 | } else if (!s.test_results.isSuccess()) { | |
| 1099 | // These first values include all of the test "statuses". Every test is either passsed, | |
| 1100 | // skipped, failed, crashed, or timed out. | |
| 1101 | try stderr.setColor(.green); | |
| 1102 | try w.print(" {d} pass", .{s.test_results.passCount()}); | |
| 1103 | try stderr.setColor(.reset); | |
| 1104 | if (dim) try stderr.setColor(.dim); | |
| 1105 | if (s.test_results.skip_count > 0) { | |
| 1106 | try w.writeAll(", "); | |
| 1107 | try stderr.setColor(.yellow); | |
| 1108 | try w.print("{d} skip", .{s.test_results.skip_count}); | |
| 1109 | try stderr.setColor(.reset); | |
| 1110 | if (dim) try stderr.setColor(.dim); | |
| 1111 | } | |
| 1112 | if (s.test_results.fail_count > 0) { | |
| 1113 | try w.writeAll(", "); | |
| 1114 | try stderr.setColor(.red); | |
| 1115 | try w.print("{d} fail", .{s.test_results.fail_count}); | |
| 1116 | try stderr.setColor(.reset); | |
| 1117 | if (dim) try stderr.setColor(.dim); | |
| 1118 | } | |
| 1119 | if (s.test_results.crash_count > 0) { | |
| 1120 | try w.writeAll(", "); | |
| 1121 | try stderr.setColor(.red); | |
| 1122 | try w.print("{d} crash", .{s.test_results.crash_count}); | |
| 1123 | try stderr.setColor(.reset); | |
| 1124 | if (dim) try stderr.setColor(.dim); | |
| 1125 | } | |
| 1126 | if (s.test_results.timeout_count > 0) { | |
| 1127 | try w.writeAll(", "); | |
| 1128 | try stderr.setColor(.red); | |
| 1129 | try w.print("{d} timeout", .{s.test_results.timeout_count}); | |
| 1130 | try stderr.setColor(.reset); | |
| 1131 | if (dim) try stderr.setColor(.dim); | |
| 1132 | } | |
| 1133 | try w.print(" ({d} total)", .{s.test_results.test_count}); | |
| 1134 | ||
| 1135 | // Memory leaks are intentionally written after the total, because is isn't a test *status*, | |
| 1136 | // but just a flag that any tests -- even passed ones -- can have. We also use a different | |
| 1137 | // separator, so it looks like: | |
| 1138 | // 2 pass, 1 skip, 2 fail (5 total); 2 leaks | |
| 1139 | if (s.test_results.leak_count > 0) { | |
| 1140 | try w.writeAll("; "); | |
| 1141 | try stderr.setColor(.red); | |
| 1142 | try w.print("{d} leaks", .{s.test_results.leak_count}); | |
| 1143 | try stderr.setColor(.reset); | |
| 1144 | if (dim) try stderr.setColor(.dim); | |
| 1145 | } | |
| 1146 | ||
| 1147 | // It's usually not helpful to know how many error logs there were because they tend to | |
| 1148 | // just come with other errors (e.g. crashes and leaks print stack traces, and clean | |
| 1149 | // failures print error traces). So only mention them if they're the only thing causing | |
| 1150 | // the failure. | |
| 1151 | const show_err_logs: bool = show: { | |
| 1152 | var alt_results = s.test_results; | |
| 1153 | alt_results.log_err_count = 0; | |
| 1154 | break :show alt_results.isSuccess(); | |
| 1155 | }; | |
| 1156 | if (show_err_logs) { | |
| 1157 | try w.writeAll("; "); | |
| 1158 | try stderr.setColor(.red); | |
| 1159 | try w.print("{d} error logs", .{s.test_results.log_err_count}); | |
| 1160 | try stderr.setColor(.reset); | |
| 1161 | if (dim) try stderr.setColor(.dim); | |
| 1162 | } | |
| 1163 | ||
| 1164 | try w.writeAll("\n"); | |
| 1165 | } else if (s.result_error_msgs.items.len > 0) { | |
| 1166 | try stderr.setColor(.red); | |
| 1167 | try w.writeAll(" failure\n"); | |
| 1168 | } else { | |
| 1169 | assert(s.result_stderr.len > 0); | |
| 1170 | try stderr.setColor(.red); | |
| 1171 | try w.writeAll(" w\n"); | |
| 1172 | } | |
| 1173 | } | |
| 1174 | ||
| 1175 | fn printTreeStep( | |
| 1176 | b: *std.Build, | |
| 1177 | s: *Step, | |
| 1178 | run: *const Run, | |
| 1179 | stderr: Io.Terminal, | |
| 1180 | parent_node: *PrintNode, | |
| 1181 | step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 1182 | ) !void { | |
| 1183 | const writer = stderr.writer; | |
| 1184 | const first = step_stack.swapRemove(s); | |
| 1185 | const summary = run.summary; | |
| 1186 | const skip = switch (summary) { | |
| 1187 | .none, .line => unreachable, | |
| 1188 | .all => false, | |
| 1189 | .new => s.result_cached, | |
| 1190 | .failures => s.state == .success, | |
| 1191 | }; | |
| 1192 | if (skip) return; | |
| 1193 | try printPrefix(parent_node, stderr); | |
| 1194 | ||
| 1195 | if (parent_node.parent != null) { | |
| 1196 | if (parent_node.last) { | |
| 1197 | try printChildNodePrefix(stderr); | |
| 1198 | } else { | |
| 1199 | try writer.writeAll(switch (stderr.mode) { | |
| 1200 | .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ | |
| 1201 | else => "+- ", | |
| 1202 | }); | |
| 1203 | } | |
| 1204 | } | |
| 1205 | ||
| 1206 | if (!first) try stderr.setColor(.dim); | |
| 1207 | ||
| 1208 | // dep_prefix omitted here because it is redundant with the tree. | |
| 1209 | try writer.writeAll(s.name); | |
| 1210 | ||
| 1211 | if (first) { | |
| 1212 | try printStepStatus(s, stderr, run); | |
| 1213 | ||
| 1214 | const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: { | |
| 1215 | var i: usize = s.dependencies.items.len; | |
| 1216 | while (i > 0) { | |
| 1217 | i -= 1; | |
| 1218 | ||
| 1219 | const step = s.dependencies.items[i]; | |
| 1220 | const found = switch (summary) { | |
| 1221 | .all, .line, .none => unreachable, | |
| 1222 | .failures => step.state != .success, | |
| 1223 | .new => !step.result_cached, | |
| 1224 | }; | |
| 1225 | if (found) break :blk i; | |
| 1226 | } | |
| 1227 | break :blk s.dependencies.items.len -| 1; | |
| 1228 | }; | |
| 1229 | for (s.dependencies.items, 0..) |dep, i| { | |
| 1230 | var print_node: PrintNode = .{ | |
| 1231 | .parent = parent_node, | |
| 1232 | .last = i == last_index, | |
| 1233 | }; | |
| 1234 | try printTreeStep(b, dep, run, stderr, &print_node, step_stack); | |
| 1235 | } | |
| 1236 | } else { | |
| 1237 | if (s.dependencies.items.len == 0) { | |
| 1238 | try writer.writeAll(" (reused)\n"); | |
| 1239 | } else { | |
| 1240 | try writer.print(" (+{d} more reused dependencies)\n", .{ | |
| 1241 | s.dependencies.items.len, | |
| 1242 | }); | |
| 1243 | } | |
| 1244 | try stderr.setColor(.reset); | |
| 1245 | } | |
| 1246 | } | |
| 1247 | ||
| 1248 | /// Traverse the dependency graph depth-first and make it undirected by having | |
| 1249 | /// steps know their dependants (they only know dependencies at start). | |
| 1250 | /// Along the way, check that there is no dependency loop, and record the steps | |
| 1251 | /// in traversal order in `step_stack`. | |
| 1252 | /// Each step has its dependencies traversed in random order, this accomplishes | |
| 1253 | /// two things: | |
| 1254 | /// - `step_stack` will be in randomized-depth-first order, so the build runner | |
| 1255 | /// spawns initial steps in a random order | |
| 1256 | /// - each step's `dependants` list is also filled in a random order, so that | |
| 1257 | /// when it finishes executing in `makeStep`, it spawns next steps to run in | |
| 1258 | /// random order | |
| 1259 | fn constructGraphAndCheckForDependencyLoop( | |
| 1260 | gpa: Allocator, | |
| 1261 | b: *std.Build, | |
| 1262 | s: *Step, | |
| 1263 | step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 1264 | rand: std.Random, | |
| 1265 | ) !void { | |
| 1266 | switch (s.state) { | |
| 1267 | .precheck_started => { | |
| 1268 | std.debug.print("dependency loop detected:\n {s}\n", .{s.name}); | |
| 1269 | return error.DependencyLoopDetected; | |
| 1270 | }, | |
| 1271 | .precheck_unstarted => { | |
| 1272 | s.state = .precheck_started; | |
| 1273 | ||
| 1274 | try step_stack.ensureUnusedCapacity(gpa, s.dependencies.items.len); | |
| 1275 | ||
| 1276 | // We dupe to avoid shuffling the steps in the summary, it depends | |
| 1277 | // on s.dependencies' order. | |
| 1278 | const deps = gpa.dupe(*Step, s.dependencies.items) catch @panic("OOM"); | |
| 1279 | defer gpa.free(deps); | |
| 1280 | ||
| 1281 | rand.shuffle(*Step, deps); | |
| 1282 | ||
| 1283 | for (deps) |dep| { | |
| 1284 | try step_stack.put(gpa, dep, {}); | |
| 1285 | try dep.dependants.append(b.allocator, s); | |
| 1286 | constructGraphAndCheckForDependencyLoop(gpa, b, dep, step_stack, rand) catch |err| { | |
| 1287 | if (err == error.DependencyLoopDetected) { | |
| 1288 | std.debug.print(" {s}\n", .{s.name}); | |
| 1289 | } | |
| 1290 | return err; | |
| 1291 | }; | |
| 1292 | } | |
| 1293 | ||
| 1294 | s.state = .precheck_done; | |
| 1295 | s.pending_deps = @intCast(s.dependencies.items.len); | |
| 1296 | }, | |
| 1297 | .precheck_done => {}, | |
| 1298 | ||
| 1299 | // These don't happen until we actually run the step graph. | |
| 1300 | .dependency_failure => unreachable, | |
| 1301 | .success => unreachable, | |
| 1302 | .failure => unreachable, | |
| 1303 | .skipped => unreachable, | |
| 1304 | .skipped_oom => unreachable, | |
| 1305 | } | |
| 1306 | } | |
| 1307 | ||
| 1308 | /// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready | |
| 1309 | /// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must | |
| 1310 | /// have already subtracted this value from `run.available_rss`. This function will release the RSS | |
| 1311 | /// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked | |
| 1312 | /// steps after "make" completes for `s`. | |
| 1313 | fn makeStep( | |
| 1314 | group: *Io.Group, | |
| 1315 | b: *std.Build, | |
| 1316 | s: *Step, | |
| 1317 | root_prog_node: std.Progress.Node, | |
| 1318 | run: *Run, | |
| 1319 | ) Io.Cancelable!void { | |
| 1320 | const graph = b.graph; | |
| 1321 | const io = graph.io; | |
| 1322 | const gpa = run.gpa; | |
| 1323 | ||
| 1324 | { | |
| 1325 | const step_prog_node = root_prog_node.start(s.name, 0); | |
| 1326 | defer step_prog_node.end(); | |
| 1327 | ||
| 1328 | if (run.web_server) |*ws| ws.updateStepStatus(s, .wip); | |
| 1329 | ||
| 1330 | const new_state: Step.State = for (s.dependencies.items) |dep| { | |
| 1331 | switch (@atomicLoad(Step.State, &dep.state, .monotonic)) { | |
| 1332 | .precheck_unstarted => unreachable, | |
| 1333 | .precheck_started => unreachable, | |
| 1334 | .precheck_done => unreachable, | |
| 1335 | ||
| 1336 | .failure, | |
| 1337 | .dependency_failure, | |
| 1338 | .skipped_oom, | |
| 1339 | => break .dependency_failure, | |
| 1340 | ||
| 1341 | .success, .skipped => {}, | |
| 1342 | } | |
| 1343 | } else if (s.make(.{ | |
| 1344 | .progress_node = step_prog_node, | |
| 1345 | .watch = run.watch, | |
| 1346 | .web_server = if (run.web_server) |*ws| ws else null, | |
| 1347 | .unit_test_timeout_ns = run.unit_test_timeout_ns, | |
| 1348 | .gpa = gpa, | |
| 1349 | })) state: { | |
| 1350 | break :state .success; | |
| 1351 | } else |err| switch (err) { | |
| 1352 | error.MakeFailed => .failure, | |
| 1353 | error.MakeSkipped => .skipped, | |
| 1354 | }; | |
| 1355 | ||
| 1356 | @atomicStore(Step.State, &s.state, new_state, .monotonic); | |
| 1357 | ||
| 1358 | switch (new_state) { | |
| 1359 | .precheck_unstarted => unreachable, | |
| 1360 | .precheck_started => unreachable, | |
| 1361 | .precheck_done => unreachable, | |
| 1362 | ||
| 1363 | .failure, | |
| 1364 | .dependency_failure, | |
| 1365 | .skipped_oom, | |
| 1366 | => { | |
| 1367 | if (run.web_server) |*ws| ws.updateStepStatus(s, .failure); | |
| 1368 | std.Progress.setStatus(.failure_working); | |
| 1369 | }, | |
| 1370 | ||
| 1371 | .success, | |
| 1372 | .skipped, | |
| 1373 | => { | |
| 1374 | if (run.web_server) |*ws| ws.updateStepStatus(s, .success); | |
| 1375 | }, | |
| 1376 | } | |
| 1377 | } | |
| 1378 | ||
| 1379 | // No matter the result, we want to display error/warning messages. | |
| 1380 | if (s.result_error_bundle.errorMessageCount() > 0 or | |
| 1381 | s.result_error_msgs.items.len > 0 or | |
| 1382 | s.result_stderr.len > 0) | |
| 1383 | { | |
| 1384 | const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); | |
| 1385 | defer io.unlockStderr(); | |
| 1386 | printErrorMessages(gpa, s, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch {}; | |
| 1387 | } | |
| 1388 | ||
| 1389 | if (s.max_rss != 0) { | |
| 1390 | var dispatch_set: std.ArrayList(*Step) = .empty; | |
| 1391 | defer dispatch_set.deinit(gpa); | |
| 1392 | ||
| 1393 | // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` | |
| 1394 | // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held. | |
| 1395 | { | |
| 1396 | try run.max_rss_mutex.lock(io); | |
| 1397 | defer run.max_rss_mutex.unlock(io); | |
| 1398 | run.available_rss += s.max_rss; | |
| 1399 | dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch @panic("OOM"); | |
| 1400 | while (run.memory_blocked_steps.getLast()) |candidate| { | |
| 1401 | if (run.available_rss < candidate.max_rss) break; | |
| 1402 | assert(run.memory_blocked_steps.pop() == candidate); | |
| 1403 | dispatch_set.appendAssumeCapacity(candidate); | |
| 1404 | } | |
| 1405 | } | |
| 1406 | for (dispatch_set.items) |candidate| { | |
| 1407 | group.async(io, makeStep, .{ group, b, candidate, root_prog_node, run }); | |
| 1408 | } | |
| 1409 | } | |
| 1410 | ||
| 1411 | for (s.dependants.items) |dependant| { | |
| 1412 | // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. | |
| 1413 | if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { | |
| 1414 | try stepReady(group, b, dependant, root_prog_node, run); | |
| 1415 | } | |
| 1416 | } | |
| 1417 | } | |
| 1418 | ||
| 1419 | fn stepReady( | |
| 1420 | group: *Io.Group, | |
| 1421 | b: *std.Build, | |
| 1422 | s: *Step, | |
| 1423 | root_prog_node: std.Progress.Node, | |
| 1424 | run: *Run, | |
| 1425 | ) !void { | |
| 1426 | const io = b.graph.io; | |
| 1427 | if (s.max_rss != 0) { | |
| 1428 | try run.max_rss_mutex.lock(io); | |
| 1429 | defer run.max_rss_mutex.unlock(io); | |
| 1430 | if (run.available_rss < s.max_rss) { | |
| 1431 | // Running this step right now could possibly exceed the allotted RSS. | |
| 1432 | run.memory_blocked_steps.append(run.gpa, s) catch @panic("OOM"); | |
| 1433 | return; | |
| 1434 | } | |
| 1435 | run.available_rss -= s.max_rss; | |
| 1436 | } | |
| 1437 | group.async(io, makeStep, .{ group, b, s, root_prog_node, run }); | |
| 1438 | } | |
| 1439 | ||
| 1440 | pub fn printErrorMessages( | |
| 1441 | gpa: Allocator, | |
| 1442 | failing_step: *Step, | |
| 1443 | options: std.zig.ErrorBundle.RenderOptions, | |
| 1444 | stderr: Io.Terminal, | |
| 1445 | error_style: ErrorStyle, | |
| 1446 | multiline_errors: MultilineErrors, | |
| 1447 | ) !void { | |
| 1448 | const writer = stderr.writer; | |
| 1449 | if (error_style.verboseContext()) { | |
| 1450 | // Provide context for where these error messages are coming from by | |
| 1451 | // printing the corresponding Step subtree. | |
| 1452 | var step_stack: std.ArrayList(*Step) = .empty; | |
| 1453 | defer step_stack.deinit(gpa); | |
| 1454 | try step_stack.append(gpa, failing_step); | |
| 1455 | while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) { | |
| 1456 | try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]); | |
| 1457 | } | |
| 1458 | ||
| 1459 | // Now, `step_stack` has the subtree that we want to print, in reverse order. | |
| 1460 | try stderr.setColor(.dim); | |
| 1461 | var indent: usize = 0; | |
| 1462 | while (step_stack.pop()) |s| : (indent += 1) { | |
| 1463 | if (indent > 0) { | |
| 1464 | try writer.splatByteAll(' ', (indent - 1) * 3); | |
| 1465 | try printChildNodePrefix(stderr); | |
| 1466 | } | |
| 1467 | ||
| 1468 | try writer.writeAll(s.name); | |
| 1469 | ||
| 1470 | if (s == failing_step) { | |
| 1471 | try printStepFailure(s, stderr, true); | |
| 1472 | } else { | |
| 1473 | try writer.writeAll("\n"); | |
| 1474 | } | |
| 1475 | } | |
| 1476 | try stderr.setColor(.reset); | |
| 1477 | } else { | |
| 1478 | // Just print the failing step itself. | |
| 1479 | try stderr.setColor(.dim); | |
| 1480 | try writer.writeAll(failing_step.name); | |
| 1481 | try printStepFailure(failing_step, stderr, true); | |
| 1482 | try stderr.setColor(.reset); | |
| 1483 | } | |
| 1484 | ||
| 1485 | if (failing_step.result_stderr.len > 0) { | |
| 1486 | try writer.writeAll(failing_step.result_stderr); | |
| 1487 | if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { | |
| 1488 | try writer.writeAll("\n"); | |
| 1489 | } | |
| 1490 | } | |
| 1491 | ||
| 1492 | try failing_step.result_error_bundle.renderToTerminal(options, stderr); | |
| 1493 | ||
| 1494 | for (failing_step.result_error_msgs.items) |msg| { | |
| 1495 | try stderr.setColor(.red); | |
| 1496 | try writer.writeAll("error:"); | |
| 1497 | try stderr.setColor(.reset); | |
| 1498 | if (std.mem.indexOfScalar(u8, msg, '\n') == null) { | |
| 1499 | try writer.print(" {s}\n", .{msg}); | |
| 1500 | } else switch (multiline_errors) { | |
| 1501 | .indent => { | |
| 1502 | var it = std.mem.splitScalar(u8, msg, '\n'); | |
| 1503 | try writer.print(" {s}\n", .{it.first()}); | |
| 1504 | while (it.next()) |line| { | |
| 1505 | try writer.print(" {s}\n", .{line}); | |
| 1506 | } | |
| 1507 | }, | |
| 1508 | .newline => try writer.print("\n{s}\n", .{msg}), | |
| 1509 | .none => try writer.print(" {s}\n", .{msg}), | |
| 1510 | } | |
| 1511 | } | |
| 1512 | ||
| 1513 | if (error_style.verboseContext()) { | |
| 1514 | if (failing_step.result_failed_command) |cmd_str| { | |
| 1515 | try stderr.setColor(.red); | |
| 1516 | try writer.writeAll("failed command: "); | |
| 1517 | try stderr.setColor(.reset); | |
| 1518 | try writer.writeAll(cmd_str); | |
| 1519 | try writer.writeByte('\n'); | |
| 1520 | } | |
| 1521 | } | |
| 1522 | ||
| 1523 | try writer.writeByte('\n'); | |
| 1524 | } | |
| 1525 | ||
| 1526 | fn printSteps(builder: *std.Build, w: *Writer) !void { | |
| 1527 | const arena = builder.graph.arena; | |
| 1528 | for (builder.top_level_steps.values()) |top_level_step| { | |
| 1529 | const name = if (&top_level_step.step == builder.default_step) | |
| 1530 | try fmt.allocPrint(arena, "{s} (default)", .{top_level_step.step.name}) | |
| 1531 | else | |
| 1532 | top_level_step.step.name; | |
| 1533 | try w.print(" {s:<28} {s}\n", .{ name, top_level_step.description }); | |
| 1534 | } | |
| 1535 | } | |
| 1536 | ||
| 1537 | fn printUsage(b: *std.Build, w: *Writer) !void { | |
| 1538 | const arena = b.graph.arena; | |
| 1539 | ||
| 1540 | try w.print( | |
| 1541 | \\Usage: {s} build [steps] [options] | |
| 1542 | \\ | |
| 1543 | \\Steps: | |
| 1544 | \\ | |
| 1545 | , .{b.graph.zig_exe}); | |
| 1546 | try printSteps(b, w); | |
| 1547 | try w.writeAll( | |
| 1548 | \\ | |
| 1549 | \\Project-Specific Options: | |
| 1550 | \\ | |
| 1551 | ); | |
| 1552 | ||
| 1553 | if (b.available_options_list.items.len == 0) { | |
| 1554 | try w.print(" (none)\n", .{}); | |
| 1555 | } else { | |
| 1556 | for (b.available_options_list.items) |option| { | |
| 1557 | const name = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ option.name, option.type_id }); | |
| 1558 | try w.print("{s:<30} {s}\n", .{ name, option.description }); | |
| 1559 | if (option.enum_options) |enum_options| { | |
| 1560 | const padding: [33]u8 = @splat(' '); | |
| 1561 | try w.writeAll(padding ++ "Supported Values:\n"); | |
| 1562 | for (enum_options) |enum_option| { | |
| 1563 | try w.print(padding ++ " {s}\n", .{enum_option}); | |
| 1564 | } | |
| 1565 | } | |
| 1566 | } | |
| 1567 | } | |
| 1568 | ||
| 1569 | try w.writeAll( | |
| 1570 | \\ | |
| 1571 | \\System Integration Options: | |
| 1572 | \\ --search-prefix [path] Add a path to look for binaries, libraries, headers | |
| 1573 | \\ --sysroot [path] Set the system root directory (usually /) | |
| 1574 | \\ --libc [file] Provide a file which specifies libc paths | |
| 1575 | \\ | |
| 1576 | \\ --system [pkgdir] Disable package fetching; enable all integrations | |
| 1577 | \\ -fsys=[name] Enable a system integration | |
| 1578 | \\ -fno-sys=[name] Disable a system integration | |
| 1579 | \\ | |
| 1580 | \\ -fdarling, -fno-darling Integration with system-installed Darling to | |
| 1581 | \\ execute macOS programs on Linux hosts | |
| 1582 | \\ (default: no) | |
| 1583 | \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute | |
| 1584 | \\ foreign-architecture programs on Linux hosts | |
| 1585 | \\ (default: no) | |
| 1586 | \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc | |
| 1587 | \\ (e.g. glibc or musl) built for multiple foreign | |
| 1588 | \\ architectures, allowing execution of non-native | |
| 1589 | \\ programs that link with libc. | |
| 1590 | \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on | |
| 1591 | \\ ARM64 macOS hosts. (default: no) | |
| 1592 | \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to | |
| 1593 | \\ execute WASI binaries. (default: no) | |
| 1594 | \\ -fwine, -fno-wine Integration with system-installed Wine to execute | |
| 1595 | \\ Windows programs on Linux hosts. (default: no) | |
| 1596 | \\ | |
| 1597 | \\ Available System Integrations: Enabled: | |
| 1598 | \\ | |
| 1599 | ); | |
| 1600 | if (b.graph.system_library_options.entries.len == 0) { | |
| 1601 | try w.writeAll(" (none) -\n"); | |
| 1602 | } else { | |
| 1603 | for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { | |
| 1604 | const status = switch (v) { | |
| 1605 | .declared_enabled => "yes", | |
| 1606 | .declared_disabled => "no", | |
| 1607 | .user_enabled, .user_disabled => unreachable, // already emitted error | |
| 1608 | }; | |
| 1609 | try w.print(" {s:<43} {s}\n", .{ k, status }); | |
| 1610 | } | |
| 1611 | } | |
| 1612 | ||
| 1613 | try w.writeAll( | |
| 1614 | \\ | |
| 1615 | \\General Options: | |
| 1616 | \\ -h, --help Print this help and exit | |
| 1617 | \\ -l, --list-steps Print available steps | |
| 1618 | \\ | |
| 1619 | \\ -p, --prefix [path] Where to install files (default: zig-out) | |
| 1620 | \\ --prefix-lib-dir [path] Where to install libraries | |
| 1621 | \\ --prefix-exe-dir [path] Where to install executables | |
| 1622 | \\ --prefix-include-dir [path] Where to install C header files | |
| 1623 | \\ --release[=mode] Request release mode, optionally specifying a | |
| 1624 | \\ preferred optimization mode: fast, safe, small | |
| 1625 | \\ | |
| 1626 | \\ --verbose Print commands before executing them | |
| 1627 | \\ --color [auto|off|on] Enable or disable colored error messages | |
| 1628 | \\ --error-style [style] Control how build errors are printed | |
| 1629 | \\ verbose (Default) Report errors with full context | |
| 1630 | \\ minimal Report errors after summary, excluding context like command lines | |
| 1631 | \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update | |
| 1632 | \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update | |
| 1633 | \\ --multiline-errors [style] Control how multi-line error messages are printed | |
| 1634 | \\ indent (Default) Indent non-initial lines to align with initial line | |
| 1635 | \\ newline Include a leading newline so that the error message is on its own lines | |
| 1636 | \\ none Print as usual so the first line is misaligned | |
| 1637 | \\ --summary [mode] Control the printing of the build summary | |
| 1638 | \\ all Print the build summary in its entirety | |
| 1639 | \\ new Omit cached steps | |
| 1640 | \\ failures (Default if short-lived) Only print failed steps | |
| 1641 | \\ line (Default if long-lived) Only print the single-line summary | |
| 1642 | \\ none Do not print the build summary | |
| 1643 | \\ -j<N> Limit concurrent jobs (default is to use all CPU cores) | |
| 1644 | \\ --maxrss <bytes> Limit memory usage (default is to use available memory) | |
| 1645 | \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss | |
| 1646 | \\ --test-timeout <timeout> Limit execution time of unit tests, terminating if exceeded. | |
| 1647 | \\ The timeout must include a unit: ns, us, ms, s, m, h | |
| 1648 | \\ --watch Continuously rebuild when source files are modified | |
| 1649 | \\ --debounce <ms> Delay before rebuilding after changed file detected | |
| 1650 | \\ --webui[=ip] Enable the web interface on the given IP address | |
| 1651 | \\ --fuzz[=limit] Continuously search for unit test failures with an optional | |
| 1652 | \\ limit to the max number of iterations. The argument supports | |
| 1653 | \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies | |
| 1654 | \\ '--webui' when no limit is specified. | |
| 1655 | \\ --time-report Force full rebuild and provide detailed information on | |
| 1656 | \\ compilation time of Zig source code (implies '--webui') | |
| 1657 | \\ -fincremental Enable incremental compilation | |
| 1658 | \\ -fno-incremental Disable incremental compilation | |
| 1659 | \\ | |
| 1660 | \\Package Management Options: | |
| 1661 | \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit | |
| 1662 | \\ needed (Default) Lazy dependencies are fetched as needed | |
| 1663 | \\ all Lazy dependencies are always fetched | |
| 1664 | \\ --fork=[path] Override one or more projects from dependency tree | |
| 1665 | \\ | |
| 1666 | \\Advanced Options: | |
| 1667 | \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error | |
| 1668 | \\ -fno-reference-trace Disable reference trace | |
| 1669 | \\ -fallow-so-scripts Allows .so files to be GNU ld scripts | |
| 1670 | \\ -fno-allow-so-scripts (default) .so files must be ELF files | |
| 1671 | \\ --build-file [file] Override path to build.zig | |
| 1672 | \\ --cache-dir [path] Override path to local Zig cache directory | |
| 1673 | \\ --global-cache-dir [path] Override path to global Zig cache directory | |
| 1674 | \\ --zig-lib-dir [arg] Override path to Zig lib directory | |
| 1675 | \\ --build-runner [file] Override path to build runner | |
| 1676 | \\ --seed [integer] For shuffling dependency traversal order (default: random) | |
| 1677 | \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries | |
| 1678 | \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) | |
| 1679 | \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) | |
| 1680 | \\ md5 16-byte cryptographic hash (ELF) | |
| 1681 | \\ uuid 16-byte random UUID (ELF, WASM) | |
| 1682 | \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) | |
| 1683 | \\ none (default) No build ID | |
| 1684 | \\ --debug-log [scope] Enable debugging the compiler | |
| 1685 | \\ --debug-pkg-config Fail if unknown pkg-config flags encountered | |
| 1686 | \\ --debug-rt Debug compiler runtime libraries | |
| 1687 | \\ --verbose-link Enable compiler debug output for linking | |
| 1688 | \\ --verbose-air Enable compiler debug output for Zig AIR | |
| 1689 | \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR | |
| 1690 | \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC | |
| 1691 | \\ --verbose-cimport Enable compiler debug output for C imports | |
| 1692 | \\ --verbose-cc Enable compiler debug output for C compilation | |
| 1693 | \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features | |
| 1694 | \\ | |
| 1695 | ); | |
| 1696 | } | |
| 1697 | ||
| 1698 | fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { | |
| 1699 | if (idx.* >= args.len) return null; | |
| 1700 | defer idx.* += 1; | |
| 1701 | return args[idx.*]; | |
| 1702 | } | |
| 1703 | ||
| 1704 | fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { | |
| 1705 | return nextArg(args, idx) orelse { | |
| 1706 | std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); | |
| 1707 | process.exit(1); | |
| 1708 | }; | |
| 1709 | } | |
| 1710 | ||
| 1711 | fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { | |
| 1712 | if (idx >= args.len) return null; | |
| 1713 | return args[idx..]; | |
| 1714 | } | |
| 1715 | ||
| 1716 | const Color = std.zig.Color; | |
| 1717 | const ErrorStyle = enum { | |
| 1718 | verbose, | |
| 1719 | minimal, | |
| 1720 | verbose_clear, | |
| 1721 | minimal_clear, | |
| 1722 | fn verboseContext(s: ErrorStyle) bool { | |
| 1723 | return switch (s) { | |
| 1724 | .verbose, .verbose_clear => true, | |
| 1725 | .minimal, .minimal_clear => false, | |
| 1726 | }; | |
| 1727 | } | |
| 1728 | fn clearOnUpdate(s: ErrorStyle) bool { | |
| 1729 | return switch (s) { | |
| 1730 | .verbose, .minimal => false, | |
| 1731 | .verbose_clear, .minimal_clear => true, | |
| 1732 | }; | |
| 1733 | } | |
| 1734 | }; | |
| 1735 | const MultilineErrors = enum { indent, newline, none }; | |
| 1736 | const Summary = enum { all, new, failures, line, none }; | |
| 1737 | ||
| 1738 | fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { | |
| 1739 | std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); | |
| 1740 | process.exit(1); | |
| 1741 | } | |
| 1742 | ||
| 1743 | fn validateSystemLibraryOptions(b: *std.Build) void { | |
| 1744 | var bad = false; | |
| 1745 | for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { | |
| 1746 | switch (v) { | |
| 1747 | .user_disabled, .user_enabled => { | |
| 1748 | // The user tried to enable or disable a system library integration, but | |
| 1749 | // the build script did not recognize that option. | |
| 1750 | std.debug.print("system library name not recognized by build script: '{s}'\n", .{k}); | |
| 1751 | bad = true; | |
| 1752 | }, | |
| 1753 | .declared_disabled, .declared_enabled => {}, | |
| 1754 | } | |
| 1755 | } | |
| 1756 | if (bad) { | |
| 1757 | std.debug.print(" access the help menu with 'zig build -h'\n", .{}); | |
| 1758 | process.exit(1); | |
| 1759 | } | |
| 1760 | } | |
| 1761 | ||
| 1762 | /// Starting from all top-level steps in `b`, traverses the entire step graph | |
| 1763 | /// and adds all step dependencies implied by module graphs. | |
| 1764 | fn createModuleDependencies(b: *std.Build) Allocator.Error!void { | |
| 1765 | const arena = b.graph.arena; | |
| 1766 | ||
| 1767 | var all_steps: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; | |
| 1768 | var next_step_idx: usize = 0; | |
| 1769 | ||
| 1770 | try all_steps.ensureUnusedCapacity(arena, b.top_level_steps.count()); | |
| 1771 | for (b.top_level_steps.values()) |tls| { | |
| 1772 | all_steps.putAssumeCapacityNoClobber(&tls.step, {}); | |
| 1773 | } | |
| 1774 | ||
| 1775 | while (next_step_idx < all_steps.count()) { | |
| 1776 | const step = all_steps.keys()[next_step_idx]; | |
| 1777 | next_step_idx += 1; | |
| 1778 | ||
| 1779 | // Set up any implied dependencies for this step. It's important that we do this first, so | |
| 1780 | // that the loop below discovers steps implied by the module graph. | |
| 1781 | try createModuleDependenciesForStep(step); | |
| 1782 | ||
| 1783 | try all_steps.ensureUnusedCapacity(arena, step.dependencies.items.len); | |
| 1784 | for (step.dependencies.items) |other_step| { | |
| 1785 | all_steps.putAssumeCapacity(other_step, {}); | |
| 1786 | } | |
| 1787 | } | |
| 1788 | } | |
| 1789 | ||
| 1790 | /// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which | |
| 1791 | /// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. | |
| 1792 | fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { | |
| 1793 | const root_module = if (step.cast(Step.Compile)) |cs| root: { | |
| 1794 | break :root cs.root_module; | |
| 1795 | } else return; // not a compile step so no module dependencies | |
| 1796 | ||
| 1797 | // Starting from `root_module`, discover all modules in this graph. | |
| 1798 | const modules = root_module.getGraph().modules; | |
| 1799 | ||
| 1800 | // For each of those modules, set up the implied step dependencies. | |
| 1801 | for (modules) |mod| { | |
| 1802 | if (mod.root_source_file) |lp| lp.addStepDependencies(step); | |
| 1803 | for (mod.include_dirs.items) |include_dir| switch (include_dir) { | |
| 1804 | .path, | |
| 1805 | .path_system, | |
| 1806 | .path_after, | |
| 1807 | .framework_path, | |
| 1808 | .framework_path_system, | |
| 1809 | .embed_path, | |
| 1810 | => |lp| lp.addStepDependencies(step), | |
| 1811 | ||
| 1812 | .other_step => |other| { | |
| 1813 | other.getEmittedIncludeTree().addStepDependencies(step); | |
| 1814 | step.dependOn(&other.step); | |
| 1815 | }, | |
| 1816 | ||
| 1817 | .config_header_step => |other| step.dependOn(&other.step), | |
| 1818 | }; | |
| 1819 | for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); | |
| 1820 | for (mod.rpaths.items) |rpath| switch (rpath) { | |
| 1821 | .lazy_path => |lp| lp.addStepDependencies(step), | |
| 1822 | .special => {}, | |
| 1823 | }; | |
| 1824 | for (mod.link_objects.items) |link_object| switch (link_object) { | |
| 1825 | .static_path, | |
| 1826 | .assembly_file, | |
| 1827 | => |lp| lp.addStepDependencies(step), | |
| 1828 | .other_step => |other| step.dependOn(&other.step), | |
| 1829 | .system_lib => {}, | |
| 1830 | .c_source_file => |source| source.file.addStepDependencies(step), | |
| 1831 | .c_source_files => |source_files| source_files.root.addStepDependencies(step), | |
| 1832 | .win32_resource_file => |rc_source| { | |
| 1833 | rc_source.file.addStepDependencies(step); | |
| 1834 | for (rc_source.include_paths) |lp| lp.addStepDependencies(step); | |
| 1835 | }, | |
| 1836 | }; | |
| 1837 | } | |
| 1838 | } | |
| 1839 | ||
| 1840 | var stdio_buffer_allocation: [256]u8 = undefined; | |
| 1841 | var stdout_writer_allocation: Io.File.Writer = undefined; | |
| 1842 | ||
| 1843 | fn initStdoutWriter(io: Io) *Writer { | |
| 1844 | stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); | |
| 1845 | return &stdout_writer_allocation.interface; | |
| 1846 | } | |
| 1847 | ||
| 1848 | fn cleanTmpFiles(io: Io, steps: []const *Step) void { | |
| 1849 | for (steps) |step| { | |
| 1850 | const wf = step.cast(std.Build.Step.WriteFile) orelse continue; | |
| 1851 | if (wf.mode != .tmp) continue; | |
| 1852 | const path = wf.generated_directory.path orelse continue; | |
| 1853 | Io.Dir.cwd().deleteTree(io, path) catch |err| { | |
| 1854 | std.log.warn("failed to delete {s}: {t}", .{ path, err }); | |
| 1855 | }; | |
| 1856 | } | |
| 1857 | } |
lib/compiler/configure_runner.zig+168-88
| ... | ... | @@ -8,12 +8,11 @@ const mem = std.mem; |
| 8 | 8 | const process = std.process; |
| 9 | 9 | const File = std.Io.File; |
| 10 | 10 | const Step = std.Build.Step; |
| 11 | const Watch = std.Build.Watch; | |
| 12 | const WebServer = std.Build.WebServer; | |
| 13 | 11 | const Allocator = std.mem.Allocator; |
| 14 | 12 | const fatal = std.process.fatal; |
| 15 | 13 | const Writer = std.Io.Writer; |
| 16 | 14 | const Color = std.zig.Color; |
| 15 | const Configuration = std.Build.Configuration; | |
| 17 | 16 | |
| 18 | 17 | pub const root = @import("@build"); |
| 19 | 18 | pub const dependencies = @import("@dependencies"); |
| ... | ... | @@ -26,7 +25,11 @@ pub const std_options: std.Options = .{ |
| 26 | 25 | pub fn main(init: process.Init.Minimal) !void { |
| 27 | 26 | // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not |
| 28 | 27 | // always the case. So, we do need a true gpa for some things. |
| 29 | var debug_gpa_state: std.heap.DebugAllocator(.{}) = .init; | |
| 28 | var debug_gpa_state: std.heap.DebugAllocator(.{ | |
| 29 | // We'd rather have `zig build` run faster than catch harmless leaks in | |
| 30 | // the user's build.zig script. | |
| 31 | .stack_trace_frames = 0, | |
| 32 | }) = .init; | |
| 30 | 33 | defer _ = debug_gpa_state.deinit(); |
| 31 | 34 | const gpa = debug_gpa_state.allocator(); |
| 32 | 35 | |
| ... | ... | @@ -47,11 +50,11 @@ pub fn main(init: process.Init.Minimal) !void { |
| 47 | 50 | // skip my own exe name |
| 48 | 51 | var arg_idx: usize = 1; |
| 49 | 52 | |
| 50 | const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); | |
| 51 | const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); | |
| 52 | const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); | |
| 53 | const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); | |
| 54 | const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); | |
| 53 | const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); | |
| 54 | const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); | |
| 55 | const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); | |
| 56 | const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); | |
| 57 | const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); | |
| 55 | 58 | |
| 56 | 59 | const cwd: Io.Dir = .cwd(); |
| 57 | 60 | |
| ... | ... | @@ -66,8 +69,8 @@ pub fn main(init: process.Init.Minimal) !void { |
| 66 | 69 | }; |
| 67 | 70 | |
| 68 | 71 | const local_cache_directory: std.Build.Cache.Directory = .{ |
| 69 | .path = cache_root, | |
| 70 | .handle = try cwd.createDirPathOpen(io, cache_root, .{}), | |
| 72 | .path = local_cache_root, | |
| 73 | .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), | |
| 71 | 74 | }; |
| 72 | 75 | |
| 73 | 76 | const global_cache_directory: std.Build.Cache.Directory = .{ |
| ... | ... | @@ -137,8 +140,6 @@ pub fn main(init: process.Init.Minimal) !void { |
| 137 | 140 | if (try builder.addUserInputFlag(option_contents)) |
| 138 | 141 | fatal(" access the help menu with 'zig build -h'", .{}); |
| 139 | 142 | } |
| 140 | } else if (mem.eql(u8, arg, "--verbose")) { | |
| 141 | builder.verbose = true; | |
| 142 | 143 | } else if (mem.startsWith(u8, arg, "-fsys=")) { |
| 143 | 144 | const name = arg["-fsys=".len..]; |
| 144 | 145 | graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); |
| ... | ... | @@ -146,19 +147,14 @@ pub fn main(init: process.Init.Minimal) !void { |
| 146 | 147 | const name = arg["-fno-sys=".len..]; |
| 147 | 148 | graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); |
| 148 | 149 | } else if (mem.eql(u8, arg, "--release")) { |
| 149 | builder.release_mode = .any; | |
| 150 | graph.release_mode = .any; | |
| 150 | 151 | } else if (mem.startsWith(u8, arg, "--release=")) { |
| 151 | 152 | const text = arg["--release=".len..]; |
| 152 | builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { | |
| 153 | graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { | |
| 153 | 154 | fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ |
| 154 | 155 | arg, text, |
| 155 | 156 | }); |
| 156 | 157 | }; |
| 157 | } else if (mem.eql(u8, arg, "--search-prefix")) { | |
| 158 | const search_prefix = nextArgOrFatal(args, &arg_idx); | |
| 159 | builder.addSearchPrefix(search_prefix); | |
| 160 | } else if (mem.eql(u8, arg, "--libc")) { | |
| 161 | builder.libc_file = nextArgOrFatal(args, &arg_idx); | |
| 162 | 158 | } else if (mem.eql(u8, arg, "--color")) { |
| 163 | 159 | const next_arg = nextArg(args, &arg_idx) orelse |
| 164 | 160 | fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); |
| ... | ... | @@ -196,8 +192,6 @@ pub fn main(init: process.Init.Minimal) !void { |
| 196 | 192 | style, @errorName(err), |
| 197 | 193 | }); |
| 198 | 194 | }; |
| 199 | } else if (mem.eql(u8, arg, "--debug-pkg-config")) { | |
| 200 | builder.debug_pkg_config = true; | |
| 201 | 195 | } else if (mem.eql(u8, arg, "--debug-rt")) { |
| 202 | 196 | graph.debug_compiler_runtime_libs = true; |
| 203 | 197 | } else if (mem.eql(u8, arg, "--debug-compile-errors")) { |
| ... | ... | @@ -209,71 +203,11 @@ pub fn main(init: process.Init.Minimal) !void { |
| 209 | 203 | // but it is handled by the parent process. The build runner |
| 210 | 204 | // only sees this flag. |
| 211 | 205 | graph.system_package_mode = true; |
| 212 | } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { | |
| 213 | // --glibc-runtimes was the old name of the flag; kept for compatibility for now. | |
| 214 | builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); | |
| 215 | } else if (mem.eql(u8, arg, "--verbose-link")) { | |
| 216 | builder.verbose_link = true; | |
| 217 | } else if (mem.eql(u8, arg, "--verbose-air")) { | |
| 218 | builder.verbose_air = true; | |
| 219 | } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { | |
| 220 | builder.verbose_llvm_ir = "-"; | |
| 221 | } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { | |
| 222 | builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; | |
| 223 | } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { | |
| 224 | builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; | |
| 225 | } else if (mem.eql(u8, arg, "--verbose-cimport")) { | |
| 226 | builder.verbose_cimport = true; | |
| 227 | } else if (mem.eql(u8, arg, "--verbose-cc")) { | |
| 228 | builder.verbose_cc = true; | |
| 229 | } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { | |
| 230 | builder.verbose_llvm_cpu_features = true; | |
| 231 | } else if (mem.eql(u8, arg, "-fincremental")) { | |
| 232 | graph.incremental = true; | |
| 233 | } else if (mem.eql(u8, arg, "-fno-incremental")) { | |
| 234 | graph.incremental = false; | |
| 235 | } else if (mem.eql(u8, arg, "-fwine")) { | |
| 236 | builder.enable_wine = true; | |
| 237 | } else if (mem.eql(u8, arg, "-fno-wine")) { | |
| 238 | builder.enable_wine = false; | |
| 239 | } else if (mem.eql(u8, arg, "-fqemu")) { | |
| 240 | builder.enable_qemu = true; | |
| 241 | } else if (mem.eql(u8, arg, "-fno-qemu")) { | |
| 242 | builder.enable_qemu = false; | |
| 243 | } else if (mem.eql(u8, arg, "-fwasmtime")) { | |
| 244 | builder.enable_wasmtime = true; | |
| 245 | } else if (mem.eql(u8, arg, "-fno-wasmtime")) { | |
| 246 | builder.enable_wasmtime = false; | |
| 247 | } else if (mem.eql(u8, arg, "-frosetta")) { | |
| 248 | builder.enable_rosetta = true; | |
| 249 | } else if (mem.eql(u8, arg, "-fno-rosetta")) { | |
| 250 | builder.enable_rosetta = false; | |
| 251 | } else if (mem.eql(u8, arg, "-fdarling")) { | |
| 252 | builder.enable_darling = true; | |
| 253 | } else if (mem.eql(u8, arg, "-fno-darling")) { | |
| 254 | builder.enable_darling = false; | |
| 255 | } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { | |
| 256 | graph.allow_so_scripts = true; | |
| 257 | } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { | |
| 258 | graph.allow_so_scripts = false; | |
| 259 | } else if (mem.eql(u8, arg, "-freference-trace")) { | |
| 260 | builder.reference_trace = 256; | |
| 261 | } else if (mem.startsWith(u8, arg, "-freference-trace=")) { | |
| 262 | const num = arg["-freference-trace=".len..]; | |
| 263 | builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { | |
| 264 | std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); | |
| 265 | process.exit(1); | |
| 266 | }; | |
| 267 | } else if (mem.eql(u8, arg, "-fno-reference-trace")) { | |
| 268 | builder.reference_trace = null; | |
| 269 | 206 | } else if (mem.cutPrefix(u8, arg, "-j")) |text| { |
| 270 | 207 | const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| |
| 271 | 208 | fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); |
| 272 | 209 | if (n < 1) fatal("number of jobs must be at least 1", .{}); |
| 273 | 210 | threaded.setAsyncLimit(.limited(n)); |
| 274 | } else if (mem.eql(u8, arg, "--")) { | |
| 275 | builder.args = argsRest(args, arg_idx); | |
| 276 | break; | |
| 277 | 211 | } else { |
| 278 | 212 | fatalWithHint("unrecognized argument: '{s}'", .{arg}); |
| 279 | 213 | } |
| ... | ... | @@ -289,6 +223,150 @@ pub fn main(init: process.Init.Minimal) !void { |
| 289 | 223 | }; |
| 290 | 224 | |
| 291 | 225 | try builder.runBuild(root); |
| 226 | ||
| 227 | var wc: Configuration.Wip = .init(gpa); | |
| 228 | defer wc.deinit(); | |
| 229 | ||
| 230 | var stdout_buffer: [1024]u8 = undefined; | |
| 231 | var file_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); | |
| 232 | serialize(builder, &wc, &file_writer.interface) catch |err| switch (err) { | |
| 233 | error.WriteFailed => fatal("failed to write configuration output: {t}", .{file_writer.err.?}), | |
| 234 | error.OutOfMemory => |e| return e, | |
| 235 | }; | |
| 236 | ||
| 237 | // This executable is short-lived and run in Debug mode, so we'd rather | |
| 238 | // have `zig build` run faster than catch resource leaks in the user's | |
| 239 | // build.zig script (or, frankly, this configure runner), therefore we call | |
| 240 | // exit directly here rather than cleanExit. | |
| 241 | process.exit(0); | |
| 242 | } | |
| 243 | ||
| 244 | fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { | |
| 245 | const graph = b.graph; | |
| 246 | const arena = graph.arena; | |
| 247 | const gpa = wc.gpa; | |
| 248 | ||
| 249 | // Starting from all top-level steps in `b`, traverse the entire step graph | |
| 250 | // and add all step dependencies implied by module graphs. | |
| 251 | const top_level_steps = b.top_level_steps.values(); | |
| 252 | // Index corresponds to `Configuration.steps` index. | |
| 253 | var step_map: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; | |
| 254 | try step_map.ensureUnusedCapacity(arena, top_level_steps.len); | |
| 255 | for (top_level_steps) |tls| { | |
| 256 | step_map.putAssumeCapacityNoClobber(&tls.step, {}); | |
| 257 | } | |
| 258 | { | |
| 259 | while (wc.steps.items.len < step_map.count()) { | |
| 260 | const step = step_map.keys()[wc.steps.items.len]; | |
| 261 | ||
| 262 | // Set up any implied dependencies for this step. It's important that we do this first, so | |
| 263 | // that the loop below discovers steps implied by the module graph. | |
| 264 | try createModuleDependenciesForStep(step); | |
| 265 | ||
| 266 | try step_map.ensureUnusedCapacity(arena, step.dependencies.items.len); | |
| 267 | for (step.dependencies.items) |other_step| { | |
| 268 | step_map.putAssumeCapacity(other_step, {}); | |
| 269 | } | |
| 270 | ||
| 271 | // Add and then de-duplicate dependencies. | |
| 272 | const deps = d: { | |
| 273 | const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len); | |
| 274 | for (try wc.prepareDeps(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step| | |
| 275 | dep.* = @intCast(step_map.getIndex(dep_step).?); | |
| 276 | break :d try wc.dedupeDeps(deps); | |
| 277 | }; | |
| 278 | ||
| 279 | try wc.steps.ensureTotalCapacity(gpa, step_map.entries.capacity); | |
| 280 | wc.steps.appendAssumeCapacity(.{ | |
| 281 | .name = try wc.addString(step.name), | |
| 282 | .flags = .{ .tag = step.tag }, | |
| 283 | .deps = deps, | |
| 284 | .extra_index = switch (step.tag) { | |
| 285 | .top_level => e: { | |
| 286 | const top_level: *Step.TopLevel = @fieldParentPtr("step", step); | |
| 287 | break :e try wc.addExtra(@as(Configuration.Step.TopLevel, .{ | |
| 288 | .description = try wc.addString(top_level.description), | |
| 289 | })); | |
| 290 | }, | |
| 291 | .compile => @panic("TODO"), | |
| 292 | .install_artifact => @panic("TODO"), | |
| 293 | .install_file => @panic("TODO"), | |
| 294 | .install_dir => @panic("TODO"), | |
| 295 | .remove_dir => @panic("TODO"), | |
| 296 | .fail => @panic("TODO"), | |
| 297 | .fmt => @panic("TODO"), | |
| 298 | .translate_c => @panic("TODO"), | |
| 299 | .write_file => @panic("TODO"), | |
| 300 | .update_source_files => @panic("TODO"), | |
| 301 | .run => @panic("TODO"), | |
| 302 | .check_file => @panic("TODO"), | |
| 303 | .check_object => @panic("TODO"), | |
| 304 | .config_header => @panic("TODO"), | |
| 305 | .objcopy => @panic("TODO"), | |
| 306 | .options => @panic("TODO"), | |
| 307 | }, | |
| 308 | }); | |
| 309 | } | |
| 310 | } | |
| 311 | ||
| 312 | try wc.unlazy_deps.ensureUnusedCapacity(gpa, graph.needed_lazy_dependencies.keys().len); | |
| 313 | for (graph.needed_lazy_dependencies.keys()) |k| { | |
| 314 | wc.unlazy_deps.appendAssumeCapacity(try wc.addString(k)); | |
| 315 | } | |
| 316 | ||
| 317 | try wc.write(writer, .{ | |
| 318 | .default_step = @intCast(step_map.getIndex(b.default_step).?), | |
| 319 | }); | |
| 320 | } | |
| 321 | ||
| 322 | /// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which | |
| 323 | /// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. | |
| 324 | fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { | |
| 325 | const root_module = if (step.cast(Step.Compile)) |cs| root: { | |
| 326 | break :root cs.root_module; | |
| 327 | } else return; // not a compile step so no module dependencies | |
| 328 | ||
| 329 | // Starting from `root_module`, discover all modules in this graph. | |
| 330 | const modules = root_module.getGraph().modules; | |
| 331 | ||
| 332 | // For each of those modules, set up the implied step dependencies. | |
| 333 | for (modules) |mod| { | |
| 334 | if (mod.root_source_file) |lp| lp.addStepDependencies(step); | |
| 335 | for (mod.include_dirs.items) |include_dir| switch (include_dir) { | |
| 336 | .path, | |
| 337 | .path_system, | |
| 338 | .path_after, | |
| 339 | .framework_path, | |
| 340 | .framework_path_system, | |
| 341 | .embed_path, | |
| 342 | => |lp| lp.addStepDependencies(step), | |
| 343 | ||
| 344 | .other_step => |other| { | |
| 345 | other.getEmittedIncludeTree().addStepDependencies(step); | |
| 346 | step.dependOn(&other.step); | |
| 347 | }, | |
| 348 | ||
| 349 | .config_header_step => |other| step.dependOn(&other.step), | |
| 350 | }; | |
| 351 | for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); | |
| 352 | for (mod.rpaths.items) |rpath| switch (rpath) { | |
| 353 | .lazy_path => |lp| lp.addStepDependencies(step), | |
| 354 | .special => {}, | |
| 355 | }; | |
| 356 | for (mod.link_objects.items) |link_object| switch (link_object) { | |
| 357 | .static_path, | |
| 358 | .assembly_file, | |
| 359 | => |lp| lp.addStepDependencies(step), | |
| 360 | .other_step => |other| step.dependOn(&other.step), | |
| 361 | .system_lib => {}, | |
| 362 | .c_source_file => |source| source.file.addStepDependencies(step), | |
| 363 | .c_source_files => |source_files| source_files.root.addStepDependencies(step), | |
| 364 | .win32_resource_file => |rc_source| { | |
| 365 | rc_source.file.addStepDependencies(step); | |
| 366 | for (rc_source.include_paths) |lp| lp.addStepDependencies(step); | |
| 367 | }, | |
| 368 | }; | |
| 369 | } | |
| 292 | 370 | } |
| 293 | 371 | |
| 294 | 372 | fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { |
| ... | ... | @@ -299,14 +377,17 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { |
| 299 | 377 | |
| 300 | 378 | fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { |
| 301 | 379 | return nextArg(args, idx) orelse { |
| 302 | std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); | |
| 303 | process.exit(1); | |
| 380 | fatal("expected argument after {q}\n access the help menu with \"zig build -h\"", .{ | |
| 381 | args[idx.* - 1], | |
| 382 | }); | |
| 304 | 383 | }; |
| 305 | 384 | } |
| 306 | 385 | |
| 307 | fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { | |
| 308 | if (idx >= args.len) return null; | |
| 309 | return args[idx..]; | |
| 386 | fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { | |
| 387 | const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); | |
| 388 | if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); | |
| 389 | const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); | |
| 390 | return arg; | |
| 310 | 391 | } |
| 311 | 392 | |
| 312 | 393 | const ErrorStyle = enum { |
| ... | ... | @@ -331,6 +412,5 @@ const MultilineErrors = enum { indent, newline, none }; |
| 331 | 412 | const Summary = enum { all, new, failures, line, none }; |
| 332 | 413 | |
| 333 | 414 | fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { |
| 334 | std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); | |
| 335 | process.exit(1); | |
| 415 | fatal(f ++ "\n access the help menu with \"zig build -h\"", args); | |
| 336 | 416 | } |
lib/compiler/maker.zig created+1717
| ... | ... | @@ -0,0 +1,1717 @@ |
| 1 | const builtin = @import("builtin"); | |
| 2 | ||
| 3 | const std = @import("std"); | |
| 4 | const Io = std.Io; | |
| 5 | const assert = std.debug.assert; | |
| 6 | const fmt = std.fmt; | |
| 7 | const mem = std.mem; | |
| 8 | const process = std.process; | |
| 9 | const File = std.Io.File; | |
| 10 | const Allocator = std.mem.Allocator; | |
| 11 | const fatal = std.process.fatal; | |
| 12 | const Writer = std.Io.Writer; | |
| 13 | const Cache = std.Build.Cache; | |
| 14 | const Configuration = std.Build.Configuration; | |
| 15 | ||
| 16 | const Fuzz = @import("maker/Fuzz.zig"); | |
| 17 | const Graph = @import("maker/Graph.zig"); | |
| 18 | const Step = @import("maker/Step.zig"); | |
| 19 | const Watch = @import("maker/Watch.zig"); | |
| 20 | const WebServer = @import("maker/WebServer.zig"); | |
| 21 | ||
| 22 | pub const std_options: std.Options = .{ | |
| 23 | .side_channels_mitigations = .none, | |
| 24 | .http_disable_tls = true, | |
| 25 | }; | |
| 26 | ||
| 27 | pub fn main(init: process.Init.Minimal) !void { | |
| 28 | // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not | |
| 29 | // always the case. So, we do need a true gpa for some things. | |
| 30 | var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); | |
| 31 | defer _ = safe_gpa_state.deinit(); | |
| 32 | const gpa = safe_gpa_state.allocator(); | |
| 33 | ||
| 34 | var threaded: std.Io.Threaded = .init(gpa, .{ | |
| 35 | .environ = init.environ, | |
| 36 | .argv0 = .init(init.args), | |
| 37 | }); | |
| 38 | defer threaded.deinit(); | |
| 39 | const io = threaded.io(); | |
| 40 | ||
| 41 | // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. | |
| 42 | var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); | |
| 43 | defer arena_instance.deinit(); | |
| 44 | const arena = arena_instance.allocator(); | |
| 45 | ||
| 46 | const args = try init.args.toSlice(arena); | |
| 47 | ||
| 48 | // skip my own exe name | |
| 49 | var arg_idx: usize = 1; | |
| 50 | ||
| 51 | const zig_exe = cutArgPrefixOrFatal(args, &arg_idx, "--zig="); | |
| 52 | const zig_lib_dir = cutArgPrefixOrFatal(args, &arg_idx, "--lib="); | |
| 53 | const build_root = cutArgPrefixOrFatal(args, &arg_idx, "--build-root="); | |
| 54 | const local_cache_root = cutArgPrefixOrFatal(args, &arg_idx, "--local-cache="); | |
| 55 | const global_cache_root = cutArgPrefixOrFatal(args, &arg_idx, "--global-cache="); | |
| 56 | const configure_path = cutArgPrefixOrFatal(args, &arg_idx, "--configure="); | |
| 57 | ||
| 58 | const cwd: Io.Dir = .cwd(); | |
| 59 | ||
| 60 | const zig_lib_directory: Cache.Directory = .{ | |
| 61 | .path = zig_lib_dir, | |
| 62 | .handle = try cwd.openDir(io, zig_lib_dir, .{}), | |
| 63 | }; | |
| 64 | ||
| 65 | const build_root_directory: Cache.Directory = .{ | |
| 66 | .path = build_root, | |
| 67 | .handle = try cwd.openDir(io, build_root, .{}), | |
| 68 | }; | |
| 69 | ||
| 70 | const local_cache_directory: Cache.Directory = .{ | |
| 71 | .path = local_cache_root, | |
| 72 | .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), | |
| 73 | }; | |
| 74 | ||
| 75 | const global_cache_directory: Cache.Directory = .{ | |
| 76 | .path = global_cache_root, | |
| 77 | .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), | |
| 78 | }; | |
| 79 | ||
| 80 | var graph: Graph = .{ | |
| 81 | .io = io, | |
| 82 | .arena = arena, | |
| 83 | .cache = .{ | |
| 84 | .io = io, | |
| 85 | .gpa = gpa, | |
| 86 | .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), | |
| 87 | .cwd = try process.currentPathAlloc(io, arena), | |
| 88 | }, | |
| 89 | .zig_exe = zig_exe, | |
| 90 | .environ_map = try init.environ.createMap(arena), | |
| 91 | .global_cache_root = global_cache_directory, | |
| 92 | .zig_lib_directory = zig_lib_directory, | |
| 93 | .host = .{ | |
| 94 | .query = .{}, | |
| 95 | .result = try std.zig.system.resolveTargetQuery(io, .{}), | |
| 96 | }, | |
| 97 | .time_report = false, | |
| 98 | }; | |
| 99 | ||
| 100 | graph.cache.addPrefix(.{ .path = null, .handle = cwd }); | |
| 101 | graph.cache.addPrefix(build_root_directory); | |
| 102 | graph.cache.addPrefix(local_cache_directory); | |
| 103 | graph.cache.addPrefix(global_cache_directory); | |
| 104 | graph.cache.hash.addBytes(builtin.zig_version_string); | |
| 105 | ||
| 106 | var targets = std.array_list.Managed([]const u8).init(arena); | |
| 107 | var debug_log_scopes = std.array_list.Managed([]const u8).init(arena); | |
| 108 | ||
| 109 | var install_prefix: ?[]const u8 = null; | |
| 110 | var dir_list: std.Build.DirList = .{}; | |
| 111 | var error_style: ErrorStyle = .verbose; | |
| 112 | var multiline_errors: MultilineErrors = .indent; | |
| 113 | var summary: ?Summary = null; | |
| 114 | var max_rss: u64 = 0; | |
| 115 | var skip_oom_steps = false; | |
| 116 | var test_timeout_ns: ?u64 = null; | |
| 117 | var color: Color = .auto; | |
| 118 | var help_menu = false; | |
| 119 | var steps_menu = false; | |
| 120 | var watch = false; | |
| 121 | var fuzz: ?Fuzz.Mode = null; | |
| 122 | var debounce_interval_ms: u16 = 50; | |
| 123 | var webui_listen: ?Io.net.IpAddress = null; | |
| 124 | var verbose = false; | |
| 125 | var sysroot: ?[]const u8 = null; | |
| 126 | var search_prefixes: std.ArrayList([]const u8) = .empty; | |
| 127 | var libc_file: ?[]const u8 = null; | |
| 128 | var debug_pkg_config: bool = false; | |
| 129 | // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, | |
| 130 | // this will be the directory $glibc-build-dir/install/glibcs | |
| 131 | // Given the example of the aarch64 target, this is the directory | |
| 132 | // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. | |
| 133 | // Also works for dynamic musl. | |
| 134 | var libc_runtimes_dir: ?[]const u8 = null; | |
| 135 | var enable_wine = false; | |
| 136 | var enable_qemu = false; | |
| 137 | var enable_wasmtime = false; | |
| 138 | var enable_darling = false; | |
| 139 | var enable_rosetta = false; | |
| 140 | var reference_trace: ?u32 = null; | |
| 141 | var run_args: ?[]const []const u8 = null; | |
| 142 | ||
| 143 | if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { | |
| 144 | if (std.meta.stringToEnum(ErrorStyle, str)) |style| { | |
| 145 | error_style = style; | |
| 146 | } | |
| 147 | } | |
| 148 | ||
| 149 | if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { | |
| 150 | if (std.meta.stringToEnum(MultilineErrors, str)) |style| { | |
| 151 | multiline_errors = style; | |
| 152 | } | |
| 153 | } | |
| 154 | ||
| 155 | var configuration: Configuration = undefined; | |
| 156 | { | |
| 157 | var file = cwd.openFile(io, configure_path, .{}) catch |err| | |
| 158 | fatal("failed to open configuration file {f}: {t}", .{ configure_path, err }); | |
| 159 | defer file.close(io); | |
| 160 | configuration = Configuration.load(arena, io, file) catch |err| | |
| 161 | fatal("failed to load configuration file {f}: {t}", .{ configure_path, err }); | |
| 162 | } | |
| 163 | graph.configuration = &configuration; | |
| 164 | graph.scanConfiguration(); | |
| 165 | ||
| 166 | std.log.err("TODO handle user -D options", .{}); | |
| 167 | ||
| 168 | while (nextArg(args, &arg_idx)) |arg| { | |
| 169 | if (mem.startsWith(u8, arg, "-")) { | |
| 170 | if (mem.eql(u8, arg, "--verbose")) { | |
| 171 | verbose = true; | |
| 172 | } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { | |
| 173 | help_menu = true; | |
| 174 | } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { | |
| 175 | install_prefix = nextArgOrFatal(args, &arg_idx); | |
| 176 | } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { | |
| 177 | steps_menu = true; | |
| 178 | } else if (mem.startsWith(u8, arg, "-fsys=")) { | |
| 179 | const name = arg["-fsys=".len..]; | |
| 180 | graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); | |
| 181 | } else if (mem.startsWith(u8, arg, "-fno-sys=")) { | |
| 182 | const name = arg["-fno-sys=".len..]; | |
| 183 | graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); | |
| 184 | } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { | |
| 185 | dir_list.lib_dir = nextArgOrFatal(args, &arg_idx); | |
| 186 | } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { | |
| 187 | dir_list.exe_dir = nextArgOrFatal(args, &arg_idx); | |
| 188 | } else if (mem.eql(u8, arg, "--prefix-include-dir")) { | |
| 189 | dir_list.include_dir = nextArgOrFatal(args, &arg_idx); | |
| 190 | } else if (mem.eql(u8, arg, "--sysroot")) { | |
| 191 | sysroot = nextArgOrFatal(args, &arg_idx); | |
| 192 | } else if (mem.eql(u8, arg, "--maxrss")) { | |
| 193 | const max_rss_text = nextArgOrFatal(args, &arg_idx); | |
| 194 | max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { | |
| 195 | std.debug.print("invalid byte size: '{s}': {s}\n", .{ | |
| 196 | max_rss_text, @errorName(err), | |
| 197 | }); | |
| 198 | process.exit(1); | |
| 199 | }; | |
| 200 | } else if (mem.eql(u8, arg, "--skip-oom-steps")) { | |
| 201 | skip_oom_steps = true; | |
| 202 | } else if (mem.eql(u8, arg, "--test-timeout")) { | |
| 203 | const units: []const struct { []const u8, u64 } = &.{ | |
| 204 | .{ "ns", 1 }, | |
| 205 | .{ "nanosecond", 1 }, | |
| 206 | .{ "us", std.time.ns_per_us }, | |
| 207 | .{ "microsecond", std.time.ns_per_us }, | |
| 208 | .{ "ms", std.time.ns_per_ms }, | |
| 209 | .{ "millisecond", std.time.ns_per_ms }, | |
| 210 | .{ "s", std.time.ns_per_s }, | |
| 211 | .{ "second", std.time.ns_per_s }, | |
| 212 | .{ "m", std.time.ns_per_min }, | |
| 213 | .{ "minute", std.time.ns_per_min }, | |
| 214 | .{ "h", std.time.ns_per_hour }, | |
| 215 | .{ "hour", std.time.ns_per_hour }, | |
| 216 | }; | |
| 217 | const timeout_str = nextArgOrFatal(args, &arg_idx); | |
| 218 | const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( | |
| 219 | "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)", | |
| 220 | .{timeout_str}, | |
| 221 | ); | |
| 222 | const num_str = timeout_str[0 .. num_end_idx + 1]; | |
| 223 | const unit_str = timeout_str[num_end_idx + 1 ..]; | |
| 224 | const unit_factor: f64 = for (units) |unit_and_factor| { | |
| 225 | if (std.mem.eql(u8, unit_str, unit_and_factor[0])) { | |
| 226 | break @floatFromInt(unit_and_factor[1]); | |
| 227 | } | |
| 228 | } else fatal( | |
| 229 | "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)", | |
| 230 | .{ timeout_str, unit_str }, | |
| 231 | ); | |
| 232 | const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal( | |
| 233 | "invalid timeout '{s}': invalid number '{s}' ({t})", | |
| 234 | .{ timeout_str, num_str, err }, | |
| 235 | ); | |
| 236 | test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); | |
| 237 | } else if (mem.eql(u8, arg, "--search-prefix")) { | |
| 238 | try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); | |
| 239 | } else if (mem.eql(u8, arg, "--libc")) { | |
| 240 | libc_file = nextArgOrFatal(args, &arg_idx); | |
| 241 | } else if (mem.eql(u8, arg, "--color")) { | |
| 242 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 243 | fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); | |
| 244 | color = std.meta.stringToEnum(Color, next_arg) orelse { | |
| 245 | fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ | |
| 246 | arg, next_arg, | |
| 247 | }); | |
| 248 | }; | |
| 249 | } else if (mem.eql(u8, arg, "--error-style")) { | |
| 250 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 251 | fatalWithHint("expected style after '{s}'", .{arg}); | |
| 252 | error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { | |
| 253 | fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); | |
| 254 | }; | |
| 255 | } else if (mem.eql(u8, arg, "--multiline-errors")) { | |
| 256 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 257 | fatalWithHint("expected style after '{s}'", .{arg}); | |
| 258 | multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { | |
| 259 | fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); | |
| 260 | }; | |
| 261 | } else if (mem.eql(u8, arg, "--summary")) { | |
| 262 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 263 | fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg}); | |
| 264 | summary = std.meta.stringToEnum(Summary, next_arg) orelse { | |
| 265 | fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{ | |
| 266 | arg, next_arg, | |
| 267 | }); | |
| 268 | }; | |
| 269 | } else if (mem.eql(u8, arg, "--seed")) { | |
| 270 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 271 | fatalWithHint("expected u32 after '{s}'", .{arg}); | |
| 272 | graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { | |
| 273 | fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ | |
| 274 | next_arg, @errorName(err), | |
| 275 | }); | |
| 276 | }; | |
| 277 | } else if (mem.eql(u8, arg, "--debounce")) { | |
| 278 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 279 | fatalWithHint("expected u16 after '{s}'", .{arg}); | |
| 280 | debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { | |
| 281 | fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{ | |
| 282 | next_arg, err, | |
| 283 | }); | |
| 284 | }; | |
| 285 | } else if (mem.eql(u8, arg, "--webui")) { | |
| 286 | if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; | |
| 287 | } else if (mem.startsWith(u8, arg, "--webui=")) { | |
| 288 | const addr_str = arg["--webui=".len..]; | |
| 289 | if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); | |
| 290 | webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { | |
| 291 | fatal("invalid web UI address '{s}': {s}", .{ addr_str, @errorName(err) }); | |
| 292 | }; | |
| 293 | } else if (mem.eql(u8, arg, "--debug-log")) { | |
| 294 | const next_arg = nextArgOrFatal(args, &arg_idx); | |
| 295 | try debug_log_scopes.append(next_arg); | |
| 296 | } else if (mem.eql(u8, arg, "--debug-pkg-config")) { | |
| 297 | debug_pkg_config = true; | |
| 298 | } else if (mem.eql(u8, arg, "--debug-rt")) { | |
| 299 | graph.debug_compiler_runtime_libs = .Debug; | |
| 300 | } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { | |
| 301 | graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse | |
| 302 | fatal("unrecognized optimization mode: {s}", .{rest}); | |
| 303 | } else if (mem.eql(u8, arg, "--system")) { | |
| 304 | // The usage text shows another argument after this parameter | |
| 305 | // but it is handled by the parent process. The build runner | |
| 306 | // only sees this flag. | |
| 307 | graph.system_package_mode = true; | |
| 308 | } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { | |
| 309 | // --glibc-runtimes was the old name of the flag; kept for compatibility for now. | |
| 310 | libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); | |
| 311 | } else if (mem.eql(u8, arg, "--watch")) { | |
| 312 | watch = true; | |
| 313 | } else if (mem.eql(u8, arg, "--time-report")) { | |
| 314 | graph.time_report = true; | |
| 315 | if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; | |
| 316 | } else if (mem.eql(u8, arg, "--fuzz")) { | |
| 317 | fuzz = .{ .forever = undefined }; | |
| 318 | if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; | |
| 319 | } else if (mem.startsWith(u8, arg, "--fuzz=")) { | |
| 320 | const value = arg["--fuzz=".len..]; | |
| 321 | if (value.len == 0) fatal("missing argument to --fuzz", .{}); | |
| 322 | ||
| 323 | const unit: u8 = value[value.len - 1]; | |
| 324 | const digits = switch (unit) { | |
| 325 | '0'...'9' => value, | |
| 326 | 'K', 'M', 'G' => value[0 .. value.len - 1], | |
| 327 | else => fatal( | |
| 328 | "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", | |
| 329 | .{}, | |
| 330 | ), | |
| 331 | }; | |
| 332 | ||
| 333 | const amount = std.fmt.parseInt(u64, digits, 10) catch { | |
| 334 | fatal( | |
| 335 | "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", | |
| 336 | .{}, | |
| 337 | ); | |
| 338 | }; | |
| 339 | ||
| 340 | const normalized_amount = std.math.mul(u64, amount, switch (unit) { | |
| 341 | else => unreachable, | |
| 342 | '0'...'9' => 1, | |
| 343 | 'K' => 1000, | |
| 344 | 'M' => 1_000_000, | |
| 345 | 'G' => 1_000_000_000, | |
| 346 | }) catch fatal("fuzzing limit amount overflows u64", .{}); | |
| 347 | ||
| 348 | fuzz = .{ | |
| 349 | .limit = .{ | |
| 350 | .amount = normalized_amount, | |
| 351 | }, | |
| 352 | }; | |
| 353 | } else if (mem.eql(u8, arg, "-fincremental")) { | |
| 354 | graph.incremental = true; | |
| 355 | } else if (mem.eql(u8, arg, "-fno-incremental")) { | |
| 356 | graph.incremental = false; | |
| 357 | } else if (mem.eql(u8, arg, "-fwine")) { | |
| 358 | enable_wine = true; | |
| 359 | } else if (mem.eql(u8, arg, "-fno-wine")) { | |
| 360 | enable_wine = false; | |
| 361 | } else if (mem.eql(u8, arg, "-fqemu")) { | |
| 362 | enable_qemu = true; | |
| 363 | } else if (mem.eql(u8, arg, "-fno-qemu")) { | |
| 364 | enable_qemu = false; | |
| 365 | } else if (mem.eql(u8, arg, "-fwasmtime")) { | |
| 366 | enable_wasmtime = true; | |
| 367 | } else if (mem.eql(u8, arg, "-fno-wasmtime")) { | |
| 368 | enable_wasmtime = false; | |
| 369 | } else if (mem.eql(u8, arg, "-frosetta")) { | |
| 370 | enable_rosetta = true; | |
| 371 | } else if (mem.eql(u8, arg, "-fno-rosetta")) { | |
| 372 | enable_rosetta = false; | |
| 373 | } else if (mem.eql(u8, arg, "-fdarling")) { | |
| 374 | enable_darling = true; | |
| 375 | } else if (mem.eql(u8, arg, "-fno-darling")) { | |
| 376 | enable_darling = false; | |
| 377 | } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { | |
| 378 | graph.allow_so_scripts = true; | |
| 379 | } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { | |
| 380 | graph.allow_so_scripts = false; | |
| 381 | } else if (mem.eql(u8, arg, "-freference-trace")) { | |
| 382 | reference_trace = 256; | |
| 383 | } else if (mem.startsWith(u8, arg, "-freference-trace=")) { | |
| 384 | const num = arg["-freference-trace=".len..]; | |
| 385 | reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { | |
| 386 | std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); | |
| 387 | process.exit(1); | |
| 388 | }; | |
| 389 | } else if (mem.eql(u8, arg, "-fno-reference-trace")) { | |
| 390 | reference_trace = null; | |
| 391 | } else if (mem.cutPrefix(u8, arg, "-j")) |text| { | |
| 392 | const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| | |
| 393 | fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); | |
| 394 | if (n < 1) fatal("number of jobs must be at least 1", .{}); | |
| 395 | threaded.setAsyncLimit(.limited(n)); | |
| 396 | graph.max_jobs = n; | |
| 397 | } else if (mem.eql(u8, arg, "--")) { | |
| 398 | run_args = argsRest(args, arg_idx); | |
| 399 | break; | |
| 400 | } else { | |
| 401 | fatalWithHint("unrecognized argument: '{s}'", .{arg}); | |
| 402 | } | |
| 403 | } else { | |
| 404 | try targets.append(arg); | |
| 405 | } | |
| 406 | } | |
| 407 | ||
| 408 | const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); | |
| 409 | const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); | |
| 410 | ||
| 411 | graph.stderr_mode = switch (color) { | |
| 412 | .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), | |
| 413 | .on => .escape_codes, | |
| 414 | .off => .no_color, | |
| 415 | }; | |
| 416 | ||
| 417 | if (webui_listen != null) { | |
| 418 | if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); | |
| 419 | if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); | |
| 420 | } | |
| 421 | ||
| 422 | const main_progress_node = std.Progress.start(io, .{ | |
| 423 | .disable_printing = (color == .off), | |
| 424 | }); | |
| 425 | defer main_progress_node.end(); | |
| 426 | ||
| 427 | graph.resolveInstallPrefix(install_prefix, dir_list); | |
| 428 | ||
| 429 | if (graph.validateUserInputDidItFail()) { | |
| 430 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 431 | } | |
| 432 | ||
| 433 | validateSystemLibraryOptions(&graph); | |
| 434 | ||
| 435 | if (help_menu) { | |
| 436 | var w = initStdoutWriter(io); | |
| 437 | printUsage(&graph, w) catch return stdout_writer_allocation.err.?; | |
| 438 | w.flush() catch return stdout_writer_allocation.err.?; | |
| 439 | return; | |
| 440 | } | |
| 441 | ||
| 442 | if (steps_menu) { | |
| 443 | var w = initStdoutWriter(io); | |
| 444 | printSteps(&graph, w) catch return stdout_writer_allocation.err.?; | |
| 445 | w.flush() catch return stdout_writer_allocation.err.?; | |
| 446 | return; | |
| 447 | } | |
| 448 | ||
| 449 | var run: Run = .{ | |
| 450 | .gpa = gpa, | |
| 451 | ||
| 452 | .available_rss = max_rss, | |
| 453 | .max_rss_is_default = false, | |
| 454 | .max_rss_mutex = .init, | |
| 455 | .skip_oom_steps = skip_oom_steps, | |
| 456 | .unit_test_timeout_ns = test_timeout_ns, | |
| 457 | ||
| 458 | .watch = watch, | |
| 459 | .web_server = undefined, // set after `prepare` | |
| 460 | .memory_blocked_steps = .empty, | |
| 461 | .step_stack = .empty, | |
| 462 | ||
| 463 | .error_style = error_style, | |
| 464 | .multiline_errors = multiline_errors, | |
| 465 | .summary = summary orelse if (watch or webui_listen != null) .line else .failures, | |
| 466 | }; | |
| 467 | defer { | |
| 468 | run.memory_blocked_steps.deinit(gpa); | |
| 469 | run.step_stack.deinit(gpa); | |
| 470 | } | |
| 471 | ||
| 472 | if (run.available_rss == 0) { | |
| 473 | run.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); | |
| 474 | run.max_rss_is_default = true; | |
| 475 | } | |
| 476 | ||
| 477 | prepare(arena, &graph, targets.items, &run) catch |err| switch (err) { | |
| 478 | error.DependencyLoopDetected, error.InsufficientMemory => { | |
| 479 | // Perhaps in the future there could be an Advanced Options flag | |
| 480 | // such as --debug-build-runner-leaks which would make this code | |
| 481 | // return instead of calling exit. | |
| 482 | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; | |
| 483 | process.exit(1); | |
| 484 | }, | |
| 485 | else => |e| return e, | |
| 486 | }; | |
| 487 | ||
| 488 | var w: Watch = w: { | |
| 489 | if (!watch) break :w undefined; | |
| 490 | if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); | |
| 491 | break :w try .init(graph.cache.cwd); | |
| 492 | }; | |
| 493 | ||
| 494 | const now = Io.Clock.Timestamp.now(io, .awake); | |
| 495 | ||
| 496 | run.web_server = if (webui_listen) |listen_address| ws: { | |
| 497 | if (builtin.single_threaded) unreachable; // `fatal` above | |
| 498 | break :ws .init(.{ | |
| 499 | .gpa = gpa, | |
| 500 | .graph = &graph, | |
| 501 | .all_steps = run.step_stack.keys(), | |
| 502 | .root_prog_node = main_progress_node, | |
| 503 | .watch = watch, | |
| 504 | .listen_address = listen_address, | |
| 505 | .base_timestamp = now, | |
| 506 | }); | |
| 507 | } else null; | |
| 508 | ||
| 509 | if (run.web_server) |*ws| { | |
| 510 | ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); | |
| 511 | } | |
| 512 | ||
| 513 | rebuild: while (true) : (if (run.error_style.clearOnUpdate()) { | |
| 514 | const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); | |
| 515 | defer io.unlockStderr(); | |
| 516 | try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H"); | |
| 517 | }) { | |
| 518 | if (run.web_server) |*ws| ws.startBuild(); | |
| 519 | ||
| 520 | try runStepNames(graph, targets.items, main_progress_node, &run, fuzz); | |
| 521 | ||
| 522 | if (run.web_server) |*web_server| { | |
| 523 | if (fuzz) |mode| if (mode != .forever) fatal( | |
| 524 | "error: limited fuzzing is not implemented yet for --webui", | |
| 525 | .{}, | |
| 526 | ); | |
| 527 | ||
| 528 | web_server.finishBuild(.{ .fuzz = fuzz != null }); | |
| 529 | } | |
| 530 | ||
| 531 | if (run.web_server) |*ws| { | |
| 532 | assert(!watch); // fatal error after CLI parsing | |
| 533 | while (true) switch (try ws.wait()) { | |
| 534 | .rebuild => { | |
| 535 | for (run.step_stack.keys()) |step| { | |
| 536 | step.state = .precheck_done; | |
| 537 | step.pending_deps = @intCast(step.dependencies.items.len); | |
| 538 | step.reset(gpa); | |
| 539 | } | |
| 540 | continue :rebuild; | |
| 541 | }, | |
| 542 | }; | |
| 543 | } | |
| 544 | ||
| 545 | // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. | |
| 546 | if (!Watch.have_impl) unreachable; | |
| 547 | ||
| 548 | try w.update(gpa, run.step_stack.keys()); | |
| 549 | ||
| 550 | // Wait until a file system notification arrives. Read all such events | |
| 551 | // until the buffer is empty. Then wait for a debounce interval, resetting | |
| 552 | // if any more events come in. After the debounce interval has passed, | |
| 553 | // trigger a rebuild on all steps with modified inputs, as well as their | |
| 554 | // recursive dependants. | |
| 555 | var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; | |
| 556 | const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ | |
| 557 | w.dir_count, countSubProcesses(run.step_stack.keys()), | |
| 558 | }) catch &caption_buf; | |
| 559 | var debouncing_node = main_progress_node.start(caption, 0); | |
| 560 | var in_debounce = false; | |
| 561 | while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { | |
| 562 | .timeout => { | |
| 563 | assert(in_debounce); | |
| 564 | debouncing_node.end(); | |
| 565 | markFailedStepsDirty(gpa, run.step_stack.keys()); | |
| 566 | continue :rebuild; | |
| 567 | }, | |
| 568 | .dirty => if (!in_debounce) { | |
| 569 | in_debounce = true; | |
| 570 | debouncing_node.end(); | |
| 571 | debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); | |
| 572 | }, | |
| 573 | .clean => {}, | |
| 574 | }; | |
| 575 | } | |
| 576 | } | |
| 577 | ||
| 578 | fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { | |
| 579 | for (all_steps) |step| switch (step.state) { | |
| 580 | .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), | |
| 581 | else => continue, | |
| 582 | }; | |
| 583 | // Now that all dirty steps have been found, the remaining steps that | |
| 584 | // succeeded from last run shall be marked "cached". | |
| 585 | for (all_steps) |step| switch (step.state) { | |
| 586 | .success => step.result_cached = true, | |
| 587 | else => continue, | |
| 588 | }; | |
| 589 | } | |
| 590 | ||
| 591 | fn countSubProcesses(all_steps: []const *Step) usize { | |
| 592 | var count: usize = 0; | |
| 593 | for (all_steps) |s| { | |
| 594 | count += @intFromBool(s.getZigProcess() != null); | |
| 595 | } | |
| 596 | return count; | |
| 597 | } | |
| 598 | ||
| 599 | const Run = struct { | |
| 600 | gpa: Allocator, | |
| 601 | ||
| 602 | available_rss: usize, | |
| 603 | max_rss_is_default: bool, | |
| 604 | max_rss_mutex: Io.Mutex, | |
| 605 | skip_oom_steps: bool, | |
| 606 | unit_test_timeout_ns: ?u64, | |
| 607 | watch: bool, | |
| 608 | web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, | |
| 609 | /// Allocated into `gpa`. | |
| 610 | memory_blocked_steps: std.ArrayList(*Step), | |
| 611 | /// Allocated into `gpa`. | |
| 612 | step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 613 | ||
| 614 | error_style: ErrorStyle, | |
| 615 | multiline_errors: MultilineErrors, | |
| 616 | summary: Summary, | |
| 617 | }; | |
| 618 | ||
| 619 | fn prepare(graph: *Graph, step_names: []const []const u8, run: *Run) !void { | |
| 620 | const arena = graph.arena; | |
| 621 | const seed: u32 = graph.random_seed; | |
| 622 | const gpa = run.gpa; | |
| 623 | const step_stack = &run.step_stack; | |
| 624 | ||
| 625 | if (step_names.len == 0) { | |
| 626 | try step_stack.put(gpa, graph.configuration.default_step, {}); | |
| 627 | } else { | |
| 628 | try step_stack.ensureUnusedCapacity(gpa, step_names.len); | |
| 629 | for (0..step_names.len) |i| { | |
| 630 | const step_name = step_names[step_names.len - i - 1]; | |
| 631 | const s = graph.top_level_steps.get(step_name) orelse { | |
| 632 | std.log.info("access the help menu with \"zig build -h\"", .{}); | |
| 633 | fatal("no step named '{s}'", .{step_name}); | |
| 634 | }; | |
| 635 | step_stack.putAssumeCapacity(&s.step, {}); | |
| 636 | } | |
| 637 | } | |
| 638 | ||
| 639 | const starting_steps = try arena.dupe(*Step, step_stack.keys()); | |
| 640 | ||
| 641 | var rng = std.Random.DefaultPrng.init(seed); | |
| 642 | const rand = rng.random(); | |
| 643 | rand.shuffle(*Step, starting_steps); | |
| 644 | ||
| 645 | for (starting_steps) |s| { | |
| 646 | try constructGraphAndCheckForDependencyLoop(gpa, s, &run.step_stack, rand); | |
| 647 | } | |
| 648 | ||
| 649 | { | |
| 650 | // Check that we have enough memory to complete the build. | |
| 651 | var any_problems = false; | |
| 652 | var max_needed: usize = 0; | |
| 653 | for (step_stack.keys()) |s| { | |
| 654 | if (s.max_rss == 0) continue; | |
| 655 | max_needed = @max(max_needed, s.max_rss); | |
| 656 | if (s.max_rss > run.available_rss) { | |
| 657 | if (run.skip_oom_steps) { | |
| 658 | s.state = .skipped_oom; | |
| 659 | for (s.dependants.items) |dependant| { | |
| 660 | dependant.pending_deps -= 1; | |
| 661 | } | |
| 662 | } else { | |
| 663 | std.log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ | |
| 664 | s.owner.dep_prefix, s.name, s.max_rss, run.available_rss, | |
| 665 | }); | |
| 666 | any_problems = true; | |
| 667 | } | |
| 668 | } | |
| 669 | } | |
| 670 | if (any_problems) { | |
| 671 | if (run.max_rss_is_default) { | |
| 672 | std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ | |
| 673 | max_needed, | |
| 674 | }); | |
| 675 | } | |
| 676 | return error.InsufficientMemory; | |
| 677 | } | |
| 678 | } | |
| 679 | } | |
| 680 | ||
| 681 | fn runStepNames( | |
| 682 | graph: *Graph, | |
| 683 | step_names: []const []const u8, | |
| 684 | parent_prog_node: std.Progress.Node, | |
| 685 | run: *Run, | |
| 686 | fuzz: ?Fuzz.Mode, | |
| 687 | ) !void { | |
| 688 | const gpa = run.gpa; | |
| 689 | const io = graph.io; | |
| 690 | const step_stack = &run.step_stack; | |
| 691 | ||
| 692 | { | |
| 693 | // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, | |
| 694 | // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking | |
| 695 | // a step is initial when it actually became ready due to an earlier initial step. | |
| 696 | var initial_set: std.ArrayList(*Step) = .empty; | |
| 697 | defer initial_set.deinit(gpa); | |
| 698 | try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); | |
| 699 | for (step_stack.keys()) |s| { | |
| 700 | if (s.state == .precheck_done and s.pending_deps == 0) { | |
| 701 | initial_set.appendAssumeCapacity(s); | |
| 702 | } | |
| 703 | } | |
| 704 | ||
| 705 | const step_prog = parent_prog_node.start("steps", step_stack.count()); | |
| 706 | defer step_prog.end(); | |
| 707 | ||
| 708 | var group: Io.Group = .init; | |
| 709 | defer group.cancel(io); | |
| 710 | // Start working on all of the initial steps... | |
| 711 | for (initial_set.items) |s| try stepReady(&group, s, step_prog, run); | |
| 712 | // ...and `makeStep` will trigger every other step when their last dependency finishes. | |
| 713 | try group.await(io); | |
| 714 | } | |
| 715 | ||
| 716 | assert(run.memory_blocked_steps.items.len == 0); | |
| 717 | ||
| 718 | var test_pass_count: usize = 0; | |
| 719 | var test_skip_count: usize = 0; | |
| 720 | var test_fail_count: usize = 0; | |
| 721 | var test_crash_count: usize = 0; | |
| 722 | var test_timeout_count: usize = 0; | |
| 723 | ||
| 724 | var test_count: usize = 0; | |
| 725 | ||
| 726 | var success_count: usize = 0; | |
| 727 | var skipped_count: usize = 0; | |
| 728 | var failure_count: usize = 0; | |
| 729 | var pending_count: usize = 0; | |
| 730 | var total_compile_errors: usize = 0; | |
| 731 | ||
| 732 | var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); | |
| 733 | defer cleanup_task.await(io); | |
| 734 | ||
| 735 | for (step_stack.keys()) |s| { | |
| 736 | test_pass_count += s.test_results.passCount(); | |
| 737 | test_skip_count += s.test_results.skip_count; | |
| 738 | test_fail_count += s.test_results.fail_count; | |
| 739 | test_crash_count += s.test_results.crash_count; | |
| 740 | test_timeout_count += s.test_results.timeout_count; | |
| 741 | ||
| 742 | test_count += s.test_results.test_count; | |
| 743 | ||
| 744 | switch (s.state) { | |
| 745 | .precheck_unstarted => unreachable, | |
| 746 | .precheck_started => unreachable, | |
| 747 | .precheck_done => unreachable, | |
| 748 | .dependency_failure => pending_count += 1, | |
| 749 | .success => success_count += 1, | |
| 750 | .skipped, .skipped_oom => skipped_count += 1, | |
| 751 | .failure => { | |
| 752 | failure_count += 1; | |
| 753 | const compile_errors_len = s.result_error_bundle.errorMessageCount(); | |
| 754 | if (compile_errors_len > 0) { | |
| 755 | total_compile_errors += compile_errors_len; | |
| 756 | } | |
| 757 | }, | |
| 758 | } | |
| 759 | } | |
| 760 | ||
| 761 | if (fuzz) |mode| blk: { | |
| 762 | switch (builtin.os.tag) { | |
| 763 | // Current implementation depends on two things that need to be ported to Windows: | |
| 764 | // * Memory-mapping to share data between the fuzzer and build runner. | |
| 765 | // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving | |
| 766 | // many addresses to source locations). | |
| 767 | .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), | |
| 768 | else => {}, | |
| 769 | } | |
| 770 | if (@bitSizeOf(usize) != 64) { | |
| 771 | // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, | |
| 772 | // being compatible with file system's u64 return value. This is not the case | |
| 773 | // on 32-bit platforms. | |
| 774 | // Affects or affected by issues #5185, #22523, and #22464. | |
| 775 | fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); | |
| 776 | } | |
| 777 | ||
| 778 | switch (mode) { | |
| 779 | .forever => break :blk, | |
| 780 | .limit => {}, | |
| 781 | } | |
| 782 | ||
| 783 | assert(mode == .limit); | |
| 784 | var f = Fuzz.init( | |
| 785 | gpa, | |
| 786 | io, | |
| 787 | step_stack.keys(), | |
| 788 | parent_prog_node, | |
| 789 | mode, | |
| 790 | ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); | |
| 791 | defer f.deinit(); | |
| 792 | ||
| 793 | f.start(); | |
| 794 | try f.waitAndPrintReport(); | |
| 795 | } | |
| 796 | ||
| 797 | // Every test has a state | |
| 798 | assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); | |
| 799 | ||
| 800 | if (failure_count == 0) { | |
| 801 | std.Progress.setStatus(.success); | |
| 802 | } else { | |
| 803 | std.Progress.setStatus(.failure); | |
| 804 | } | |
| 805 | ||
| 806 | summary: { | |
| 807 | switch (run.summary) { | |
| 808 | .all, .new, .line => {}, | |
| 809 | .failures => if (failure_count == 0) break :summary, | |
| 810 | .none => break :summary, | |
| 811 | } | |
| 812 | ||
| 813 | const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); | |
| 814 | defer io.unlockStderr(); | |
| 815 | const t = stderr.terminal(); | |
| 816 | const w = &stderr.file_writer.interface; | |
| 817 | ||
| 818 | const total_count = success_count + failure_count + pending_count + skipped_count; | |
| 819 | t.setColor(.cyan) catch {}; | |
| 820 | t.setColor(.bold) catch {}; | |
| 821 | w.writeAll("Build Summary: ") catch {}; | |
| 822 | t.setColor(.reset) catch {}; | |
| 823 | w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; | |
| 824 | { | |
| 825 | t.setColor(.dim) catch {}; | |
| 826 | var first = true; | |
| 827 | if (skipped_count > 0) { | |
| 828 | w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; | |
| 829 | first = false; | |
| 830 | } | |
| 831 | if (failure_count > 0) { | |
| 832 | w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; | |
| 833 | first = false; | |
| 834 | } | |
| 835 | if (!first) w.writeByte(')') catch {}; | |
| 836 | t.setColor(.reset) catch {}; | |
| 837 | } | |
| 838 | ||
| 839 | if (test_count > 0) { | |
| 840 | w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; | |
| 841 | t.setColor(.dim) catch {}; | |
| 842 | var first = true; | |
| 843 | if (test_skip_count > 0) { | |
| 844 | w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; | |
| 845 | first = false; | |
| 846 | } | |
| 847 | if (test_fail_count > 0) { | |
| 848 | w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; | |
| 849 | first = false; | |
| 850 | } | |
| 851 | if (test_crash_count > 0) { | |
| 852 | w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; | |
| 853 | first = false; | |
| 854 | } | |
| 855 | if (test_timeout_count > 0) { | |
| 856 | w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; | |
| 857 | first = false; | |
| 858 | } | |
| 859 | if (!first) w.writeByte(')') catch {}; | |
| 860 | t.setColor(.reset) catch {}; | |
| 861 | } | |
| 862 | ||
| 863 | w.writeAll("\n") catch {}; | |
| 864 | ||
| 865 | if (run.summary == .line) break :summary; | |
| 866 | ||
| 867 | // Print a fancy tree with build results. | |
| 868 | var step_stack_copy = try step_stack.clone(gpa); | |
| 869 | defer step_stack_copy.deinit(gpa); | |
| 870 | ||
| 871 | var print_node: PrintNode = .{ .parent = null }; | |
| 872 | if (step_names.len == 0) { | |
| 873 | print_node.last = true; | |
| 874 | printTreeStep(graph, graph.default_step, run, t, &print_node, &step_stack_copy) catch {}; | |
| 875 | } else { | |
| 876 | const last_index = if (run.summary == .all) graph.top_level_steps.count() else blk: { | |
| 877 | var i: usize = step_names.len; | |
| 878 | while (i > 0) { | |
| 879 | i -= 1; | |
| 880 | const step = graph.top_level_steps.get(step_names[i]).?.step; | |
| 881 | const found = switch (run.summary) { | |
| 882 | .all, .line, .none => unreachable, | |
| 883 | .failures => step.state != .success, | |
| 884 | .new => !step.result_cached, | |
| 885 | }; | |
| 886 | if (found) break :blk i; | |
| 887 | } | |
| 888 | break :blk graph.top_level_steps.count(); | |
| 889 | }; | |
| 890 | for (step_names, 0..) |step_name, i| { | |
| 891 | const tls = graph.top_level_steps.get(step_name).?; | |
| 892 | print_node.last = i + 1 == last_index; | |
| 893 | printTreeStep(graph, &tls.step, run, t, &print_node, &step_stack_copy) catch {}; | |
| 894 | } | |
| 895 | } | |
| 896 | w.writeByte('\n') catch {}; | |
| 897 | } | |
| 898 | ||
| 899 | if (run.watch or run.web_server != null) return; | |
| 900 | ||
| 901 | // Perhaps in the future there could be an Advanced Options flag such as | |
| 902 | // --debug-build-runner-leaks which would make this code return instead of | |
| 903 | // calling exit. | |
| 904 | ||
| 905 | const code: u8 = code: { | |
| 906 | if (failure_count == 0) break :code 0; // success | |
| 907 | if (run.error_style.verboseContext()) break :code 1; // failure; print build command | |
| 908 | break :code 2; // failure; do not print build command | |
| 909 | }; | |
| 910 | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; | |
| 911 | process.exit(code); | |
| 912 | } | |
| 913 | ||
| 914 | const PrintNode = struct { | |
| 915 | parent: ?*PrintNode, | |
| 916 | last: bool = false, | |
| 917 | }; | |
| 918 | ||
| 919 | fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { | |
| 920 | const parent = node.parent orelse return; | |
| 921 | const writer = stderr.writer; | |
| 922 | if (parent.parent == null) return; | |
| 923 | try printPrefix(parent, stderr); | |
| 924 | if (parent.last) { | |
| 925 | try writer.writeAll(" "); | |
| 926 | } else { | |
| 927 | try writer.writeAll(switch (stderr.mode) { | |
| 928 | .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ | |
| 929 | else => "| ", | |
| 930 | }); | |
| 931 | } | |
| 932 | } | |
| 933 | ||
| 934 | fn printChildNodePrefix(stderr: Io.Terminal) !void { | |
| 935 | try stderr.writer.writeAll(switch (stderr.mode) { | |
| 936 | .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ | |
| 937 | else => "+- ", | |
| 938 | }); | |
| 939 | } | |
| 940 | ||
| 941 | fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void { | |
| 942 | const writer = stderr.writer; | |
| 943 | switch (s.state) { | |
| 944 | .precheck_unstarted => unreachable, | |
| 945 | .precheck_started => unreachable, | |
| 946 | .precheck_done => unreachable, | |
| 947 | ||
| 948 | .dependency_failure => { | |
| 949 | try stderr.setColor(.dim); | |
| 950 | try writer.writeAll(" transitive failure\n"); | |
| 951 | try stderr.setColor(.reset); | |
| 952 | }, | |
| 953 | ||
| 954 | .success => { | |
| 955 | try stderr.setColor(.green); | |
| 956 | if (s.result_cached) { | |
| 957 | try writer.writeAll(" cached"); | |
| 958 | } else if (s.test_results.test_count > 0) { | |
| 959 | const pass_count = s.test_results.passCount(); | |
| 960 | assert(s.test_results.test_count == pass_count + s.test_results.skip_count); | |
| 961 | try writer.print(" {d} pass", .{pass_count}); | |
| 962 | if (s.test_results.skip_count > 0) { | |
| 963 | try stderr.setColor(.reset); | |
| 964 | try writer.writeAll(", "); | |
| 965 | try stderr.setColor(.yellow); | |
| 966 | try writer.print("{d} skip", .{s.test_results.skip_count}); | |
| 967 | } | |
| 968 | try stderr.setColor(.reset); | |
| 969 | try writer.print(" ({d} total)", .{s.test_results.test_count}); | |
| 970 | } else { | |
| 971 | try writer.writeAll(" success"); | |
| 972 | } | |
| 973 | try stderr.setColor(.reset); | |
| 974 | if (s.result_duration_ns) |ns| { | |
| 975 | try stderr.setColor(.dim); | |
| 976 | if (ns >= std.time.ns_per_min) { | |
| 977 | try writer.print(" {d}m", .{ns / std.time.ns_per_min}); | |
| 978 | } else if (ns >= std.time.ns_per_s) { | |
| 979 | try writer.print(" {d}s", .{ns / std.time.ns_per_s}); | |
| 980 | } else if (ns >= std.time.ns_per_ms) { | |
| 981 | try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); | |
| 982 | } else if (ns >= std.time.ns_per_us) { | |
| 983 | try writer.print(" {d}us", .{ns / std.time.ns_per_us}); | |
| 984 | } else { | |
| 985 | try writer.print(" {d}ns", .{ns}); | |
| 986 | } | |
| 987 | try stderr.setColor(.reset); | |
| 988 | } | |
| 989 | if (s.result_peak_rss != 0) { | |
| 990 | const rss = s.result_peak_rss; | |
| 991 | try stderr.setColor(.dim); | |
| 992 | if (rss >= 1000_000_000) { | |
| 993 | try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); | |
| 994 | } else if (rss >= 1000_000) { | |
| 995 | try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); | |
| 996 | } else if (rss >= 1000) { | |
| 997 | try writer.print(" MaxRSS:{d}K", .{rss / 1000}); | |
| 998 | } else { | |
| 999 | try writer.print(" MaxRSS:{d}B", .{rss}); | |
| 1000 | } | |
| 1001 | try stderr.setColor(.reset); | |
| 1002 | } | |
| 1003 | try writer.writeAll("\n"); | |
| 1004 | }, | |
| 1005 | .skipped => { | |
| 1006 | try stderr.setColor(.yellow); | |
| 1007 | try writer.writeAll(" skipped\n"); | |
| 1008 | try stderr.setColor(.reset); | |
| 1009 | }, | |
| 1010 | .skipped_oom => { | |
| 1011 | try stderr.setColor(.yellow); | |
| 1012 | try writer.writeAll(" skipped (not enough memory)"); | |
| 1013 | try stderr.setColor(.dim); | |
| 1014 | try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ s.max_rss, run.available_rss }); | |
| 1015 | try stderr.setColor(.reset); | |
| 1016 | }, | |
| 1017 | .failure => { | |
| 1018 | try printStepFailure(s, stderr, false); | |
| 1019 | try stderr.setColor(.reset); | |
| 1020 | }, | |
| 1021 | } | |
| 1022 | } | |
| 1023 | ||
| 1024 | fn printStepFailure(s: *Step, stderr: Io.Terminal, dim: bool) !void { | |
| 1025 | const w = stderr.writer; | |
| 1026 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 1027 | try stderr.setColor(.red); | |
| 1028 | try w.print(" {d} errors\n", .{ | |
| 1029 | s.result_error_bundle.errorMessageCount(), | |
| 1030 | }); | |
| 1031 | } else if (!s.test_results.isSuccess()) { | |
| 1032 | // These first values include all of the test "statuses". Every test is either passsed, | |
| 1033 | // skipped, failed, crashed, or timed out. | |
| 1034 | try stderr.setColor(.green); | |
| 1035 | try w.print(" {d} pass", .{s.test_results.passCount()}); | |
| 1036 | try stderr.setColor(.reset); | |
| 1037 | if (dim) try stderr.setColor(.dim); | |
| 1038 | if (s.test_results.skip_count > 0) { | |
| 1039 | try w.writeAll(", "); | |
| 1040 | try stderr.setColor(.yellow); | |
| 1041 | try w.print("{d} skip", .{s.test_results.skip_count}); | |
| 1042 | try stderr.setColor(.reset); | |
| 1043 | if (dim) try stderr.setColor(.dim); | |
| 1044 | } | |
| 1045 | if (s.test_results.fail_count > 0) { | |
| 1046 | try w.writeAll(", "); | |
| 1047 | try stderr.setColor(.red); | |
| 1048 | try w.print("{d} fail", .{s.test_results.fail_count}); | |
| 1049 | try stderr.setColor(.reset); | |
| 1050 | if (dim) try stderr.setColor(.dim); | |
| 1051 | } | |
| 1052 | if (s.test_results.crash_count > 0) { | |
| 1053 | try w.writeAll(", "); | |
| 1054 | try stderr.setColor(.red); | |
| 1055 | try w.print("{d} crash", .{s.test_results.crash_count}); | |
| 1056 | try stderr.setColor(.reset); | |
| 1057 | if (dim) try stderr.setColor(.dim); | |
| 1058 | } | |
| 1059 | if (s.test_results.timeout_count > 0) { | |
| 1060 | try w.writeAll(", "); | |
| 1061 | try stderr.setColor(.red); | |
| 1062 | try w.print("{d} timeout", .{s.test_results.timeout_count}); | |
| 1063 | try stderr.setColor(.reset); | |
| 1064 | if (dim) try stderr.setColor(.dim); | |
| 1065 | } | |
| 1066 | try w.print(" ({d} total)", .{s.test_results.test_count}); | |
| 1067 | ||
| 1068 | // Memory leaks are intentionally written after the total, because is isn't a test *status*, | |
| 1069 | // but just a flag that any tests -- even passed ones -- can have. We also use a different | |
| 1070 | // separator, so it looks like: | |
| 1071 | // 2 pass, 1 skip, 2 fail (5 total); 2 leaks | |
| 1072 | if (s.test_results.leak_count > 0) { | |
| 1073 | try w.writeAll("; "); | |
| 1074 | try stderr.setColor(.red); | |
| 1075 | try w.print("{d} leaks", .{s.test_results.leak_count}); | |
| 1076 | try stderr.setColor(.reset); | |
| 1077 | if (dim) try stderr.setColor(.dim); | |
| 1078 | } | |
| 1079 | ||
| 1080 | // It's usually not helpful to know how many error logs there were because they tend to | |
| 1081 | // just come with other errors (e.g. crashes and leaks print stack traces, and clean | |
| 1082 | // failures print error traces). So only mention them if they're the only thing causing | |
| 1083 | // the failure. | |
| 1084 | const show_err_logs: bool = show: { | |
| 1085 | var alt_results = s.test_results; | |
| 1086 | alt_results.log_err_count = 0; | |
| 1087 | break :show alt_results.isSuccess(); | |
| 1088 | }; | |
| 1089 | if (show_err_logs) { | |
| 1090 | try w.writeAll("; "); | |
| 1091 | try stderr.setColor(.red); | |
| 1092 | try w.print("{d} error logs", .{s.test_results.log_err_count}); | |
| 1093 | try stderr.setColor(.reset); | |
| 1094 | if (dim) try stderr.setColor(.dim); | |
| 1095 | } | |
| 1096 | ||
| 1097 | try w.writeAll("\n"); | |
| 1098 | } else if (s.result_error_msgs.items.len > 0) { | |
| 1099 | try stderr.setColor(.red); | |
| 1100 | try w.writeAll(" failure\n"); | |
| 1101 | } else { | |
| 1102 | assert(s.result_stderr.len > 0); | |
| 1103 | try stderr.setColor(.red); | |
| 1104 | try w.writeAll(" w\n"); | |
| 1105 | } | |
| 1106 | } | |
| 1107 | ||
| 1108 | fn printTreeStep( | |
| 1109 | graph: *Graph, | |
| 1110 | s: *Step, | |
| 1111 | run: *const Run, | |
| 1112 | stderr: Io.Terminal, | |
| 1113 | parent_node: *PrintNode, | |
| 1114 | step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 1115 | ) !void { | |
| 1116 | const writer = stderr.writer; | |
| 1117 | const first = step_stack.swapRemove(s); | |
| 1118 | const summary = run.summary; | |
| 1119 | const skip = switch (summary) { | |
| 1120 | .none, .line => unreachable, | |
| 1121 | .all => false, | |
| 1122 | .new => s.result_cached, | |
| 1123 | .failures => s.state == .success, | |
| 1124 | }; | |
| 1125 | if (skip) return; | |
| 1126 | try printPrefix(parent_node, stderr); | |
| 1127 | ||
| 1128 | if (parent_node.parent != null) { | |
| 1129 | if (parent_node.last) { | |
| 1130 | try printChildNodePrefix(stderr); | |
| 1131 | } else { | |
| 1132 | try writer.writeAll(switch (stderr.mode) { | |
| 1133 | .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ | |
| 1134 | else => "+- ", | |
| 1135 | }); | |
| 1136 | } | |
| 1137 | } | |
| 1138 | ||
| 1139 | if (!first) try stderr.setColor(.dim); | |
| 1140 | ||
| 1141 | // dep_prefix omitted here because it is redundant with the tree. | |
| 1142 | try writer.writeAll(s.name); | |
| 1143 | ||
| 1144 | if (first) { | |
| 1145 | try printStepStatus(s, stderr, run); | |
| 1146 | ||
| 1147 | const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: { | |
| 1148 | var i: usize = s.dependencies.items.len; | |
| 1149 | while (i > 0) { | |
| 1150 | i -= 1; | |
| 1151 | ||
| 1152 | const step = s.dependencies.items[i]; | |
| 1153 | const found = switch (summary) { | |
| 1154 | .all, .line, .none => unreachable, | |
| 1155 | .failures => step.state != .success, | |
| 1156 | .new => !step.result_cached, | |
| 1157 | }; | |
| 1158 | if (found) break :blk i; | |
| 1159 | } | |
| 1160 | break :blk s.dependencies.items.len -| 1; | |
| 1161 | }; | |
| 1162 | for (s.dependencies.items, 0..) |dep, i| { | |
| 1163 | var print_node: PrintNode = .{ | |
| 1164 | .parent = parent_node, | |
| 1165 | .last = i == last_index, | |
| 1166 | }; | |
| 1167 | try printTreeStep(graph, dep, run, stderr, &print_node, step_stack); | |
| 1168 | } | |
| 1169 | } else { | |
| 1170 | if (s.dependencies.items.len == 0) { | |
| 1171 | try writer.writeAll(" (reused)\n"); | |
| 1172 | } else { | |
| 1173 | try writer.print(" (+{d} more reused dependencies)\n", .{ | |
| 1174 | s.dependencies.items.len, | |
| 1175 | }); | |
| 1176 | } | |
| 1177 | try stderr.setColor(.reset); | |
| 1178 | } | |
| 1179 | } | |
| 1180 | ||
| 1181 | /// Traverse the dependency graph depth-first and make it undirected by having | |
| 1182 | /// steps know their dependants (they only know dependencies at start). | |
| 1183 | /// Along the way, check that there is no dependency loop, and record the steps | |
| 1184 | /// in traversal order in `step_stack`. | |
| 1185 | /// Each step has its dependencies traversed in random order, this accomplishes | |
| 1186 | /// two things: | |
| 1187 | /// - `step_stack` will be in randomized-depth-first order, so the build runner | |
| 1188 | /// spawns initial steps in a random order | |
| 1189 | /// - each step's `dependants` list is also filled in a random order, so that | |
| 1190 | /// when it finishes executing in `makeStep`, it spawns next steps to run in | |
| 1191 | /// random order | |
| 1192 | fn constructGraphAndCheckForDependencyLoop( | |
| 1193 | gpa: Allocator, | |
| 1194 | s: *Step, | |
| 1195 | step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 1196 | rand: std.Random, | |
| 1197 | ) !void { | |
| 1198 | switch (s.state) { | |
| 1199 | .precheck_started => { | |
| 1200 | std.debug.print("dependency loop detected:\n {s}\n", .{s.name}); | |
| 1201 | return error.DependencyLoopDetected; | |
| 1202 | }, | |
| 1203 | .precheck_unstarted => { | |
| 1204 | s.state = .precheck_started; | |
| 1205 | ||
| 1206 | try step_stack.ensureUnusedCapacity(gpa, s.dependencies.items.len); | |
| 1207 | ||
| 1208 | // We dupe to avoid shuffling the steps in the summary, it depends | |
| 1209 | // on s.dependencies' order. | |
| 1210 | const deps = gpa.dupe(*Step, s.dependencies.items) catch @panic("OOM"); | |
| 1211 | defer gpa.free(deps); | |
| 1212 | ||
| 1213 | rand.shuffle(*Step, deps); | |
| 1214 | ||
| 1215 | for (deps) |dep| { | |
| 1216 | try step_stack.put(gpa, dep, {}); | |
| 1217 | try dep.dependants.append(gpa, s); | |
| 1218 | constructGraphAndCheckForDependencyLoop(gpa, dep, step_stack, rand) catch |err| { | |
| 1219 | if (err == error.DependencyLoopDetected) { | |
| 1220 | std.debug.print(" {s}\n", .{s.name}); | |
| 1221 | } | |
| 1222 | return err; | |
| 1223 | }; | |
| 1224 | } | |
| 1225 | ||
| 1226 | s.state = .precheck_done; | |
| 1227 | s.pending_deps = @intCast(s.dependencies.items.len); | |
| 1228 | }, | |
| 1229 | .precheck_done => {}, | |
| 1230 | ||
| 1231 | // These don't happen until we actually run the step graph. | |
| 1232 | .dependency_failure => unreachable, | |
| 1233 | .success => unreachable, | |
| 1234 | .failure => unreachable, | |
| 1235 | .skipped => unreachable, | |
| 1236 | .skipped_oom => unreachable, | |
| 1237 | } | |
| 1238 | } | |
| 1239 | ||
| 1240 | /// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready | |
| 1241 | /// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must | |
| 1242 | /// have already subtracted this value from `run.available_rss`. This function will release the RSS | |
| 1243 | /// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked | |
| 1244 | /// steps after "make" completes for `s`. | |
| 1245 | fn makeStep( | |
| 1246 | graph: *Graph, | |
| 1247 | group: *Io.Group, | |
| 1248 | s: *Step, | |
| 1249 | root_prog_node: std.Progress.Node, | |
| 1250 | run: *Run, | |
| 1251 | ) Io.Cancelable!void { | |
| 1252 | const io = graph.io; | |
| 1253 | const gpa = run.gpa; | |
| 1254 | ||
| 1255 | { | |
| 1256 | const step_prog_node = root_prog_node.start(s.name, 0); | |
| 1257 | defer step_prog_node.end(); | |
| 1258 | ||
| 1259 | if (run.web_server) |*ws| ws.updateStepStatus(s, .wip); | |
| 1260 | ||
| 1261 | const new_state: Step.State = for (s.dependencies.items) |dep| { | |
| 1262 | switch (@atomicLoad(Step.State, &dep.state, .monotonic)) { | |
| 1263 | .precheck_unstarted => unreachable, | |
| 1264 | .precheck_started => unreachable, | |
| 1265 | .precheck_done => unreachable, | |
| 1266 | ||
| 1267 | .failure, | |
| 1268 | .dependency_failure, | |
| 1269 | .skipped_oom, | |
| 1270 | => break .dependency_failure, | |
| 1271 | ||
| 1272 | .success, .skipped => {}, | |
| 1273 | } | |
| 1274 | } else if (s.make(.{ | |
| 1275 | .progress_node = step_prog_node, | |
| 1276 | .watch = run.watch, | |
| 1277 | .web_server = if (run.web_server) |*ws| ws else null, | |
| 1278 | .unit_test_timeout_ns = run.unit_test_timeout_ns, | |
| 1279 | .gpa = gpa, | |
| 1280 | })) state: { | |
| 1281 | break :state .success; | |
| 1282 | } else |err| switch (err) { | |
| 1283 | error.MakeFailed => .failure, | |
| 1284 | error.MakeSkipped => .skipped, | |
| 1285 | }; | |
| 1286 | ||
| 1287 | @atomicStore(Step.State, &s.state, new_state, .monotonic); | |
| 1288 | ||
| 1289 | switch (new_state) { | |
| 1290 | .precheck_unstarted => unreachable, | |
| 1291 | .precheck_started => unreachable, | |
| 1292 | .precheck_done => unreachable, | |
| 1293 | ||
| 1294 | .failure, | |
| 1295 | .dependency_failure, | |
| 1296 | .skipped_oom, | |
| 1297 | => { | |
| 1298 | if (run.web_server) |*ws| ws.updateStepStatus(s, .failure); | |
| 1299 | std.Progress.setStatus(.failure_working); | |
| 1300 | }, | |
| 1301 | ||
| 1302 | .success, | |
| 1303 | .skipped, | |
| 1304 | => { | |
| 1305 | if (run.web_server) |*ws| ws.updateStepStatus(s, .success); | |
| 1306 | }, | |
| 1307 | } | |
| 1308 | } | |
| 1309 | ||
| 1310 | // No matter the result, we want to display error/warning messages. | |
| 1311 | if (s.result_error_bundle.errorMessageCount() > 0 or | |
| 1312 | s.result_error_msgs.items.len > 0 or | |
| 1313 | s.result_stderr.len > 0) | |
| 1314 | { | |
| 1315 | const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); | |
| 1316 | defer io.unlockStderr(); | |
| 1317 | printErrorMessages(gpa, s, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch {}; | |
| 1318 | } | |
| 1319 | ||
| 1320 | if (s.max_rss != 0) { | |
| 1321 | var dispatch_set: std.ArrayList(*Step) = .empty; | |
| 1322 | defer dispatch_set.deinit(gpa); | |
| 1323 | ||
| 1324 | // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` | |
| 1325 | // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held. | |
| 1326 | { | |
| 1327 | try run.max_rss_mutex.lock(io); | |
| 1328 | defer run.max_rss_mutex.unlock(io); | |
| 1329 | run.available_rss += s.max_rss; | |
| 1330 | dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch @panic("OOM"); | |
| 1331 | while (run.memory_blocked_steps.getLast()) |candidate| { | |
| 1332 | if (run.available_rss < candidate.max_rss) break; | |
| 1333 | assert(run.memory_blocked_steps.pop() == candidate); | |
| 1334 | dispatch_set.appendAssumeCapacity(candidate); | |
| 1335 | } | |
| 1336 | } | |
| 1337 | for (dispatch_set.items) |candidate| { | |
| 1338 | group.async(io, makeStep, .{ graph, group, candidate, root_prog_node, run }); | |
| 1339 | } | |
| 1340 | } | |
| 1341 | ||
| 1342 | for (s.dependants.items) |dependant| { | |
| 1343 | // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. | |
| 1344 | if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { | |
| 1345 | try stepReady(graph, group, dependant, root_prog_node, run); | |
| 1346 | } | |
| 1347 | } | |
| 1348 | } | |
| 1349 | ||
| 1350 | fn stepReady( | |
| 1351 | graph: *Graph, | |
| 1352 | group: *Io.Group, | |
| 1353 | s: *Step, | |
| 1354 | root_prog_node: std.Progress.Node, | |
| 1355 | run: *Run, | |
| 1356 | ) !void { | |
| 1357 | const io = graph.io; | |
| 1358 | if (s.max_rss != 0) { | |
| 1359 | try run.max_rss_mutex.lock(io); | |
| 1360 | defer run.max_rss_mutex.unlock(io); | |
| 1361 | if (run.available_rss < s.max_rss) { | |
| 1362 | // Running this step right now could possibly exceed the allotted RSS. | |
| 1363 | run.memory_blocked_steps.append(run.gpa, s) catch @panic("OOM"); | |
| 1364 | return; | |
| 1365 | } | |
| 1366 | run.available_rss -= s.max_rss; | |
| 1367 | } | |
| 1368 | group.async(io, makeStep, .{ graph, group, s, root_prog_node, run }); | |
| 1369 | } | |
| 1370 | ||
| 1371 | pub fn printErrorMessages( | |
| 1372 | gpa: Allocator, | |
| 1373 | failing_step: *Step, | |
| 1374 | options: std.zig.ErrorBundle.RenderOptions, | |
| 1375 | stderr: Io.Terminal, | |
| 1376 | error_style: ErrorStyle, | |
| 1377 | multiline_errors: MultilineErrors, | |
| 1378 | ) !void { | |
| 1379 | const writer = stderr.writer; | |
| 1380 | if (error_style.verboseContext()) { | |
| 1381 | // Provide context for where these error messages are coming from by | |
| 1382 | // printing the corresponding Step subtree. | |
| 1383 | var step_stack: std.ArrayList(*Step) = .empty; | |
| 1384 | defer step_stack.deinit(gpa); | |
| 1385 | try step_stack.append(gpa, failing_step); | |
| 1386 | while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) { | |
| 1387 | try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]); | |
| 1388 | } | |
| 1389 | ||
| 1390 | // Now, `step_stack` has the subtree that we want to print, in reverse order. | |
| 1391 | try stderr.setColor(.dim); | |
| 1392 | var indent: usize = 0; | |
| 1393 | while (step_stack.pop()) |s| : (indent += 1) { | |
| 1394 | if (indent > 0) { | |
| 1395 | try writer.splatByteAll(' ', (indent - 1) * 3); | |
| 1396 | try printChildNodePrefix(stderr); | |
| 1397 | } | |
| 1398 | ||
| 1399 | try writer.writeAll(s.name); | |
| 1400 | ||
| 1401 | if (s == failing_step) { | |
| 1402 | try printStepFailure(s, stderr, true); | |
| 1403 | } else { | |
| 1404 | try writer.writeAll("\n"); | |
| 1405 | } | |
| 1406 | } | |
| 1407 | try stderr.setColor(.reset); | |
| 1408 | } else { | |
| 1409 | // Just print the failing step itself. | |
| 1410 | try stderr.setColor(.dim); | |
| 1411 | try writer.writeAll(failing_step.name); | |
| 1412 | try printStepFailure(failing_step, stderr, true); | |
| 1413 | try stderr.setColor(.reset); | |
| 1414 | } | |
| 1415 | ||
| 1416 | if (failing_step.result_stderr.len > 0) { | |
| 1417 | try writer.writeAll(failing_step.result_stderr); | |
| 1418 | if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { | |
| 1419 | try writer.writeAll("\n"); | |
| 1420 | } | |
| 1421 | } | |
| 1422 | ||
| 1423 | try failing_step.result_error_bundle.renderToTerminal(options, stderr); | |
| 1424 | ||
| 1425 | for (failing_step.result_error_msgs.items) |msg| { | |
| 1426 | try stderr.setColor(.red); | |
| 1427 | try writer.writeAll("error:"); | |
| 1428 | try stderr.setColor(.reset); | |
| 1429 | if (std.mem.indexOfScalar(u8, msg, '\n') == null) { | |
| 1430 | try writer.print(" {s}\n", .{msg}); | |
| 1431 | } else switch (multiline_errors) { | |
| 1432 | .indent => { | |
| 1433 | var it = std.mem.splitScalar(u8, msg, '\n'); | |
| 1434 | try writer.print(" {s}\n", .{it.first()}); | |
| 1435 | while (it.next()) |line| { | |
| 1436 | try writer.print(" {s}\n", .{line}); | |
| 1437 | } | |
| 1438 | }, | |
| 1439 | .newline => try writer.print("\n{s}\n", .{msg}), | |
| 1440 | .none => try writer.print(" {s}\n", .{msg}), | |
| 1441 | } | |
| 1442 | } | |
| 1443 | ||
| 1444 | if (error_style.verboseContext()) { | |
| 1445 | if (failing_step.result_failed_command) |cmd_str| { | |
| 1446 | try stderr.setColor(.red); | |
| 1447 | try writer.writeAll("failed command: "); | |
| 1448 | try stderr.setColor(.reset); | |
| 1449 | try writer.writeAll(cmd_str); | |
| 1450 | try writer.writeByte('\n'); | |
| 1451 | } | |
| 1452 | } | |
| 1453 | ||
| 1454 | try writer.writeByte('\n'); | |
| 1455 | } | |
| 1456 | ||
| 1457 | fn printSteps(graph: *Graph, w: *Writer) !void { | |
| 1458 | const arena = graph.arena; | |
| 1459 | for (graph.top_level_steps.values()) |top_level_step| { | |
| 1460 | const name = if (&top_level_step.step == graph.default_step) | |
| 1461 | try fmt.allocPrint(arena, "{s} (default)", .{top_level_step.step.name}) | |
| 1462 | else | |
| 1463 | top_level_step.step.name; | |
| 1464 | try w.print(" {s:<28} {s}\n", .{ name, top_level_step.description }); | |
| 1465 | } | |
| 1466 | } | |
| 1467 | ||
| 1468 | fn printUsage(graph: *Graph, w: *Writer) !void { | |
| 1469 | const arena = graph.arena; | |
| 1470 | ||
| 1471 | try w.print( | |
| 1472 | \\Usage: {s} build [steps] [options] | |
| 1473 | \\ | |
| 1474 | \\Steps: | |
| 1475 | \\ | |
| 1476 | , .{graph.zig_exe}); | |
| 1477 | try printSteps(graph, w); | |
| 1478 | try w.writeAll( | |
| 1479 | \\ | |
| 1480 | \\Project-Specific Options: | |
| 1481 | \\ | |
| 1482 | ); | |
| 1483 | ||
| 1484 | if (graph.available_options_list.items.len == 0) { | |
| 1485 | try w.print(" (none)\n", .{}); | |
| 1486 | } else { | |
| 1487 | for (graph.available_options_list.items) |option| { | |
| 1488 | const name = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ option.name, option.type_id }); | |
| 1489 | try w.print("{s:<30} {s}\n", .{ name, option.description }); | |
| 1490 | if (option.enum_options) |enum_options| { | |
| 1491 | const padding: [33]u8 = @splat(' '); | |
| 1492 | try w.writeAll(padding ++ "Supported Values:\n"); | |
| 1493 | for (enum_options) |enum_option| { | |
| 1494 | try w.print(padding ++ " {s}\n", .{enum_option}); | |
| 1495 | } | |
| 1496 | } | |
| 1497 | } | |
| 1498 | } | |
| 1499 | ||
| 1500 | try w.writeAll( | |
| 1501 | \\ | |
| 1502 | \\System Integration Options: | |
| 1503 | \\ --search-prefix [path] Add a path to look for binaries, libraries, headers | |
| 1504 | \\ --sysroot [path] Set the system root directory (usually /) | |
| 1505 | \\ --libc [file] Provide a file which specifies libc paths | |
| 1506 | \\ | |
| 1507 | \\ --system [pkgdir] Disable package fetching; enable all integrations | |
| 1508 | \\ -fsys=[name] Enable a system integration | |
| 1509 | \\ -fno-sys=[name] Disable a system integration | |
| 1510 | \\ | |
| 1511 | \\ -fdarling, -fno-darling Integration with system-installed Darling to | |
| 1512 | \\ execute macOS programs on Linux hosts | |
| 1513 | \\ (default: no) | |
| 1514 | \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute | |
| 1515 | \\ foreign-architecture programs on Linux hosts | |
| 1516 | \\ (default: no) | |
| 1517 | \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc | |
| 1518 | \\ (e.g. glibc or musl) built for multiple foreign | |
| 1519 | \\ architectures, allowing execution of non-native | |
| 1520 | \\ programs that link with libc. | |
| 1521 | \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on | |
| 1522 | \\ ARM64 macOS hosts. (default: no) | |
| 1523 | \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to | |
| 1524 | \\ execute WASI binaries. (default: no) | |
| 1525 | \\ -fwine, -fno-wine Integration with system-installed Wine to execute | |
| 1526 | \\ Windows programs on Linux hosts. (default: no) | |
| 1527 | \\ | |
| 1528 | \\ Available System Integrations: Enabled: | |
| 1529 | \\ | |
| 1530 | ); | |
| 1531 | if (graph.system_library_options.entries.len == 0) { | |
| 1532 | try w.writeAll(" (none) -\n"); | |
| 1533 | } else { | |
| 1534 | for (graph.system_library_options.keys(), graph.system_library_options.values()) |k, v| { | |
| 1535 | const status = switch (v) { | |
| 1536 | .declared_enabled => "yes", | |
| 1537 | .declared_disabled => "no", | |
| 1538 | .user_enabled, .user_disabled => unreachable, // already emitted error | |
| 1539 | }; | |
| 1540 | try w.print(" {s:<43} {s}\n", .{ k, status }); | |
| 1541 | } | |
| 1542 | } | |
| 1543 | ||
| 1544 | try w.writeAll( | |
| 1545 | \\ | |
| 1546 | \\General Options: | |
| 1547 | \\ -h, --help Print this help and exit | |
| 1548 | \\ -l, --list-steps Print available steps | |
| 1549 | \\ | |
| 1550 | \\ -p, --prefix [path] Where to install files (default: zig-out) | |
| 1551 | \\ --prefix-lib-dir [path] Where to install libraries | |
| 1552 | \\ --prefix-exe-dir [path] Where to install executables | |
| 1553 | \\ --prefix-include-dir [path] Where to install C header files | |
| 1554 | \\ --release[=mode] Request release mode, optionally specifying a | |
| 1555 | \\ preferred optimization mode: fast, safe, small | |
| 1556 | \\ | |
| 1557 | \\ --verbose Print commands before executing them | |
| 1558 | \\ --color [auto|off|on] Enable or disable colored error messages | |
| 1559 | \\ --error-style [style] Control how build errors are printed | |
| 1560 | \\ verbose (Default) Report errors with full context | |
| 1561 | \\ minimal Report errors after summary, excluding context like command lines | |
| 1562 | \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update | |
| 1563 | \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update | |
| 1564 | \\ --multiline-errors [style] Control how multi-line error messages are printed | |
| 1565 | \\ indent (Default) Indent non-initial lines to align with initial line | |
| 1566 | \\ newline Include a leading newline so that the error message is on its own lines | |
| 1567 | \\ none Print as usual so the first line is misaligned | |
| 1568 | \\ --summary [mode] Control the printing of the build summary | |
| 1569 | \\ all Print the build summary in its entirety | |
| 1570 | \\ new Omit cached steps | |
| 1571 | \\ failures (Default if short-lived) Only print failed steps | |
| 1572 | \\ line (Default if long-lived) Only print the single-line summary | |
| 1573 | \\ none Do not print the build summary | |
| 1574 | \\ -j<N> Limit concurrent jobs (default is to use all CPU cores) | |
| 1575 | \\ --maxrss <bytes> Limit memory usage (default is to use available memory) | |
| 1576 | \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss | |
| 1577 | \\ --test-timeout <timeout> Limit execution time of unit tests, terminating if exceeded. | |
| 1578 | \\ The timeout must include a unit: ns, us, ms, s, m, h | |
| 1579 | \\ --watch Continuously rebuild when source files are modified | |
| 1580 | \\ --debounce <ms> Delay before rebuilding after changed file detected | |
| 1581 | \\ --webui[=ip] Enable the web interface on the given IP address | |
| 1582 | \\ --fuzz[=limit] Continuously search for unit test failures with an optional | |
| 1583 | \\ limit to the max number of iterations. The argument supports | |
| 1584 | \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies | |
| 1585 | \\ '--webui' when no limit is specified. | |
| 1586 | \\ --time-report Force full rebuild and provide detailed information on | |
| 1587 | \\ compilation time of Zig source code (implies '--webui') | |
| 1588 | \\ -fincremental Enable incremental compilation | |
| 1589 | \\ -fno-incremental Disable incremental compilation | |
| 1590 | \\ | |
| 1591 | \\Package Management Options: | |
| 1592 | \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit | |
| 1593 | \\ needed (Default) Lazy dependencies are fetched as needed | |
| 1594 | \\ all Lazy dependencies are always fetched | |
| 1595 | \\ --fork=[path] Override one or more projects from dependency tree | |
| 1596 | \\ | |
| 1597 | \\Advanced Options: | |
| 1598 | \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error | |
| 1599 | \\ -fno-reference-trace Disable reference trace | |
| 1600 | \\ -fallow-so-scripts Allows .so files to be GNU ld scripts | |
| 1601 | \\ -fno-allow-so-scripts (default) .so files must be ELF files | |
| 1602 | \\ --build-file [file] Override path to build.zig | |
| 1603 | \\ --cache-dir [path] Override path to local Zig cache directory | |
| 1604 | \\ --global-cache-dir [path] Override path to global Zig cache directory | |
| 1605 | \\ --zig-lib-dir [arg] Override path to Zig lib directory | |
| 1606 | \\ --build-runner [file] Override path to build runner | |
| 1607 | \\ --seed [integer] For shuffling dependency traversal order (default: random) | |
| 1608 | \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries | |
| 1609 | \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) | |
| 1610 | \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) | |
| 1611 | \\ md5 16-byte cryptographic hash (ELF) | |
| 1612 | \\ uuid 16-byte random UUID (ELF, WASM) | |
| 1613 | \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) | |
| 1614 | \\ none (default) No build ID | |
| 1615 | \\ --debug-log [scope] Enable debugging the compiler | |
| 1616 | \\ --debug-pkg-config Fail if unknown pkg-config flags encountered | |
| 1617 | \\ --debug-rt Debug compiler runtime libraries | |
| 1618 | \\ --verbose-link Enable compiler debug output for linking | |
| 1619 | \\ --verbose-air Enable compiler debug output for Zig AIR | |
| 1620 | \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR | |
| 1621 | \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC | |
| 1622 | \\ --verbose-cimport Enable compiler debug output for C imports | |
| 1623 | \\ --verbose-cc Enable compiler debug output for C compilation | |
| 1624 | \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features | |
| 1625 | \\ | |
| 1626 | ); | |
| 1627 | } | |
| 1628 | ||
| 1629 | fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { | |
| 1630 | if (idx.* >= args.len) return null; | |
| 1631 | defer idx.* += 1; | |
| 1632 | return args[idx.*]; | |
| 1633 | } | |
| 1634 | ||
| 1635 | fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { | |
| 1636 | return nextArg(args, idx) orelse | |
| 1637 | fatal("expected argument after {q}\n access the help menu with \"zig build -h\"", .{args[idx.* - 1]}); | |
| 1638 | } | |
| 1639 | ||
| 1640 | fn cutArgPrefixOrFatal(args: []const [:0]const u8, idx: *usize, prefix: []const u8) []const u8 { | |
| 1641 | if (nextArg(args, idx)) |next_arg| { | |
| 1642 | if (mem.cutPrefix(u8, next_arg, prefix)) |arg| { | |
| 1643 | return arg; | |
| 1644 | } | |
| 1645 | } | |
| 1646 | fatal("expected argument after {q} to start with {q}", .{ args[idx.* - 1], prefix }); | |
| 1647 | } | |
| 1648 | ||
| 1649 | fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { | |
| 1650 | if (idx >= args.len) return null; | |
| 1651 | return args[idx..]; | |
| 1652 | } | |
| 1653 | ||
| 1654 | const Color = std.zig.Color; | |
| 1655 | const ErrorStyle = enum { | |
| 1656 | verbose, | |
| 1657 | minimal, | |
| 1658 | verbose_clear, | |
| 1659 | minimal_clear, | |
| 1660 | fn verboseContext(s: ErrorStyle) bool { | |
| 1661 | return switch (s) { | |
| 1662 | .verbose, .verbose_clear => true, | |
| 1663 | .minimal, .minimal_clear => false, | |
| 1664 | }; | |
| 1665 | } | |
| 1666 | fn clearOnUpdate(s: ErrorStyle) bool { | |
| 1667 | return switch (s) { | |
| 1668 | .verbose, .minimal => false, | |
| 1669 | .verbose_clear, .minimal_clear => true, | |
| 1670 | }; | |
| 1671 | } | |
| 1672 | }; | |
| 1673 | const MultilineErrors = enum { indent, newline, none }; | |
| 1674 | const Summary = enum { all, new, failures, line, none }; | |
| 1675 | ||
| 1676 | fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { | |
| 1677 | std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); | |
| 1678 | process.exit(1); | |
| 1679 | } | |
| 1680 | ||
| 1681 | fn validateSystemLibraryOptions(graph: *Graph) void { | |
| 1682 | var bad = false; | |
| 1683 | for (graph.system_library_options.keys(), graph.system_library_options.values()) |k, v| { | |
| 1684 | switch (v) { | |
| 1685 | .user_disabled, .user_enabled => { | |
| 1686 | // The user tried to enable or disable a system library integration, but | |
| 1687 | // the build script did not recognize that option. | |
| 1688 | std.debug.print("system library name not recognized by build script: '{s}'\n", .{k}); | |
| 1689 | bad = true; | |
| 1690 | }, | |
| 1691 | .declared_disabled, .declared_enabled => {}, | |
| 1692 | } | |
| 1693 | } | |
| 1694 | if (bad) { | |
| 1695 | std.debug.print(" access the help menu with 'zig build -h'\n", .{}); | |
| 1696 | process.exit(1); | |
| 1697 | } | |
| 1698 | } | |
| 1699 | ||
| 1700 | var stdio_buffer_allocation: [256]u8 = undefined; | |
| 1701 | var stdout_writer_allocation: Io.File.Writer = undefined; | |
| 1702 | ||
| 1703 | fn initStdoutWriter(io: Io) *Writer { | |
| 1704 | stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); | |
| 1705 | return &stdout_writer_allocation.interface; | |
| 1706 | } | |
| 1707 | ||
| 1708 | fn cleanTmpFiles(io: Io, steps: []const *Step) void { | |
| 1709 | for (steps) |step| { | |
| 1710 | const wf = step.cast(Step.WriteFile) orelse continue; | |
| 1711 | if (wf.mode != .tmp) continue; | |
| 1712 | const path = wf.generated_directory.path orelse continue; | |
| 1713 | Io.Dir.cwd().deleteTree(io, path) catch |err| { | |
| 1714 | std.log.warn("failed to delete {s}: {t}", .{ path, err }); | |
| 1715 | }; | |
| 1716 | } | |
| 1717 | } |
lib/compiler/maker/Fuzz.zig created+597
| ... | ... | @@ -0,0 +1,597 @@ |
| 1 | const std = @import("Std"); | |
| 2 | const Io = std.Io; | |
| 3 | const Build = std.Build; | |
| 4 | const Cache = Build.Cache; | |
| 5 | const Step = std.Build.Step; | |
| 6 | const assert = std.debug.assert; | |
| 7 | const fatal = std.process.fatal; | |
| 8 | const Allocator = std.mem.Allocator; | |
| 9 | const log = std.log; | |
| 10 | const Coverage = std.debug.Coverage; | |
| 11 | const abi = Build.abi.fuzz; | |
| 12 | ||
| 13 | const Fuzz = @This(); | |
| 14 | const build_runner = @import("root"); | |
| 15 | ||
| 16 | gpa: Allocator, | |
| 17 | io: Io, | |
| 18 | mode: Mode, | |
| 19 | ||
| 20 | /// Allocated into `gpa`. | |
| 21 | run_steps: []const *Step.Run, | |
| 22 | ||
| 23 | group: Io.Group, | |
| 24 | root_prog_node: std.Progress.Node, | |
| 25 | prog_node: std.Progress.Node, | |
| 26 | ||
| 27 | /// Protects `coverage_files`. | |
| 28 | coverage_mutex: Io.Mutex, | |
| 29 | coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap), | |
| 30 | ||
| 31 | queue_mutex: Io.Mutex, | |
| 32 | queue_cond: Io.Condition, | |
| 33 | msg_queue: std.ArrayList(Msg), | |
| 34 | ||
| 35 | pub const Mode = union(enum) { | |
| 36 | forever: struct { ws: *Build.WebServer }, | |
| 37 | limit: Limited, | |
| 38 | ||
| 39 | pub const Limited = struct { | |
| 40 | amount: u64, | |
| 41 | }; | |
| 42 | }; | |
| 43 | ||
| 44 | const Msg = union(enum) { | |
| 45 | coverage: struct { | |
| 46 | id: u64, | |
| 47 | cumulative: struct { | |
| 48 | runs: u64, | |
| 49 | unique: u64, | |
| 50 | coverage: u64, | |
| 51 | }, | |
| 52 | run: *Step.Run, | |
| 53 | }, | |
| 54 | entry_point: struct { | |
| 55 | coverage_id: u64, | |
| 56 | addr: u64, | |
| 57 | }, | |
| 58 | }; | |
| 59 | ||
| 60 | const CoverageMap = struct { | |
| 61 | mapped_memory: []align(std.heap.page_size_min) const u8, | |
| 62 | coverage: Coverage, | |
| 63 | source_locations: []Coverage.SourceLocation, | |
| 64 | /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. | |
| 65 | entry_points: std.ArrayList(u32), | |
| 66 | start_timestamp: i64, | |
| 67 | start_n_runs: u64, | |
| 68 | ||
| 69 | fn deinit(cm: *CoverageMap, gpa: Allocator) void { | |
| 70 | std.posix.munmap(cm.mapped_memory); | |
| 71 | cm.coverage.deinit(gpa); | |
| 72 | cm.* = undefined; | |
| 73 | } | |
| 74 | }; | |
| 75 | ||
| 76 | pub fn init( | |
| 77 | gpa: Allocator, | |
| 78 | io: Io, | |
| 79 | all_steps: []const *Build.Step, | |
| 80 | root_prog_node: std.Progress.Node, | |
| 81 | mode: Mode, | |
| 82 | ) error{ OutOfMemory, Canceled }!Fuzz { | |
| 83 | const run_steps: []const *Step.Run = steps: { | |
| 84 | var steps: std.ArrayList(*Step.Run) = .empty; | |
| 85 | defer steps.deinit(gpa); | |
| 86 | const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0); | |
| 87 | defer rebuild_node.end(); | |
| 88 | var rebuild_group: Io.Group = .init; | |
| 89 | defer rebuild_group.cancel(io); | |
| 90 | ||
| 91 | for (all_steps) |step| { | |
| 92 | const run = step.cast(Step.Run) orelse continue; | |
| 93 | if (run.producer == null) continue; | |
| 94 | if (run.fuzz_tests.items.len == 0) continue; | |
| 95 | try steps.append(gpa, run); | |
| 96 | rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node }); | |
| 97 | } | |
| 98 | ||
| 99 | if (steps.items.len == 0) fatal("no fuzz tests found", .{}); | |
| 100 | rebuild_node.setEstimatedTotalItems(steps.items.len); | |
| 101 | const run_steps = try gpa.dupe(*Step.Run, steps.items); | |
| 102 | try rebuild_group.await(io); | |
| 103 | break :steps run_steps; | |
| 104 | }; | |
| 105 | errdefer gpa.free(run_steps); | |
| 106 | ||
| 107 | for (run_steps) |run| { | |
| 108 | assert(run.fuzz_tests.items.len > 0); | |
| 109 | if (run.rebuilt_executable == null) | |
| 110 | fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); | |
| 111 | } | |
| 112 | ||
| 113 | return .{ | |
| 114 | .gpa = gpa, | |
| 115 | .io = io, | |
| 116 | .mode = mode, | |
| 117 | .run_steps = run_steps, | |
| 118 | .group = .init, | |
| 119 | .root_prog_node = root_prog_node, | |
| 120 | .prog_node = .none, | |
| 121 | .coverage_files = .empty, | |
| 122 | .coverage_mutex = .init, | |
| 123 | .queue_mutex = .init, | |
| 124 | .queue_cond = .init, | |
| 125 | .msg_queue = .empty, | |
| 126 | }; | |
| 127 | } | |
| 128 | ||
| 129 | pub fn start(fuzz: *Fuzz) void { | |
| 130 | const io = fuzz.io; | |
| 131 | fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0); | |
| 132 | ||
| 133 | if (fuzz.mode == .forever) { | |
| 134 | // For polling messages and sending updates to subscribers. | |
| 135 | fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err| | |
| 136 | fatal("unable to spawn coverage task: {t}", .{err}); | |
| 137 | } | |
| 138 | ||
| 139 | for (fuzz.run_steps) |run| { | |
| 140 | assert(run.rebuilt_executable != null); | |
| 141 | fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run }); | |
| 142 | } | |
| 143 | } | |
| 144 | ||
| 145 | pub fn deinit(fuzz: *Fuzz) void { | |
| 146 | const io = fuzz.io; | |
| 147 | fuzz.group.cancel(io); | |
| 148 | fuzz.prog_node.end(); | |
| 149 | fuzz.gpa.free(fuzz.run_steps); | |
| 150 | } | |
| 151 | ||
| 152 | fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) void { | |
| 153 | rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { | |
| 154 | const compile = run.producer.?; | |
| 155 | log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); | |
| 156 | }; | |
| 157 | } | |
| 158 | ||
| 159 | fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { | |
| 160 | const graph = run.step.owner.graph; | |
| 161 | const io = graph.io; | |
| 162 | const compile = run.producer.?; | |
| 163 | const prog_node = parent_prog_node.start(compile.step.name, 0); | |
| 164 | defer prog_node.end(); | |
| 165 | ||
| 166 | const result = compile.rebuildInFuzzMode(gpa, prog_node); | |
| 167 | ||
| 168 | const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; | |
| 169 | const show_error_msgs = compile.step.result_error_msgs.items.len > 0; | |
| 170 | const show_stderr = compile.step.result_stderr.len > 0; | |
| 171 | ||
| 172 | if (show_error_msgs or show_compile_errors or show_stderr) { | |
| 173 | var buf: [256]u8 = undefined; | |
| 174 | const stderr = try io.lockStderr(&buf, graph.stderr_mode); | |
| 175 | defer io.unlockStderr(); | |
| 176 | build_runner.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; | |
| 177 | } | |
| 178 | ||
| 179 | const rebuilt_bin_path = result catch |err| switch (err) { | |
| 180 | error.MakeFailed => return, | |
| 181 | else => |other| return other, | |
| 182 | }; | |
| 183 | run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); | |
| 184 | } | |
| 185 | ||
| 186 | fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run) void { | |
| 187 | const owner = run.step.owner; | |
| 188 | const gpa = owner.allocator; | |
| 189 | const graph = owner.graph; | |
| 190 | const io = graph.io; | |
| 191 | ||
| 192 | run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) { | |
| 193 | error.MakeFailed => { | |
| 194 | var buf: [256]u8 = undefined; | |
| 195 | const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) { | |
| 196 | error.Canceled => return, | |
| 197 | }; | |
| 198 | defer io.unlockStderr(); | |
| 199 | build_runner.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; | |
| 200 | return; | |
| 201 | }, | |
| 202 | else => { | |
| 203 | log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err }); | |
| 204 | return; | |
| 205 | }, | |
| 206 | }; | |
| 207 | } | |
| 208 | ||
| 209 | pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { | |
| 210 | assert(fuzz.mode == .forever); | |
| 211 | ||
| 212 | var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); | |
| 213 | defer arena_state.deinit(); | |
| 214 | const arena = arena_state.allocator(); | |
| 215 | ||
| 216 | const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); | |
| 217 | var dedup_table: DedupTable = .empty; | |
| 218 | defer dedup_table.deinit(fuzz.gpa); | |
| 219 | ||
| 220 | for (fuzz.run_steps) |run_step| { | |
| 221 | const compile_inputs = run_step.producer.?.step.inputs.table; | |
| 222 | for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { | |
| 223 | try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len); | |
| 224 | for (file_list.items) |sub_path| { | |
| 225 | if (!std.mem.endsWith(u8, sub_path, ".zig")) continue; | |
| 226 | const joined_path = try dir_path.join(arena, sub_path); | |
| 227 | dedup_table.putAssumeCapacity(joined_path, {}); | |
| 228 | } | |
| 229 | } | |
| 230 | } | |
| 231 | ||
| 232 | const deduped_paths = dedup_table.keys(); | |
| 233 | const SortContext = struct { | |
| 234 | pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { | |
| 235 | _ = this; | |
| 236 | return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { | |
| 237 | .lt => true, | |
| 238 | .gt => false, | |
| 239 | .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), | |
| 240 | }; | |
| 241 | } | |
| 242 | }; | |
| 243 | std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); | |
| 244 | return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths); | |
| 245 | } | |
| 246 | ||
| 247 | pub const Previous = struct { | |
| 248 | unique_runs: usize, | |
| 249 | entry_points: usize, | |
| 250 | sent_source_index: bool, | |
| 251 | pub const init: Previous = .{ | |
| 252 | .unique_runs = 0, | |
| 253 | .entry_points = 0, | |
| 254 | .sent_source_index = false, | |
| 255 | }; | |
| 256 | }; | |
| 257 | pub fn sendUpdate( | |
| 258 | fuzz: *Fuzz, | |
| 259 | socket: *std.http.Server.WebSocket, | |
| 260 | prev: *Previous, | |
| 261 | ) !void { | |
| 262 | const io = fuzz.io; | |
| 263 | ||
| 264 | try fuzz.coverage_mutex.lock(io); | |
| 265 | defer fuzz.coverage_mutex.unlock(io); | |
| 266 | ||
| 267 | const coverage_maps = fuzz.coverage_files.values(); | |
| 268 | if (coverage_maps.len == 0) return; | |
| 269 | // TODO: handle multiple fuzz steps in the WebSocket packets | |
| 270 | const coverage_map = &coverage_maps[0]; | |
| 271 | const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); | |
| 272 | // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the | |
| 273 | // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the | |
| 274 | // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass | |
| 275 | // this data straight to the socket with sendfile... | |
| 276 | const seen_pcs = cov_header.seenBits(); | |
| 277 | const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); | |
| 278 | const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); | |
| 279 | { | |
| 280 | if (!prev.sent_source_index) { | |
| 281 | prev.sent_source_index = true; | |
| 282 | // We need to send initial context. | |
| 283 | const header: abi.SourceIndexHeader = .{ | |
| 284 | .directories_len = @intCast(coverage_map.coverage.directories.entries.len), | |
| 285 | .files_len = @intCast(coverage_map.coverage.files.entries.len), | |
| 286 | .source_locations_len = @intCast(coverage_map.source_locations.len), | |
| 287 | .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), | |
| 288 | .start_timestamp = coverage_map.start_timestamp, | |
| 289 | .start_n_runs = coverage_map.start_n_runs, | |
| 290 | }; | |
| 291 | var iovecs: [5][]const u8 = .{ | |
| 292 | @ptrCast(&header), | |
| 293 | @ptrCast(coverage_map.coverage.directories.keys()), | |
| 294 | @ptrCast(coverage_map.coverage.files.keys()), | |
| 295 | @ptrCast(coverage_map.source_locations), | |
| 296 | coverage_map.coverage.string_bytes.items, | |
| 297 | }; | |
| 298 | try socket.writeMessageVec(&iovecs, .binary); | |
| 299 | } | |
| 300 | ||
| 301 | const header: abi.CoverageUpdateHeader = .{ | |
| 302 | .n_runs = n_runs, | |
| 303 | .unique_runs = unique_runs, | |
| 304 | }; | |
| 305 | var iovecs: [2][]const u8 = .{ | |
| 306 | @ptrCast(&header), | |
| 307 | @ptrCast(seen_pcs), | |
| 308 | }; | |
| 309 | try socket.writeMessageVec(&iovecs, .binary); | |
| 310 | ||
| 311 | prev.unique_runs = unique_runs; | |
| 312 | } | |
| 313 | ||
| 314 | if (prev.entry_points != coverage_map.entry_points.items.len) { | |
| 315 | const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len)); | |
| 316 | var iovecs: [2][]const u8 = .{ | |
| 317 | @ptrCast(&header), | |
| 318 | @ptrCast(coverage_map.entry_points.items), | |
| 319 | }; | |
| 320 | try socket.writeMessageVec(&iovecs, .binary); | |
| 321 | ||
| 322 | prev.entry_points = coverage_map.entry_points.items.len; | |
| 323 | } | |
| 324 | } | |
| 325 | ||
| 326 | fn coverageRun(fuzz: *Fuzz) void { | |
| 327 | coverageRunCancelable(fuzz) catch |err| switch (err) { | |
| 328 | error.Canceled => return, | |
| 329 | }; | |
| 330 | } | |
| 331 | ||
| 332 | fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { | |
| 333 | const io = fuzz.io; | |
| 334 | ||
| 335 | try fuzz.queue_mutex.lock(io); | |
| 336 | defer fuzz.queue_mutex.unlock(io); | |
| 337 | ||
| 338 | while (true) { | |
| 339 | try fuzz.queue_cond.wait(io, &fuzz.queue_mutex); | |
| 340 | for (fuzz.msg_queue.items) |msg| switch (msg) { | |
| 341 | .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) { | |
| 342 | error.AlreadyReported => continue, | |
| 343 | error.Canceled => return, | |
| 344 | else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), | |
| 345 | }, | |
| 346 | .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) { | |
| 347 | error.AlreadyReported => continue, | |
| 348 | error.Canceled => return, | |
| 349 | else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), | |
| 350 | }, | |
| 351 | }; | |
| 352 | fuzz.msg_queue.clearRetainingCapacity(); | |
| 353 | } | |
| 354 | } | |
| 355 | fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { | |
| 356 | assert(fuzz.mode == .forever); | |
| 357 | const ws = fuzz.mode.forever.ws; | |
| 358 | const gpa = fuzz.gpa; | |
| 359 | const io = fuzz.io; | |
| 360 | ||
| 361 | try fuzz.coverage_mutex.lock(io); | |
| 362 | defer fuzz.coverage_mutex.unlock(io); | |
| 363 | ||
| 364 | const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id); | |
| 365 | if (gop.found_existing) { | |
| 366 | // We are fuzzing the same executable with multiple threads. | |
| 367 | // Perhaps the same unit test; perhaps a different one. In any | |
| 368 | // case, since the coverage file is the same, we only have to | |
| 369 | // notice changes to that one file in order to learn coverage for | |
| 370 | // this particular executable. | |
| 371 | return; | |
| 372 | } | |
| 373 | errdefer _ = fuzz.coverage_files.pop(); | |
| 374 | ||
| 375 | gop.value_ptr.* = .{ | |
| 376 | .coverage = std.debug.Coverage.init, | |
| 377 | .mapped_memory = undefined, // populated below | |
| 378 | .source_locations = undefined, // populated below | |
| 379 | .entry_points = .empty, | |
| 380 | .start_timestamp = ws.now(), | |
| 381 | .start_n_runs = undefined, // populated below | |
| 382 | }; | |
| 383 | errdefer gop.value_ptr.coverage.deinit(gpa); | |
| 384 | ||
| 385 | const rebuilt_exe_path = run_step.rebuilt_executable.?; | |
| 386 | const target = run_step.producer.?.rootModuleTarget(); | |
| 387 | var debug_info = std.debug.Info.load( | |
| 388 | gpa, | |
| 389 | io, | |
| 390 | rebuilt_exe_path, | |
| 391 | &gop.value_ptr.coverage, | |
| 392 | target.ofmt, | |
| 393 | target.cpu.arch, | |
| 394 | ) catch |err| { | |
| 395 | log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ | |
| 396 | run_step.step.name, rebuilt_exe_path, err, | |
| 397 | }); | |
| 398 | return error.AlreadyReported; | |
| 399 | }; | |
| 400 | defer debug_info.deinit(gpa); | |
| 401 | ||
| 402 | const coverage_file_path: Build.Cache.Path = .{ | |
| 403 | .root_dir = run_step.step.owner.cache_root, | |
| 404 | .sub_path = "v/" ++ std.fmt.hex(coverage_id), | |
| 405 | }; | |
| 406 | var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { | |
| 407 | log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ | |
| 408 | run_step.step.name, coverage_file_path, err, | |
| 409 | }); | |
| 410 | return error.AlreadyReported; | |
| 411 | }; | |
| 412 | defer coverage_file.close(io); | |
| 413 | ||
| 414 | const file_size = coverage_file.length(io) catch |err| { | |
| 415 | log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err }); | |
| 416 | return error.AlreadyReported; | |
| 417 | }; | |
| 418 | ||
| 419 | const mapped_memory = std.posix.mmap( | |
| 420 | null, | |
| 421 | file_size, | |
| 422 | .{ .READ = true }, | |
| 423 | .{ .TYPE = .SHARED }, | |
| 424 | coverage_file.handle, | |
| 425 | 0, | |
| 426 | ) catch |err| { | |
| 427 | log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err }); | |
| 428 | return error.AlreadyReported; | |
| 429 | }; | |
| 430 | gop.value_ptr.mapped_memory = mapped_memory; | |
| 431 | ||
| 432 | const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); | |
| 433 | const pcs = header.pcAddrs(); | |
| 434 | const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len); | |
| 435 | errdefer gpa.free(source_locations); | |
| 436 | ||
| 437 | // Unfortunately the PCs array that LLVM gives us from the 8-bit PC | |
| 438 | // counters feature is not sorted. | |
| 439 | var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .empty; | |
| 440 | defer sorted_pcs.deinit(gpa); | |
| 441 | try sorted_pcs.resize(gpa, pcs.len); | |
| 442 | @memcpy(sorted_pcs.items(.pc), pcs); | |
| 443 | for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i); | |
| 444 | sorted_pcs.sortUnstable(struct { | |
| 445 | addrs: []const u64, | |
| 446 | ||
| 447 | pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { | |
| 448 | return ctx.addrs[a_index] < ctx.addrs[b_index]; | |
| 449 | } | |
| 450 | }{ .addrs = sorted_pcs.items(.pc) }); | |
| 451 | ||
| 452 | debug_info.resolveAddresses(gpa, io, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| { | |
| 453 | log.err("failed to resolve addresses to source locations: {t}", .{err}); | |
| 454 | return error.AlreadyReported; | |
| 455 | }; | |
| 456 | ||
| 457 | for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl; | |
| 458 | gop.value_ptr.source_locations = source_locations; | |
| 459 | gop.value_ptr.start_n_runs = header.n_runs; | |
| 460 | ||
| 461 | ws.notifyUpdate(); | |
| 462 | } | |
| 463 | ||
| 464 | fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void { | |
| 465 | const io = fuzz.io; | |
| 466 | ||
| 467 | try fuzz.coverage_mutex.lock(io); | |
| 468 | defer fuzz.coverage_mutex.unlock(io); | |
| 469 | ||
| 470 | const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?; | |
| 471 | const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); | |
| 472 | const pcs = header.pcAddrs(); | |
| 473 | ||
| 474 | // Since this pcs list is unsorted, we must linear scan for the best index. | |
| 475 | const index = i: { | |
| 476 | var best: usize = 0; | |
| 477 | for (pcs[1..], 1..) |elem_addr, i| { | |
| 478 | if (elem_addr == addr) break :i i; | |
| 479 | if (elem_addr > addr) continue; | |
| 480 | if (elem_addr > pcs[best]) best = i; | |
| 481 | } | |
| 482 | break :i best; | |
| 483 | }; | |
| 484 | if (index >= pcs.len) { | |
| 485 | log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{ | |
| 486 | addr, pcs[0], pcs[pcs.len - 1], | |
| 487 | }); | |
| 488 | return error.AlreadyReported; | |
| 489 | } | |
| 490 | if (false) { | |
| 491 | const sl = coverage_map.source_locations[index]; | |
| 492 | const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename); | |
| 493 | if (pcs.len == 1) { | |
| 494 | log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{ | |
| 495 | addr, file_name, sl.line, sl.column, | |
| 496 | }); | |
| 497 | } else if (index == 0) { | |
| 498 | log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{ | |
| 499 | addr, file_name, sl.line, sl.column, pcs[index + 1], | |
| 500 | }); | |
| 501 | } else if (index == pcs.len - 1) { | |
| 502 | log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{ | |
| 503 | addr, file_name, sl.line, sl.column, index, pcs[index - 1], | |
| 504 | }); | |
| 505 | } else { | |
| 506 | log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{ | |
| 507 | addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1], | |
| 508 | }); | |
| 509 | } | |
| 510 | } | |
| 511 | try coverage_map.entry_points.append(fuzz.gpa, @intCast(index)); | |
| 512 | } | |
| 513 | ||
| 514 | pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { | |
| 515 | assert(fuzz.mode == .limit); | |
| 516 | const io = fuzz.io; | |
| 517 | ||
| 518 | try fuzz.group.await(io); | |
| 519 | fuzz.group = .init; | |
| 520 | ||
| 521 | std.debug.print("======= FUZZING REPORT =======\n", .{}); | |
| 522 | for (fuzz.msg_queue.items) |msg| { | |
| 523 | if (msg != .coverage) continue; | |
| 524 | ||
| 525 | const cov = msg.coverage; | |
| 526 | const coverage_file_path: std.Build.Cache.Path = .{ | |
| 527 | .root_dir = cov.run.step.owner.cache_root, | |
| 528 | .sub_path = "v/" ++ std.fmt.hex(cov.id), | |
| 529 | }; | |
| 530 | var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { | |
| 531 | fatal("step '{s}': failed to load coverage file '{f}': {t}", .{ | |
| 532 | cov.run.step.name, coverage_file_path, err, | |
| 533 | }); | |
| 534 | }; | |
| 535 | defer coverage_file.close(io); | |
| 536 | ||
| 537 | const fuzz_abi = std.Build.abi.fuzz; | |
| 538 | var rbuf: [0x1000]u8 = undefined; | |
| 539 | var r = coverage_file.reader(io, &rbuf); | |
| 540 | ||
| 541 | var header: fuzz_abi.SeenPcsHeader = undefined; | |
| 542 | r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { | |
| 543 | fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ | |
| 544 | cov.run.step.name, coverage_file_path, err, | |
| 545 | }); | |
| 546 | }; | |
| 547 | ||
| 548 | if (header.pcs_len == 0) { | |
| 549 | fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{ | |
| 550 | cov.run.step.name, coverage_file_path, | |
| 551 | }); | |
| 552 | } | |
| 553 | ||
| 554 | var seen_count: usize = 0; | |
| 555 | const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len); | |
| 556 | for (0..chunk_count) |_| { | |
| 557 | const seen = r.interface.takeInt(usize, .little) catch |err| { | |
| 558 | fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ | |
| 559 | cov.run.step.name, coverage_file_path, err, | |
| 560 | }); | |
| 561 | }; | |
| 562 | seen_count += @popCount(seen); | |
| 563 | } | |
| 564 | ||
| 565 | const seen_f: f64 = @floatFromInt(seen_count); | |
| 566 | const total_f: f64 = @floatFromInt(header.pcs_len); | |
| 567 | const ratio = seen_f / total_f; | |
| 568 | std.debug.print( | |
| 569 | \\Step: {s} | |
| 570 | \\Fuzz test: "{s}" ({x}) | |
| 571 | \\Runs: {} -> {} | |
| 572 | \\Unique runs: {} -> {} | |
| 573 | \\Coverage: {}/{} -> {}/{} ({:.02}%) | |
| 574 | \\ | |
| 575 | , .{ | |
| 576 | cov.run.step.name, | |
| 577 | cov.run.fuzz_tests.items[0], | |
| 578 | cov.id, | |
| 579 | cov.cumulative.runs, | |
| 580 | header.n_runs, | |
| 581 | cov.cumulative.unique, | |
| 582 | header.unique_runs, | |
| 583 | cov.cumulative.coverage, | |
| 584 | header.pcs_len, | |
| 585 | seen_count, | |
| 586 | header.pcs_len, | |
| 587 | ratio * 100, | |
| 588 | }); | |
| 589 | ||
| 590 | std.debug.print("------------------------------\n", .{}); | |
| 591 | } | |
| 592 | std.debug.print( | |
| 593 | \\Values are accumulated across multiple runs when preserving the cache. | |
| 594 | \\============================== | |
| 595 | \\ | |
| 596 | , .{}); | |
| 597 | } |
lib/compiler/maker/Graph.zig created+92
| ... | ... | @@ -0,0 +1,92 @@ |
| 1 | //! Shared maker state among all steps. | |
| 2 | const Graph = @This(); | |
| 3 | ||
| 4 | const std = @import("std"); | |
| 5 | const Io = std.Io; | |
| 6 | const Allocator = std.mem.Allocator; | |
| 7 | const Configuration = std.Build.Configuration; | |
| 8 | ||
| 9 | const Step = @import("Step.zig"); | |
| 10 | const Package = @import("Package.zig"); | |
| 11 | ||
| 12 | io: Io, | |
| 13 | /// Process lifetime. | |
| 14 | arena: Allocator, | |
| 15 | system_library_options: std.StringArrayHashMapUnmanaged(std.Build.SystemLibraryMode), | |
| 16 | system_package_mode: bool, | |
| 17 | debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, | |
| 18 | cache: std.Build.Cache, | |
| 19 | zig_exe: [:0]const u8, | |
| 20 | environ_map: std.process.Environ.Map, | |
| 21 | global_cache_root: std.Build.Cache.Directory, | |
| 22 | zig_lib_directory: std.Build.Cache.Directory, | |
| 23 | incremental: ?bool, | |
| 24 | random_seed: u32, | |
| 25 | allow_so_scripts: ?bool, | |
| 26 | time_report: bool, | |
| 27 | /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also | |
| 28 | /// respects the '--color' flag. | |
| 29 | stderr_mode: ?Io.Terminal.Mode, | |
| 30 | ||
| 31 | configuration: *const Configuration, | |
| 32 | top_level_steps: std.AutoArrayHashMapUnmanaged(Configuration.String, Configuration.Step.Index), | |
| 33 | ||
| 34 | pub const DirList = struct { | |
| 35 | lib_dir: ?[]const u8 = null, | |
| 36 | exe_dir: ?[]const u8 = null, | |
| 37 | include_dir: ?[]const u8 = null, | |
| 38 | }; | |
| 39 | ||
| 40 | /// This function is intended to be called by lib/build_runner.zig, not a build.zig file. | |
| 41 | pub fn resolveInstallPrefix(graph: *Graph, p: *Package, install_prefix: ?[]const u8, dir_list: DirList) !void { | |
| 42 | if (p.dest_dir) |dest_dir| { | |
| 43 | p.install_prefix = install_prefix orelse "/usr"; | |
| 44 | p.install_path = b.pathJoin(&.{ dest_dir, p.install_prefix }); | |
| 45 | } else { | |
| 46 | p.install_prefix = install_prefix orelse | |
| 47 | (p.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); | |
| 48 | b.install_path = b.install_prefix; | |
| 49 | } | |
| 50 | ||
| 51 | var lib_list = [_][]const u8{ b.install_path, "lib" }; | |
| 52 | var exe_list = [_][]const u8{ b.install_path, "bin" }; | |
| 53 | var h_list = [_][]const u8{ b.install_path, "include" }; | |
| 54 | ||
| 55 | if (dir_list.lib_dir) |dir| { | |
| 56 | if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; | |
| 57 | lib_list[1] = dir; | |
| 58 | } | |
| 59 | ||
| 60 | if (dir_list.exe_dir) |dir| { | |
| 61 | if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; | |
| 62 | exe_list[1] = dir; | |
| 63 | } | |
| 64 | ||
| 65 | if (dir_list.include_dir) |dir| { | |
| 66 | if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; | |
| 67 | h_list[1] = dir; | |
| 68 | } | |
| 69 | ||
| 70 | b.lib_dir = b.pathJoin(&lib_list); | |
| 71 | b.exe_dir = b.pathJoin(&exe_list); | |
| 72 | b.h_dir = b.pathJoin(&h_list); | |
| 73 | } | |
| 74 | ||
| 75 | fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { | |
| 76 | // Create an installation directory local to this package. This will be used when | |
| 77 | // dependant packages require a standard prefix, such as include directories for C headers. | |
| 78 | var hash = b.graph.cache.hash; | |
| 79 | // Random bytes to make unique. Refresh this with new random bytes when | |
| 80 | // implementation is modified in a non-backwards-compatible way. | |
| 81 | hash.add(@as(u32, 0xd8cb0056)); | |
| 82 | hash.addBytes(b.dep_prefix); | |
| 83 | ||
| 84 | var wyhash = std.hash.Wyhash.init(0); | |
| 85 | hashUserInputOptionsMap(b.allocator, b.user_input_options, &wyhash); | |
| 86 | hash.add(wyhash.final()); | |
| 87 | ||
| 88 | const digest = hash.final(); | |
| 89 | const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); | |
| 90 | b.resolveInstallPrefix(install_prefix, .{}); | |
| 91 | } | |
| 92 |
lib/compiler/maker/Package.zig created+12
| ... | ... | @@ -0,0 +1,12 @@ |
| 1 | const Package = @This(); | |
| 2 | ||
| 3 | const std = @import("std"); | |
| 4 | ||
| 5 | install_prefix: []const u8, | |
| 6 | install_path: []const u8, | |
| 7 | dest_dir: ?[]const u8, | |
| 8 | lib_dir: []const u8, | |
| 9 | exe_dir: []const u8, | |
| 10 | h_dir: []const u8, | |
| 11 | /// Path to the directory containing build.zig. | |
| 12 | build_root: std.Build.Cache.Path, |
lib/compiler/maker/Step.zig created+850
| ... | ... | @@ -0,0 +1,850 @@ |
| 1 | const Step = @This(); | |
| 2 | ||
| 3 | const std = @import("std"); | |
| 4 | const Io = std.Io; | |
| 5 | const Allocator = std.mem.Allocator; | |
| 6 | const Cache = std.Build.Cache; | |
| 7 | const assert = std.debug.assert; | |
| 8 | ||
| 9 | const WebServer = @import("WebServer.zig"); | |
| 10 | ||
| 11 | pub const Compile = @import("Step/Compile.zig"); | |
| 12 | pub const Run = @import("Step/Run.zig"); | |
| 13 | ||
| 14 | state: State, | |
| 15 | makeFn: MakeFn, | |
| 16 | dependants: std.ArrayList(*Step), | |
| 17 | /// Collects the set of files that retrigger this step to run. | |
| 18 | /// | |
| 19 | /// This is used by the build system's implementation of `--watch` but it can | |
| 20 | /// also be potentially useful for IDEs to know what effects editing a | |
| 21 | /// particular file has. | |
| 22 | /// | |
| 23 | /// Populated within `make`. Implementation may choose to clear and repopulate, | |
| 24 | /// retain previous value, or update. | |
| 25 | inputs: Inputs = .init, | |
| 26 | pending_deps: u32, | |
| 27 | ||
| 28 | result_error_msgs: std.ArrayList([]const u8), | |
| 29 | result_error_bundle: std.zig.ErrorBundle, | |
| 30 | result_stderr: []const u8, | |
| 31 | result_cached: bool, | |
| 32 | result_duration_ns: ?u64, | |
| 33 | /// 0 means unavailable or not reported. | |
| 34 | result_peak_rss: usize, | |
| 35 | /// If the step is failed and this field is populated, this is the command which failed. | |
| 36 | /// This field may be populated even if the step succeeded. | |
| 37 | result_failed_command: ?[]const u8, | |
| 38 | test_results: TestResults, | |
| 39 | ||
| 40 | ||
| 41 | pub const State = enum { | |
| 42 | precheck_unstarted, | |
| 43 | precheck_started, | |
| 44 | /// This is also used to indicate "dirty" steps that have been modified | |
| 45 | /// after a previous build completed, in which case, the step may or may | |
| 46 | /// not have been completed before. Either way, one or more of its direct | |
| 47 | /// file system inputs have been modified, meaning that the step needs to | |
| 48 | /// be re-evaluated. | |
| 49 | precheck_done, | |
| 50 | dependency_failure, | |
| 51 | success, | |
| 52 | failure, | |
| 53 | /// This state indicates that the step did not complete, however, it also did not fail, | |
| 54 | /// and it is safe to continue executing its dependencies. | |
| 55 | skipped, | |
| 56 | /// This step was skipped because it specified a max_rss that exceeded the runner's maximum. | |
| 57 | /// It is not safe to run its dependencies. | |
| 58 | skipped_oom, | |
| 59 | }; | |
| 60 | ||
| 61 | pub const Inputs = struct { | |
| 62 | table: Table, | |
| 63 | ||
| 64 | pub const init: Inputs = .{ | |
| 65 | .table = .{}, | |
| 66 | }; | |
| 67 | ||
| 68 | pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, Files, Cache.Path.TableAdapter, false); | |
| 69 | /// The special file name "." means any changes inside the directory. | |
| 70 | pub const Files = std.ArrayList([]const u8); | |
| 71 | ||
| 72 | pub fn populated(inputs: *Inputs) bool { | |
| 73 | return inputs.table.count() != 0; | |
| 74 | } | |
| 75 | ||
| 76 | pub fn clear(inputs: *Inputs, gpa: Allocator) void { | |
| 77 | for (inputs.table.values()) |*files| files.deinit(gpa); | |
| 78 | inputs.table.clearRetainingCapacity(); | |
| 79 | } | |
| 80 | }; | |
| 81 | ||
| 82 | pub const TestResults = struct { | |
| 83 | /// The total number of tests in the step. Every test has a "status" from the following: | |
| 84 | /// * passed | |
| 85 | /// * skipped | |
| 86 | /// * failed cleanly | |
| 87 | /// * crashed | |
| 88 | /// * timed out | |
| 89 | test_count: u32 = 0, | |
| 90 | ||
| 91 | /// The number of tests which were skipped (`error.SkipZigTest`). | |
| 92 | skip_count: u32 = 0, | |
| 93 | /// The number of tests which failed cleanly. | |
| 94 | fail_count: u32 = 0, | |
| 95 | /// The number of tests which terminated unexpectedly, i.e. crashed. | |
| 96 | crash_count: u32 = 0, | |
| 97 | /// The number of tests which timed out. | |
| 98 | timeout_count: u32 = 0, | |
| 99 | ||
| 100 | /// The number of detected memory leaks. The associated test may still have passed; indeed, *all* | |
| 101 | /// individual tests may have passed. However, the step as a whole fails if any test has leaks. | |
| 102 | leak_count: u32 = 0, | |
| 103 | /// The number of detected error logs. The associated test may still have passed; indeed, *all* | |
| 104 | /// individual tests may have passed. However, the step as a whole fails if any test logs errors. | |
| 105 | log_err_count: u32 = 0, | |
| 106 | ||
| 107 | pub fn isSuccess(tr: TestResults) bool { | |
| 108 | // all steps are success or skip | |
| 109 | return tr.fail_count == 0 and | |
| 110 | tr.crash_count == 0 and | |
| 111 | tr.timeout_count == 0 and | |
| 112 | // no (otherwise successful) step leaked memory or logged errors | |
| 113 | tr.leak_count == 0 and | |
| 114 | tr.log_err_count == 0; | |
| 115 | } | |
| 116 | ||
| 117 | /// Computes the number of tests which passed from the other values. | |
| 118 | pub fn passCount(tr: TestResults) u32 { | |
| 119 | return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count; | |
| 120 | } | |
| 121 | }; | |
| 122 | ||
| 123 | pub const MakeOptions = struct { | |
| 124 | progress_node: std.Progress.Node, | |
| 125 | watch: bool, | |
| 126 | web_server: ?*WebServer, | |
| 127 | /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. | |
| 128 | unit_test_timeout_ns: ?u64, | |
| 129 | /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. | |
| 130 | gpa: Allocator, | |
| 131 | }; | |
| 132 | ||
| 133 | pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; | |
| 134 | ||
| 135 | /// If the Step's `make` function reports `error.MakeFailed`, it indicates they | |
| 136 | /// have already reported the error. Otherwise, we add a simple error report | |
| 137 | /// here. | |
| 138 | pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { | |
| 139 | const arena = s.owner.allocator; | |
| 140 | const graph = s.owner.graph; | |
| 141 | const io = graph.io; | |
| 142 | ||
| 143 | var start_ts: ?Io.Timestamp = t: { | |
| 144 | if (!graph.time_report) break :t null; | |
| 145 | if (s.id == .compile) break :t null; | |
| 146 | if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; | |
| 147 | break :t Io.Clock.awake.now(io); | |
| 148 | }; | |
| 149 | const make_result = s.makeFn(s, options); | |
| 150 | if (start_ts) |*ts| { | |
| 151 | const duration = ts.untilNow(io, .awake); | |
| 152 | options.web_server.?.updateTimeReportGeneric(s, duration); | |
| 153 | } | |
| 154 | ||
| 155 | make_result catch |err| switch (err) { | |
| 156 | error.MakeFailed, error.MakeSkipped => |e| return e, | |
| 157 | else => { | |
| 158 | s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); | |
| 159 | return error.MakeFailed; | |
| 160 | }, | |
| 161 | }; | |
| 162 | ||
| 163 | if (!s.test_results.isSuccess()) { | |
| 164 | return error.MakeFailed; | |
| 165 | } | |
| 166 | ||
| 167 | if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { | |
| 168 | const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ | |
| 169 | s.result_peak_rss, s.max_rss, | |
| 170 | }) catch @panic("OOM"); | |
| 171 | s.result_error_msgs.append(arena, msg) catch @panic("OOM"); | |
| 172 | } | |
| 173 | } | |
| 174 | ||
| 175 | fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { | |
| 176 | _ = options; | |
| 177 | ||
| 178 | var all_cached = true; | |
| 179 | ||
| 180 | for (step.dependencies.items) |dep| { | |
| 181 | all_cached = all_cached and dep.result_cached; | |
| 182 | } | |
| 183 | ||
| 184 | step.result_cached = all_cached; | |
| 185 | } | |
| 186 | ||
| 187 | /// Implementation detail of file watching. Prepares the step for being re-evaluated. | |
| 188 | /// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. | |
| 189 | pub fn invalidateResult(step: *Step, gpa: Allocator) bool { | |
| 190 | if (step.state == .precheck_done) return false; | |
| 191 | assert(step.pending_deps == 0); | |
| 192 | step.state = .precheck_done; | |
| 193 | step.reset(gpa); | |
| 194 | for (step.dependants.items) |dependant| { | |
| 195 | _ = dependant.invalidateResult(gpa); | |
| 196 | dependant.pending_deps += 1; | |
| 197 | } | |
| 198 | return true; | |
| 199 | } | |
| 200 | ||
| 201 | /// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. | |
| 202 | pub fn reset(step: *Step, gpa: Allocator) void { | |
| 203 | assert(step.state == .precheck_done); | |
| 204 | ||
| 205 | if (step.result_failed_command) |cmd| gpa.free(cmd); | |
| 206 | ||
| 207 | step.result_error_msgs.clearRetainingCapacity(); | |
| 208 | step.result_stderr = ""; | |
| 209 | step.result_cached = false; | |
| 210 | step.result_duration_ns = null; | |
| 211 | step.result_peak_rss = 0; | |
| 212 | step.result_failed_command = null; | |
| 213 | step.test_results = .{}; | |
| 214 | step.clearWatchInputs(); | |
| 215 | ||
| 216 | step.result_error_bundle.deinit(gpa); | |
| 217 | step.result_error_bundle = std.zig.ErrorBundle.empty; | |
| 218 | } | |
| 219 | ||
| 220 | /// Populates `s.result_failed_command`. | |
| 221 | pub fn captureChildProcess( | |
| 222 | s: *Step, | |
| 223 | gpa: Allocator, | |
| 224 | progress_node: std.Progress.Node, | |
| 225 | argv: []const []const u8, | |
| 226 | ) !std.process.RunResult { | |
| 227 | const graph = s.owner.graph; | |
| 228 | const arena = graph.arena; | |
| 229 | const io = graph.io; | |
| 230 | ||
| 231 | // If an error occurs, it's happened in this command: | |
| 232 | assert(s.result_failed_command == null); | |
| 233 | s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); | |
| 234 | ||
| 235 | try handleChildProcUnsupported(s); | |
| 236 | try handleVerbose(s.owner, .inherit, argv); | |
| 237 | ||
| 238 | const result = std.process.run(arena, io, .{ | |
| 239 | .argv = argv, | |
| 240 | .environ_map = &graph.environ_map, | |
| 241 | .progress_node = progress_node, | |
| 242 | }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); | |
| 243 | ||
| 244 | if (result.stderr.len > 0) { | |
| 245 | try s.result_error_msgs.append(arena, result.stderr); | |
| 246 | } | |
| 247 | ||
| 248 | return result; | |
| 249 | } | |
| 250 | ||
| 251 | pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { | |
| 252 | try step.addError(fmt, args); | |
| 253 | return error.MakeFailed; | |
| 254 | } | |
| 255 | ||
| 256 | pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { | |
| 257 | const arena = step.owner.allocator; | |
| 258 | const msg = try std.fmt.allocPrint(arena, fmt, args); | |
| 259 | try step.result_error_msgs.append(arena, msg); | |
| 260 | } | |
| 261 | ||
| 262 | pub const ZigProcess = struct { | |
| 263 | child: std.process.Child, | |
| 264 | multi_reader_buffer: Io.File.MultiReader.Buffer(2), | |
| 265 | multi_reader: Io.File.MultiReader, | |
| 266 | progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn, | |
| 267 | ||
| 268 | pub const StreamEnum = enum { stdout, stderr }; | |
| 269 | ||
| 270 | pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void { | |
| 271 | zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null; | |
| 272 | } | |
| 273 | ||
| 274 | pub fn deinit(zp: *ZigProcess, io: Io) void { | |
| 275 | zp.child.kill(io); | |
| 276 | zp.multi_reader.deinit(); | |
| 277 | zp.* = undefined; | |
| 278 | } | |
| 279 | }; | |
| 280 | ||
| 281 | /// Assumes that argv contains `--listen=-` and that the process being spawned | |
| 282 | /// is the zig compiler - the same version that compiled the build runner. | |
| 283 | /// Populates `s.result_failed_command`. | |
| 284 | pub fn evalZigProcess( | |
| 285 | s: *Step, | |
| 286 | argv: []const []const u8, | |
| 287 | prog_node: std.Progress.Node, | |
| 288 | watch: bool, | |
| 289 | web_server: ?*WebServer, | |
| 290 | gpa: Allocator, | |
| 291 | ) !?Cache.Path { | |
| 292 | const b = s.owner; | |
| 293 | const io = b.graph.io; | |
| 294 | ||
| 295 | // If an error occurs, it's happened in this command: | |
| 296 | assert(s.result_failed_command == null); | |
| 297 | s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); | |
| 298 | ||
| 299 | if (s.getZigProcess()) |zp| update: { | |
| 300 | assert(watch); | |
| 301 | if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index); | |
| 302 | zp.progress_ipc_index = null; | |
| 303 | var exited = false; | |
| 304 | defer if (exited) { | |
| 305 | s.cast(Compile).?.zig_process = null; | |
| 306 | zp.deinit(io); | |
| 307 | gpa.destroy(zp); | |
| 308 | } else zp.saveState(prog_node); | |
| 309 | const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { | |
| 310 | error.BrokenPipe, error.EndOfStream => |reason| { | |
| 311 | std.log.info("{s} restart required: {t}", .{ argv[0], reason }); | |
| 312 | // Process restart required. | |
| 313 | const term = zp.child.wait(io) catch |e| { | |
| 314 | return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); | |
| 315 | }; | |
| 316 | _ = term; | |
| 317 | exited = true; | |
| 318 | break :update; | |
| 319 | }, | |
| 320 | else => |e| return e, | |
| 321 | }; | |
| 322 | ||
| 323 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 324 | return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); | |
| 325 | } | |
| 326 | ||
| 327 | if (s.result_error_msgs.items.len > 0 and result == null) { | |
| 328 | // Crash detected. | |
| 329 | const term = zp.child.wait(io) catch |e| { | |
| 330 | return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); | |
| 331 | }; | |
| 332 | s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; | |
| 333 | exited = true; | |
| 334 | try handleChildProcessTerm(s, term); | |
| 335 | return error.MakeFailed; | |
| 336 | } | |
| 337 | ||
| 338 | return result; | |
| 339 | } | |
| 340 | assert(argv.len != 0); | |
| 341 | ||
| 342 | try handleChildProcUnsupported(s); | |
| 343 | try handleVerbose(s.owner, .inherit, argv); | |
| 344 | ||
| 345 | const zp = try gpa.create(ZigProcess); | |
| 346 | defer if (!watch) gpa.destroy(zp); | |
| 347 | ||
| 348 | zp.child = std.process.spawn(io, .{ | |
| 349 | .argv = argv, | |
| 350 | .environ_map = &b.graph.environ_map, | |
| 351 | .stdin = .pipe, | |
| 352 | .stdout = .pipe, | |
| 353 | .stderr = .pipe, | |
| 354 | .request_resource_usage_statistics = true, | |
| 355 | .progress_node = prog_node, | |
| 356 | }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); | |
| 357 | ||
| 358 | zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ | |
| 359 | zp.child.stdout.?, zp.child.stderr.?, | |
| 360 | }); | |
| 361 | if (watch) s.cast(Compile).?.zig_process = zp; | |
| 362 | defer if (!watch) zp.deinit(io); | |
| 363 | ||
| 364 | const result = result: { | |
| 365 | defer if (watch) zp.saveState(prog_node); | |
| 366 | break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); | |
| 367 | }; | |
| 368 | ||
| 369 | if (!watch) { | |
| 370 | // Send EOF to stdin. | |
| 371 | zp.child.stdin.?.close(io); | |
| 372 | zp.child.stdin = null; | |
| 373 | ||
| 374 | const term = zp.child.wait(io) catch |err| { | |
| 375 | return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); | |
| 376 | }; | |
| 377 | s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; | |
| 378 | ||
| 379 | // Special handling for Compile step that is expecting compile errors. | |
| 380 | if (s.cast(Compile)) |compile| switch (term) { | |
| 381 | .exited => { | |
| 382 | // Note that the exit code may be 0 in this case due to the | |
| 383 | // compiler server protocol. | |
| 384 | if (compile.expect_errors != null) { | |
| 385 | return error.NeedCompileErrorCheck; | |
| 386 | } | |
| 387 | }, | |
| 388 | else => {}, | |
| 389 | }; | |
| 390 | ||
| 391 | try handleChildProcessTerm(s, term); | |
| 392 | } | |
| 393 | ||
| 394 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 395 | return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); | |
| 396 | } | |
| 397 | ||
| 398 | return result; | |
| 399 | } | |
| 400 | ||
| 401 | /// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. | |
| 402 | pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { | |
| 403 | const b = s.owner; | |
| 404 | const io = b.graph.io; | |
| 405 | const src_path = src_lazy_path.getPath3(b, s); | |
| 406 | try handleVerbose(b, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); | |
| 407 | return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| | |
| 408 | return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); | |
| 409 | } | |
| 410 | ||
| 411 | /// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. | |
| 412 | pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { | |
| 413 | const b = s.owner; | |
| 414 | const io = b.graph.io; | |
| 415 | try handleVerbose(b, .inherit, &.{ "install", "-d", dest_path }); | |
| 416 | return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| | |
| 417 | return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); | |
| 418 | } | |
| 419 | ||
| 420 | fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*WebServer, gpa: Allocator) !?Path { | |
| 421 | const b = s.owner; | |
| 422 | const arena = b.allocator; | |
| 423 | const io = b.graph.io; | |
| 424 | ||
| 425 | const start_ts = Io.Clock.awake.now(io); | |
| 426 | ||
| 427 | try sendMessage(io, zp.child.stdin.?, .update); | |
| 428 | if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); | |
| 429 | ||
| 430 | var result: ?Path = null; | |
| 431 | var eos_err: error{EndOfStream}!void = {}; | |
| 432 | ||
| 433 | const stdout = zp.multi_reader.fileReader(0); | |
| 434 | ||
| 435 | while (true) { | |
| 436 | const Header = std.zig.Server.Message.Header; | |
| 437 | const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) { | |
| 438 | error.EndOfStream => break, | |
| 439 | error.ReadFailed => return stdout.err.?, | |
| 440 | }; | |
| 441 | const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) { | |
| 442 | error.EndOfStream => |e| { | |
| 443 | // Better to report the crash with stderr below, but we set | |
| 444 | // this in case the child exits successfully while violating | |
| 445 | // this protocol. | |
| 446 | eos_err = e; | |
| 447 | break; | |
| 448 | }, | |
| 449 | error.ReadFailed => return stdout.err.?, | |
| 450 | }; | |
| 451 | switch (header.tag) { | |
| 452 | .zig_version => { | |
| 453 | if (!std.mem.eql(u8, builtin.zig_version_string, body)) { | |
| 454 | return s.fail( | |
| 455 | "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", | |
| 456 | .{ builtin.zig_version_string, body }, | |
| 457 | ); | |
| 458 | } | |
| 459 | }, | |
| 460 | .error_bundle => { | |
| 461 | s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); | |
| 462 | // This message indicates the end of the update. | |
| 463 | if (watch) break; | |
| 464 | }, | |
| 465 | .emit_digest => { | |
| 466 | const EmitDigest = std.zig.Server.Message.EmitDigest; | |
| 467 | const emit_digest: *align(1) const EmitDigest = @ptrCast(body); | |
| 468 | s.result_cached = emit_digest.flags.cache_hit; | |
| 469 | const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; | |
| 470 | result = .{ | |
| 471 | .root_dir = b.cache_root, | |
| 472 | .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), | |
| 473 | }; | |
| 474 | }, | |
| 475 | .file_system_inputs => { | |
| 476 | s.clearWatchInputs(); | |
| 477 | var it = std.mem.splitScalar(u8, body, 0); | |
| 478 | while (it.next()) |prefixed_path| { | |
| 479 | const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); | |
| 480 | const sub_path = try arena.dupe(u8, prefixed_path[1..]); | |
| 481 | const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; | |
| 482 | switch (prefix_index) { | |
| 483 | .cwd => { | |
| 484 | const path: Cache.Path = .{ | |
| 485 | .root_dir = Cache.Directory.cwd(), | |
| 486 | .sub_path = sub_path_dirname, | |
| 487 | }; | |
| 488 | try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); | |
| 489 | }, | |
| 490 | .zig_lib => zl: { | |
| 491 | if (s.cast(Step.Compile)) |compile| { | |
| 492 | if (compile.zig_lib_dir) |zig_lib_dir| { | |
| 493 | const lp = try zig_lib_dir.join(arena, sub_path); | |
| 494 | try addWatchInput(s, lp); | |
| 495 | break :zl; | |
| 496 | } | |
| 497 | } | |
| 498 | const path: Cache.Path = .{ | |
| 499 | .root_dir = s.owner.graph.zig_lib_directory, | |
| 500 | .sub_path = sub_path_dirname, | |
| 501 | }; | |
| 502 | try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); | |
| 503 | }, | |
| 504 | .local_cache => { | |
| 505 | const path: Cache.Path = .{ | |
| 506 | .root_dir = b.cache_root, | |
| 507 | .sub_path = sub_path_dirname, | |
| 508 | }; | |
| 509 | try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); | |
| 510 | }, | |
| 511 | .global_cache => { | |
| 512 | const path: Cache.Path = .{ | |
| 513 | .root_dir = s.owner.graph.global_cache_root, | |
| 514 | .sub_path = sub_path_dirname, | |
| 515 | }; | |
| 516 | try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); | |
| 517 | }, | |
| 518 | } | |
| 519 | } | |
| 520 | }, | |
| 521 | .time_report => if (web_server) |ws| { | |
| 522 | const TimeReport = std.zig.Server.Message.TimeReport; | |
| 523 | const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); | |
| 524 | ws.updateTimeReportCompile(.{ | |
| 525 | .compile = s.cast(Step.Compile).?, | |
| 526 | .use_llvm = tr.flags.use_llvm, | |
| 527 | .stats = tr.stats, | |
| 528 | .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), | |
| 529 | .llvm_pass_timings_len = tr.llvm_pass_timings_len, | |
| 530 | .files_len = tr.files_len, | |
| 531 | .decls_len = tr.decls_len, | |
| 532 | .trailing = body[@sizeOf(TimeReport)..], | |
| 533 | }); | |
| 534 | }, | |
| 535 | else => {}, // ignore other messages | |
| 536 | } | |
| 537 | } | |
| 538 | ||
| 539 | s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()); | |
| 540 | ||
| 541 | const stderr_contents = zp.multi_reader.reader(1).buffered(); | |
| 542 | if (stderr_contents.len > 0) { | |
| 543 | try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); | |
| 544 | } | |
| 545 | ||
| 546 | try eos_err; | |
| 547 | ||
| 548 | return result; | |
| 549 | } | |
| 550 | ||
| 551 | pub fn getZigProcess(s: *Step) ?*ZigProcess { | |
| 552 | return switch (s.id) { | |
| 553 | .compile => s.cast(Compile).?.zig_process, | |
| 554 | else => null, | |
| 555 | }; | |
| 556 | } | |
| 557 | ||
| 558 | fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { | |
| 559 | const header: std.zig.Client.Message.Header = .{ | |
| 560 | .tag = tag, | |
| 561 | .bytes_len = 0, | |
| 562 | }; | |
| 563 | var w = file.writer(io, &.{}); | |
| 564 | w.interface.writeStruct(header, .little) catch |err| switch (err) { | |
| 565 | error.WriteFailed => return w.err.?, | |
| 566 | }; | |
| 567 | } | |
| 568 | ||
| 569 | pub fn handleVerbose( | |
| 570 | b: *Build, | |
| 571 | cwd: std.process.Child.Cwd, | |
| 572 | argv: []const []const u8, | |
| 573 | ) error{OutOfMemory}!void { | |
| 574 | return handleVerbose2(b, cwd, null, argv); | |
| 575 | } | |
| 576 | ||
| 577 | pub fn handleVerbose2( | |
| 578 | b: *Build, | |
| 579 | cwd: std.process.Child.Cwd, | |
| 580 | opt_env: ?*const std.process.Environ.Map, | |
| 581 | argv: []const []const u8, | |
| 582 | ) error{OutOfMemory}!void { | |
| 583 | if (b.verbose) { | |
| 584 | const graph = b.graph; | |
| 585 | // Intention of verbose is to print all sub-process command lines to | |
| 586 | // stderr before spawning them. | |
| 587 | const text = try allocPrintCmd(b.allocator, cwd, if (opt_env) |env| .{ | |
| 588 | .child = env, | |
| 589 | .parent = &graph.environ_map, | |
| 590 | } else null, argv); | |
| 591 | std.debug.print("{s}\n", .{text}); | |
| 592 | } | |
| 593 | } | |
| 594 | ||
| 595 | /// Asserts that the caller has already populated `s.result_failed_command`. | |
| 596 | pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { | |
| 597 | if (!std.process.can_spawn) { | |
| 598 | return s.fail("unable to spawn process: host cannot spawn child processes", .{}); | |
| 599 | } | |
| 600 | } | |
| 601 | ||
| 602 | /// Asserts that the caller has already populated `s.result_failed_command`. | |
| 603 | pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { | |
| 604 | assert(s.result_failed_command != null); | |
| 605 | return switch (term) { | |
| 606 | .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), | |
| 607 | .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), | |
| 608 | .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), | |
| 609 | .unknown => s.fail("process terminated unexpectedly", .{}), | |
| 610 | }; | |
| 611 | } | |
| 612 | ||
| 613 | /// Prefer `cacheHitAndWatch` unless you already added watch inputs | |
| 614 | /// separately from using the cache system. | |
| 615 | pub fn cacheHit(s: *Step, man: *Cache.Manifest) !bool { | |
| 616 | s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); | |
| 617 | return s.result_cached; | |
| 618 | } | |
| 619 | ||
| 620 | /// Clears previous watch inputs, if any, and then populates watch inputs from | |
| 621 | /// the full set of files picked up by the cache manifest. | |
| 622 | /// | |
| 623 | /// Must be accompanied with `writeManifestAndWatch`. | |
| 624 | pub fn cacheHitAndWatch(s: *Step, man: *Cache.Manifest) !bool { | |
| 625 | const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); | |
| 626 | s.result_cached = is_hit; | |
| 627 | // The above call to hit() populates the manifest with files, so in case of | |
| 628 | // a hit, we need to populate watch inputs. | |
| 629 | if (is_hit) try setWatchInputsFromManifest(s, man); | |
| 630 | return is_hit; | |
| 631 | } | |
| 632 | ||
| 633 | fn failWithCacheError( | |
| 634 | s: *Step, | |
| 635 | man: *const Cache.Manifest, | |
| 636 | err: Cache.Manifest.HitError, | |
| 637 | ) error{ OutOfMemory, Canceled, MakeFailed } { | |
| 638 | switch (err) { | |
| 639 | error.CacheCheckFailed => switch (man.diagnostic) { | |
| 640 | .none => unreachable, | |
| 641 | .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ | |
| 642 | man.diagnostic, e, | |
| 643 | }), | |
| 644 | .file_open, .file_stat, .file_read, .file_hash => |op| { | |
| 645 | const pp = man.files.keys()[op.file_index].prefixed_path; | |
| 646 | const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; | |
| 647 | return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ | |
| 648 | prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, | |
| 649 | }); | |
| 650 | }, | |
| 651 | }, | |
| 652 | error.OutOfMemory, error.Canceled => |e| return e, | |
| 653 | error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), | |
| 654 | } | |
| 655 | } | |
| 656 | ||
| 657 | /// Prefer `writeManifestAndWatch` unless you already added watch inputs | |
| 658 | /// separately from using the cache system. | |
| 659 | pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { | |
| 660 | if (s.test_results.isSuccess()) { | |
| 661 | man.writeManifest() catch |err| { | |
| 662 | try s.addError("unable to write cache manifest: {t}", .{err}); | |
| 663 | }; | |
| 664 | } | |
| 665 | } | |
| 666 | ||
| 667 | /// Clears previous watch inputs, if any, and then populates watch inputs from | |
| 668 | /// the full set of files picked up by the cache manifest. | |
| 669 | /// | |
| 670 | /// Must be accompanied with `cacheHitAndWatch`. | |
| 671 | pub fn writeManifestAndWatch(s: *Step, man: *Cache.Manifest) !void { | |
| 672 | try writeManifest(s, man); | |
| 673 | try setWatchInputsFromManifest(s, man); | |
| 674 | } | |
| 675 | ||
| 676 | fn setWatchInputsFromManifest(s: *Step, man: *Cache.Manifest) !void { | |
| 677 | const arena = s.owner.allocator; | |
| 678 | const prefixes = man.cache.prefixes(); | |
| 679 | clearWatchInputs(s); | |
| 680 | for (man.files.keys()) |file| { | |
| 681 | // The file path data is freed when the cache manifest is cleaned up at the end of `make`. | |
| 682 | const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); | |
| 683 | try addWatchInputFromPath(s, .{ | |
| 684 | .root_dir = prefixes[file.prefixed_path.prefix], | |
| 685 | .sub_path = std.fs.path.dirname(sub_path) orelse "", | |
| 686 | }, std.fs.path.basename(sub_path)); | |
| 687 | } | |
| 688 | } | |
| 689 | ||
| 690 | /// For steps that have a single input that never changes when re-running `make`. | |
| 691 | pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { | |
| 692 | if (!step.inputs.populated()) try step.addWatchInput(lazy_path); | |
| 693 | } | |
| 694 | ||
| 695 | pub fn clearWatchInputs(step: *Step) void { | |
| 696 | const gpa = step.owner.allocator; | |
| 697 | step.inputs.clear(gpa); | |
| 698 | } | |
| 699 | ||
| 700 | /// Places a *file* dependency on the path. | |
| 701 | pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!void { | |
| 702 | switch (lazy_file) { | |
| 703 | .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), | |
| 704 | .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), | |
| 705 | .cwd_relative => |path_string| { | |
| 706 | try addWatchInputFromPath(step, .{ | |
| 707 | .root_dir = .{ | |
| 708 | .path = null, | |
| 709 | .handle = Io.Dir.cwd(), | |
| 710 | }, | |
| 711 | .sub_path = std.fs.path.dirname(path_string) orelse "", | |
| 712 | }, std.fs.path.basename(path_string)); | |
| 713 | }, | |
| 714 | // Nothing to watch because this dependency edge is modeled instead via `dependants`. | |
| 715 | .generated => {}, | |
| 716 | } | |
| 717 | } | |
| 718 | ||
| 719 | /// Any changes inside the directory will trigger invalidation. | |
| 720 | /// | |
| 721 | /// See also `addDirectoryWatchInputFromPath` which takes a `Cache.Path` instead. | |
| 722 | /// | |
| 723 | /// Paths derived from this directory should also be manually added via | |
| 724 | /// `addDirectoryWatchInputFromPath` if and only if this function returns | |
| 725 | /// `true`. | |
| 726 | pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool { | |
| 727 | switch (lazy_directory) { | |
| 728 | .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), | |
| 729 | .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), | |
| 730 | .cwd_relative => |path_string| { | |
| 731 | try addDirectoryWatchInputFromPath(step, .{ | |
| 732 | .root_dir = .{ | |
| 733 | .path = null, | |
| 734 | .handle = Io.Dir.cwd(), | |
| 735 | }, | |
| 736 | .sub_path = path_string, | |
| 737 | }); | |
| 738 | }, | |
| 739 | // Nothing to watch because this dependency edge is modeled instead via `dependants`. | |
| 740 | .generated => return false, | |
| 741 | } | |
| 742 | return true; | |
| 743 | } | |
| 744 | ||
| 745 | /// Any changes inside the directory will trigger invalidation. | |
| 746 | /// | |
| 747 | /// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. | |
| 748 | /// | |
| 749 | /// This function should only be called when it has been verified that the | |
| 750 | /// dependency on `path` is not already accounted for by a `Step` dependency. | |
| 751 | /// In other words, before calling this function, first check that the | |
| 752 | /// `Build.LazyPath` which this `path` is derived from is not `generated`. | |
| 753 | pub fn addDirectoryWatchInputFromPath(step: *Step, path: Cache.Path) !void { | |
| 754 | return addWatchInputFromPath(step, path, "."); | |
| 755 | } | |
| 756 | ||
| 757 | fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { | |
| 758 | return addWatchInputFromPath(step, .{ | |
| 759 | .root_dir = builder.build_root, | |
| 760 | .sub_path = std.fs.path.dirname(sub_path) orelse "", | |
| 761 | }, std.fs.path.basename(sub_path)); | |
| 762 | } | |
| 763 | ||
| 764 | fn addDirectoryWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { | |
| 765 | return addDirectoryWatchInputFromPath(step, .{ | |
| 766 | .root_dir = builder.build_root, | |
| 767 | .sub_path = sub_path, | |
| 768 | }); | |
| 769 | } | |
| 770 | ||
| 771 | fn addWatchInputFromPath(step: *Step, path: Cache.Path, basename: []const u8) !void { | |
| 772 | const gpa = step.owner.allocator; | |
| 773 | const gop = try step.inputs.table.getOrPut(gpa, path); | |
| 774 | if (!gop.found_existing) gop.value_ptr.* = .empty; | |
| 775 | try gop.value_ptr.append(gpa, basename); | |
| 776 | } | |
| 777 | ||
| 778 | pub fn allocPrintCmd( | |
| 779 | gpa: Allocator, | |
| 780 | cwd: std.process.Child.Cwd, | |
| 781 | opt_env: ?struct { | |
| 782 | child: *const std.process.Environ.Map, | |
| 783 | parent: *const std.process.Environ.Map, | |
| 784 | }, | |
| 785 | argv: []const []const u8, | |
| 786 | ) Allocator.Error![]u8 { | |
| 787 | const shell = struct { | |
| 788 | fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { | |
| 789 | for (string) |c| { | |
| 790 | if (switch (c) { | |
| 791 | else => true, | |
| 792 | '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, | |
| 793 | '=' => is_argv0, | |
| 794 | }) break; | |
| 795 | } else return writer.writeAll(string); | |
| 796 | ||
| 797 | try writer.writeByte('"'); | |
| 798 | for (string) |c| { | |
| 799 | if (switch (c) { | |
| 800 | std.ascii.control_code.nul => break, | |
| 801 | '!', '"', '$', '\\', '`' => true, | |
| 802 | else => !std.ascii.isPrint(c), | |
| 803 | }) try writer.writeByte('\\'); | |
| 804 | switch (c) { | |
| 805 | std.ascii.control_code.nul => unreachable, | |
| 806 | std.ascii.control_code.bel => try writer.writeByte('a'), | |
| 807 | std.ascii.control_code.bs => try writer.writeByte('b'), | |
| 808 | std.ascii.control_code.ht => try writer.writeByte('t'), | |
| 809 | std.ascii.control_code.lf => try writer.writeByte('n'), | |
| 810 | std.ascii.control_code.vt => try writer.writeByte('v'), | |
| 811 | std.ascii.control_code.ff => try writer.writeByte('f'), | |
| 812 | std.ascii.control_code.cr => try writer.writeByte('r'), | |
| 813 | std.ascii.control_code.esc => try writer.writeByte('E'), | |
| 814 | ' '...'~' => try writer.writeByte(c), | |
| 815 | else => try writer.print("{o:0>3}", .{c}), | |
| 816 | } | |
| 817 | } | |
| 818 | try writer.writeByte('"'); | |
| 819 | } | |
| 820 | }; | |
| 821 | ||
| 822 | var aw: Io.Writer.Allocating = .init(gpa); | |
| 823 | defer aw.deinit(); | |
| 824 | const writer = &aw.writer; | |
| 825 | switch (cwd) { | |
| 826 | .inherit => {}, | |
| 827 | .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, | |
| 828 | .dir => @panic("TODO"), | |
| 829 | } | |
| 830 | if (opt_env) |env| { | |
| 831 | var it = env.child.iterator(); | |
| 832 | while (it.next()) |entry| { | |
| 833 | const key = entry.key_ptr.*; | |
| 834 | const value = entry.value_ptr.*; | |
| 835 | if (env.parent.get(key)) |process_value| { | |
| 836 | if (std.mem.eql(u8, value, process_value)) continue; | |
| 837 | } | |
| 838 | writer.print("{s}=", .{key}) catch return error.OutOfMemory; | |
| 839 | shell.escape(writer, value, false) catch return error.OutOfMemory; | |
| 840 | writer.writeByte(' ') catch return error.OutOfMemory; | |
| 841 | } | |
| 842 | } | |
| 843 | shell.escape(writer, argv[0], true) catch return error.OutOfMemory; | |
| 844 | for (argv[1..]) |arg| { | |
| 845 | writer.writeByte(' ') catch return error.OutOfMemory; | |
| 846 | shell.escape(writer, arg, false) catch return error.OutOfMemory; | |
| 847 | } | |
| 848 | return aw.toOwnedSlice(); | |
| 849 | } | |
| 850 |
lib/compiler/maker/Step/Compile.zig created+1074
| ... | ... | @@ -0,0 +1,1074 @@ |
| 1 | /// Populated during the make phase when there is a long-lived compiler process. | |
| 2 | /// Managed by the build runner, not user build script. | |
| 3 | zig_process: ?*Step.ZigProcess, | |
| 4 | ||
| 5 | fn make(step: *Step, options: Step.MakeOptions) !void { | |
| 6 | const b = step.owner; | |
| 7 | const compile: *Compile = @fieldParentPtr("step", step); | |
| 8 | ||
| 9 | const zig_args = try getZigArgs(compile, false); | |
| 10 | ||
| 11 | const maybe_output_dir = step.evalZigProcess( | |
| 12 | zig_args, | |
| 13 | options.progress_node, | |
| 14 | (b.graph.incremental == true) and (options.watch or options.web_server != null), | |
| 15 | options.web_server, | |
| 16 | options.gpa, | |
| 17 | ) catch |err| switch (err) { | |
| 18 | error.NeedCompileErrorCheck => { | |
| 19 | assert(compile.expect_errors != null); | |
| 20 | try checkCompileErrors(compile); | |
| 21 | return; | |
| 22 | }, | |
| 23 | else => |e| return e, | |
| 24 | }; | |
| 25 | ||
| 26 | // Update generated files | |
| 27 | if (maybe_output_dir) |output_dir| { | |
| 28 | if (compile.emit_directory) |lp| { | |
| 29 | lp.path = b.fmt("{f}", .{output_dir}); | |
| 30 | } | |
| 31 | ||
| 32 | // zig fmt: off | |
| 33 | if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); | |
| 34 | if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); | |
| 35 | // hack for stage2_x86_64 + coff | |
| 36 | if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); | |
| 37 | if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); | |
| 38 | if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); | |
| 39 | if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); | |
| 40 | if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); | |
| 41 | if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); | |
| 42 | if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); | |
| 43 | // zig fmt: on | |
| 44 | } | |
| 45 | ||
| 46 | if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and | |
| 47 | compile.version != null and compile.generated_bin != null and | |
| 48 | std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) | |
| 49 | { | |
| 50 | try doAtomicSymLinks( | |
| 51 | step, | |
| 52 | compile.getEmittedBin().getPath2(b, step), | |
| 53 | compile.major_only_filename.?, | |
| 54 | compile.name_only_filename.?, | |
| 55 | ); | |
| 56 | } | |
| 57 | } | |
| 58 | ||
| 59 | fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { | |
| 60 | const step = &compile.step; | |
| 61 | const b = step.owner; | |
| 62 | const arena = b.allocator; | |
| 63 | ||
| 64 | var zig_args = std.array_list.Managed([]const u8).init(arena); | |
| 65 | defer zig_args.deinit(); | |
| 66 | ||
| 67 | try zig_args.append(b.graph.zig_exe); | |
| 68 | ||
| 69 | const cmd = switch (compile.kind) { | |
| 70 | .lib => "build-lib", | |
| 71 | .exe => "build-exe", | |
| 72 | .obj => "build-obj", | |
| 73 | .@"test" => "test", | |
| 74 | .test_obj => "test-obj", | |
| 75 | }; | |
| 76 | try zig_args.append(cmd); | |
| 77 | ||
| 78 | if (b.reference_trace) |some| { | |
| 79 | try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); | |
| 80 | } | |
| 81 | try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); | |
| 82 | ||
| 83 | try addFlag(&zig_args, "llvm", compile.use_llvm); | |
| 84 | try addFlag(&zig_args, "lld", compile.use_lld); | |
| 85 | try addFlag(&zig_args, "new-linker", compile.use_new_linker); | |
| 86 | ||
| 87 | if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { | |
| 88 | try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); | |
| 89 | } | |
| 90 | ||
| 91 | switch (compile.entry) { | |
| 92 | .default => {}, | |
| 93 | .disabled => try zig_args.append("-fno-entry"), | |
| 94 | .enabled => try zig_args.append("-fentry"), | |
| 95 | .symbol_name => |entry_name| { | |
| 96 | try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); | |
| 97 | }, | |
| 98 | } | |
| 99 | ||
| 100 | { | |
| 101 | var symbol_it = compile.force_undefined_symbols.keyIterator(); | |
| 102 | while (symbol_it.next()) |symbol_name| { | |
| 103 | try zig_args.append("--force_undefined"); | |
| 104 | try zig_args.append(symbol_name.*); | |
| 105 | } | |
| 106 | } | |
| 107 | ||
| 108 | if (compile.stack_size) |stack_size| { | |
| 109 | try zig_args.append("--stack"); | |
| 110 | try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); | |
| 111 | } | |
| 112 | ||
| 113 | if (fuzz) { | |
| 114 | try zig_args.append("-ffuzz"); | |
| 115 | } | |
| 116 | ||
| 117 | { | |
| 118 | // Stores system libraries that have already been seen for at least one | |
| 119 | // module, along with any arguments that need to be passed to the | |
| 120 | // compiler for each module individually. | |
| 121 | var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; | |
| 122 | var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; | |
| 123 | ||
| 124 | var prev_has_cflags = false; | |
| 125 | var prev_has_rcflags = false; | |
| 126 | var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; | |
| 127 | var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; | |
| 128 | // Track the number of positional arguments so that a nice error can be | |
| 129 | // emitted if there is nothing to link. | |
| 130 | var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); | |
| 131 | ||
| 132 | // Fully recursive iteration including dynamic libraries to detect | |
| 133 | // libc and libc++ linkage. | |
| 134 | for (compile.getCompileDependencies(true)) |some_compile| { | |
| 135 | for (some_compile.root_module.getGraph().modules) |mod| { | |
| 136 | if (mod.link_libc == true) compile.is_linking_libc = true; | |
| 137 | if (mod.link_libcpp == true) compile.is_linking_libcpp = true; | |
| 138 | } | |
| 139 | } | |
| 140 | ||
| 141 | var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); | |
| 142 | ||
| 143 | // For this loop, don't chase dynamic libraries because their link | |
| 144 | // objects are already linked. | |
| 145 | for (compile.getCompileDependencies(false)) |dep_compile| { | |
| 146 | for (dep_compile.root_module.getGraph().modules) |mod| { | |
| 147 | // While walking transitive dependencies, if a given link object is | |
| 148 | // already included in a library, it should not redundantly be | |
| 149 | // placed on the linker line of the dependee. | |
| 150 | const my_responsibility = dep_compile == compile; | |
| 151 | const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); | |
| 152 | ||
| 153 | // Inherit dependencies on darwin frameworks. | |
| 154 | if (!already_linked) { | |
| 155 | for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { | |
| 156 | try frameworks.put(arena, name, info); | |
| 157 | } | |
| 158 | } | |
| 159 | ||
| 160 | // Inherit dependencies on system libraries and static libraries. | |
| 161 | for (mod.link_objects.items) |link_object| { | |
| 162 | switch (link_object) { | |
| 163 | .static_path => |static_path| { | |
| 164 | if (my_responsibility) { | |
| 165 | try zig_args.append(static_path.getPath2(mod.owner, step)); | |
| 166 | total_linker_objects += 1; | |
| 167 | } | |
| 168 | }, | |
| 169 | .system_lib => |system_lib| { | |
| 170 | const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); | |
| 171 | if (system_lib_gop.found_existing) { | |
| 172 | try zig_args.appendSlice(system_lib_gop.value_ptr.*); | |
| 173 | continue; | |
| 174 | } else { | |
| 175 | system_lib_gop.value_ptr.* = &.{}; | |
| 176 | } | |
| 177 | ||
| 178 | if (already_linked) | |
| 179 | continue; | |
| 180 | ||
| 181 | if ((system_lib.search_strategy != prev_search_strategy or | |
| 182 | system_lib.preferred_link_mode != prev_preferred_link_mode) and | |
| 183 | compile.linkage != .static) | |
| 184 | { | |
| 185 | switch (system_lib.search_strategy) { | |
| 186 | .no_fallback => switch (system_lib.preferred_link_mode) { | |
| 187 | .dynamic => try zig_args.append("-search_dylibs_only"), | |
| 188 | .static => try zig_args.append("-search_static_only"), | |
| 189 | }, | |
| 190 | .paths_first => switch (system_lib.preferred_link_mode) { | |
| 191 | .dynamic => try zig_args.append("-search_paths_first"), | |
| 192 | .static => try zig_args.append("-search_paths_first_static"), | |
| 193 | }, | |
| 194 | .mode_first => switch (system_lib.preferred_link_mode) { | |
| 195 | .dynamic => try zig_args.append("-search_dylibs_first"), | |
| 196 | .static => try zig_args.append("-search_static_first"), | |
| 197 | }, | |
| 198 | } | |
| 199 | prev_search_strategy = system_lib.search_strategy; | |
| 200 | prev_preferred_link_mode = system_lib.preferred_link_mode; | |
| 201 | } | |
| 202 | ||
| 203 | const prefix: []const u8 = prefix: { | |
| 204 | if (system_lib.needed) break :prefix "-needed-l"; | |
| 205 | if (system_lib.weak) break :prefix "-weak-l"; | |
| 206 | break :prefix "-l"; | |
| 207 | }; | |
| 208 | switch (system_lib.use_pkg_config) { | |
| 209 | .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), | |
| 210 | .yes, .force => { | |
| 211 | if (compile.runPkgConfig(system_lib.name)) |result| { | |
| 212 | try zig_args.appendSlice(result.cflags); | |
| 213 | try zig_args.appendSlice(result.libs); | |
| 214 | try seen_system_libs.put(arena, system_lib.name, result.cflags); | |
| 215 | } else |err| switch (err) { | |
| 216 | error.PkgConfigInvalidOutput, | |
| 217 | error.PkgConfigCrashed, | |
| 218 | error.PkgConfigFailed, | |
| 219 | error.PkgConfigNotInstalled, | |
| 220 | error.PackageNotFound, | |
| 221 | => switch (system_lib.use_pkg_config) { | |
| 222 | .yes => { | |
| 223 | // pkg-config failed, so fall back to linking the library | |
| 224 | // by name directly. | |
| 225 | try zig_args.append(b.fmt("{s}{s}", .{ | |
| 226 | prefix, | |
| 227 | system_lib.name, | |
| 228 | })); | |
| 229 | }, | |
| 230 | .force => { | |
| 231 | panic("pkg-config failed for library {s}", .{system_lib.name}); | |
| 232 | }, | |
| 233 | .no => unreachable, | |
| 234 | }, | |
| 235 | ||
| 236 | else => |e| return e, | |
| 237 | } | |
| 238 | }, | |
| 239 | } | |
| 240 | }, | |
| 241 | .other_step => |other| { | |
| 242 | switch (other.kind) { | |
| 243 | .exe => return step.fail("cannot link with an executable build artifact", .{}), | |
| 244 | .@"test" => return step.fail("cannot link with a test", .{}), | |
| 245 | .obj, .test_obj => { | |
| 246 | const included_in_lib_or_obj = !my_responsibility and | |
| 247 | (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); | |
| 248 | if (!already_linked and !included_in_lib_or_obj) { | |
| 249 | try zig_args.append(other.getEmittedBin().getPath2(b, step)); | |
| 250 | total_linker_objects += 1; | |
| 251 | } | |
| 252 | }, | |
| 253 | .lib => l: { | |
| 254 | const other_produces_implib = other.producesImplib(); | |
| 255 | const other_is_static = other_produces_implib or other.isStaticLibrary(); | |
| 256 | ||
| 257 | if (compile.isStaticLibrary() and other_is_static) { | |
| 258 | // Avoid putting a static library inside a static library. | |
| 259 | break :l; | |
| 260 | } | |
| 261 | ||
| 262 | // For DLLs, we must link against the implib. | |
| 263 | // For everything else, we directly link | |
| 264 | // against the library file. | |
| 265 | const full_path_lib = if (other_produces_implib) | |
| 266 | try other.getGeneratedFilePath("generated_implib", &compile.step) | |
| 267 | else | |
| 268 | try other.getGeneratedFilePath("generated_bin", &compile.step); | |
| 269 | ||
| 270 | try zig_args.append(full_path_lib); | |
| 271 | total_linker_objects += 1; | |
| 272 | ||
| 273 | if (other.linkage == .dynamic and | |
| 274 | compile.rootModuleTarget().os.tag != .windows) | |
| 275 | { | |
| 276 | if (fs.path.dirname(full_path_lib)) |dirname| { | |
| 277 | try zig_args.append("-rpath"); | |
| 278 | try zig_args.append(dirname); | |
| 279 | } | |
| 280 | } | |
| 281 | }, | |
| 282 | } | |
| 283 | }, | |
| 284 | .assembly_file => |asm_file| l: { | |
| 285 | if (!my_responsibility) break :l; | |
| 286 | ||
| 287 | if (prev_has_cflags) { | |
| 288 | try zig_args.append("-cflags"); | |
| 289 | try zig_args.append("--"); | |
| 290 | prev_has_cflags = false; | |
| 291 | } | |
| 292 | try zig_args.append(asm_file.getPath2(mod.owner, step)); | |
| 293 | total_linker_objects += 1; | |
| 294 | }, | |
| 295 | ||
| 296 | .c_source_file => |c_source_file| l: { | |
| 297 | if (!my_responsibility) break :l; | |
| 298 | ||
| 299 | if (prev_has_cflags or c_source_file.flags.len != 0) { | |
| 300 | try zig_args.append("-cflags"); | |
| 301 | for (c_source_file.flags) |arg| { | |
| 302 | try zig_args.append(arg); | |
| 303 | } | |
| 304 | try zig_args.append("--"); | |
| 305 | } | |
| 306 | prev_has_cflags = (c_source_file.flags.len != 0); | |
| 307 | ||
| 308 | if (c_source_file.language) |lang| { | |
| 309 | try zig_args.append("-x"); | |
| 310 | try zig_args.append(lang.internalIdentifier()); | |
| 311 | } | |
| 312 | ||
| 313 | try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); | |
| 314 | ||
| 315 | if (c_source_file.language != null) { | |
| 316 | try zig_args.append("-x"); | |
| 317 | try zig_args.append("none"); | |
| 318 | } | |
| 319 | total_linker_objects += 1; | |
| 320 | }, | |
| 321 | ||
| 322 | .c_source_files => |c_source_files| l: { | |
| 323 | if (!my_responsibility) break :l; | |
| 324 | ||
| 325 | if (prev_has_cflags or c_source_files.flags.len != 0) { | |
| 326 | try zig_args.append("-cflags"); | |
| 327 | for (c_source_files.flags) |arg| { | |
| 328 | try zig_args.append(arg); | |
| 329 | } | |
| 330 | try zig_args.append("--"); | |
| 331 | } | |
| 332 | prev_has_cflags = (c_source_files.flags.len != 0); | |
| 333 | ||
| 334 | if (c_source_files.language) |lang| { | |
| 335 | try zig_args.append("-x"); | |
| 336 | try zig_args.append(lang.internalIdentifier()); | |
| 337 | } | |
| 338 | ||
| 339 | const root_path = c_source_files.root.getPath2(mod.owner, step); | |
| 340 | for (c_source_files.files) |file| { | |
| 341 | try zig_args.append(b.pathJoin(&.{ root_path, file })); | |
| 342 | } | |
| 343 | ||
| 344 | if (c_source_files.language != null) { | |
| 345 | try zig_args.append("-x"); | |
| 346 | try zig_args.append("none"); | |
| 347 | } | |
| 348 | ||
| 349 | total_linker_objects += c_source_files.files.len; | |
| 350 | }, | |
| 351 | ||
| 352 | .win32_resource_file => |rc_source_file| l: { | |
| 353 | if (!my_responsibility) break :l; | |
| 354 | ||
| 355 | if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { | |
| 356 | if (prev_has_rcflags) { | |
| 357 | try zig_args.append("-rcflags"); | |
| 358 | try zig_args.append("--"); | |
| 359 | prev_has_rcflags = false; | |
| 360 | } | |
| 361 | } else { | |
| 362 | try zig_args.append("-rcflags"); | |
| 363 | for (rc_source_file.flags) |arg| { | |
| 364 | try zig_args.append(arg); | |
| 365 | } | |
| 366 | for (rc_source_file.include_paths) |include_path| { | |
| 367 | try zig_args.append("/I"); | |
| 368 | try zig_args.append(include_path.getPath2(mod.owner, step)); | |
| 369 | } | |
| 370 | try zig_args.append("--"); | |
| 371 | prev_has_rcflags = true; | |
| 372 | } | |
| 373 | try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); | |
| 374 | total_linker_objects += 1; | |
| 375 | }, | |
| 376 | } | |
| 377 | } | |
| 378 | ||
| 379 | // We need to emit the --mod argument here so that the above link objects | |
| 380 | // have the correct parent module, but only if the module is part of | |
| 381 | // this compilation. | |
| 382 | if (!my_responsibility) continue; | |
| 383 | if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { | |
| 384 | const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; | |
| 385 | try mod.appendZigProcessFlags(&zig_args, step); | |
| 386 | ||
| 387 | // --dep arguments | |
| 388 | try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); | |
| 389 | for (mod.import_table.keys(), mod.import_table.values()) |name, import| { | |
| 390 | const import_index = cli_named_modules.modules.getIndex(import).?; | |
| 391 | const import_cli_name = cli_named_modules.names.keys()[import_index]; | |
| 392 | zig_args.appendAssumeCapacity("--dep"); | |
| 393 | if (std.mem.eql(u8, import_cli_name, name)) { | |
| 394 | zig_args.appendAssumeCapacity(import_cli_name); | |
| 395 | } else { | |
| 396 | zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); | |
| 397 | } | |
| 398 | } | |
| 399 | ||
| 400 | // When the CLI sees a -M argument, it determines whether it | |
| 401 | // implies the existence of a Zig compilation unit based on | |
| 402 | // whether there is a root source file. If there is no root | |
| 403 | // source file, then this is not a zig compilation unit - it is | |
| 404 | // perhaps a set of linker objects, or C source files instead. | |
| 405 | // Linker objects are added to the CLI globally, while C source | |
| 406 | // files must have a module parent. | |
| 407 | if (mod.root_source_file) |lp| { | |
| 408 | const src = lp.getPath2(mod.owner, step); | |
| 409 | try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); | |
| 410 | } else if (moduleNeedsCliArg(mod)) { | |
| 411 | try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); | |
| 412 | } | |
| 413 | } | |
| 414 | } | |
| 415 | } | |
| 416 | ||
| 417 | if (total_linker_objects == 0) { | |
| 418 | return step.fail("the linker needs one or more objects to link", .{}); | |
| 419 | } | |
| 420 | ||
| 421 | for (frameworks.keys(), frameworks.values()) |name, info| { | |
| 422 | if (info.needed) { | |
| 423 | try zig_args.append("-needed_framework"); | |
| 424 | } else if (info.weak) { | |
| 425 | try zig_args.append("-weak_framework"); | |
| 426 | } else { | |
| 427 | try zig_args.append("-framework"); | |
| 428 | } | |
| 429 | try zig_args.append(name); | |
| 430 | } | |
| 431 | ||
| 432 | if (compile.is_linking_libcpp) { | |
| 433 | try zig_args.append("-lc++"); | |
| 434 | } | |
| 435 | ||
| 436 | if (compile.is_linking_libc) { | |
| 437 | try zig_args.append("-lc"); | |
| 438 | } | |
| 439 | } | |
| 440 | ||
| 441 | if (compile.win32_manifest) |manifest_file| { | |
| 442 | try zig_args.append(manifest_file.getPath2(b, step)); | |
| 443 | } | |
| 444 | ||
| 445 | if (compile.win32_module_definition) |module_file| { | |
| 446 | try zig_args.append(module_file.getPath2(b, step)); | |
| 447 | } | |
| 448 | ||
| 449 | if (compile.image_base) |image_base| { | |
| 450 | try zig_args.append("--image-base"); | |
| 451 | try zig_args.append(b.fmt("0x{x}", .{image_base})); | |
| 452 | } | |
| 453 | ||
| 454 | for (compile.filters) |filter| { | |
| 455 | try zig_args.append("--test-filter"); | |
| 456 | try zig_args.append(filter); | |
| 457 | } | |
| 458 | ||
| 459 | if (compile.test_runner) |test_runner| { | |
| 460 | try zig_args.append("--test-runner"); | |
| 461 | try zig_args.append(test_runner.path.getPath2(b, step)); | |
| 462 | } | |
| 463 | ||
| 464 | for (b.debug_log_scopes) |log_scope| { | |
| 465 | try zig_args.append("--debug-log"); | |
| 466 | try zig_args.append(log_scope); | |
| 467 | } | |
| 468 | ||
| 469 | if (b.debug_compile_errors) { | |
| 470 | try zig_args.append("--debug-compile-errors"); | |
| 471 | } | |
| 472 | ||
| 473 | if (b.debug_incremental) { | |
| 474 | try zig_args.append("--debug-incremental"); | |
| 475 | } | |
| 476 | ||
| 477 | if (b.verbose_air) try zig_args.append("--verbose-air"); | |
| 478 | if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); | |
| 479 | if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); | |
| 480 | if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); | |
| 481 | if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); | |
| 482 | if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); | |
| 483 | if (b.graph.time_report) try zig_args.append("--time-report"); | |
| 484 | ||
| 485 | if (compile.generated_asm != null) try zig_args.append("-femit-asm"); | |
| 486 | if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); | |
| 487 | if (compile.generated_docs != null) try zig_args.append("-femit-docs"); | |
| 488 | if (compile.generated_implib != null) try zig_args.append("-femit-implib"); | |
| 489 | if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); | |
| 490 | if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); | |
| 491 | if (compile.generated_h != null) try zig_args.append("-femit-h"); | |
| 492 | ||
| 493 | try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); | |
| 494 | ||
| 495 | switch (compile.compress_debug_sections) { | |
| 496 | .none => {}, | |
| 497 | .zlib => try zig_args.append("--compress-debug-sections=zlib"), | |
| 498 | .zstd => try zig_args.append("--compress-debug-sections=zstd"), | |
| 499 | } | |
| 500 | ||
| 501 | if (compile.link_eh_frame_hdr) { | |
| 502 | try zig_args.append("--eh-frame-hdr"); | |
| 503 | } | |
| 504 | if (compile.link_emit_relocs) { | |
| 505 | try zig_args.append("--emit-relocs"); | |
| 506 | } | |
| 507 | if (compile.link_function_sections) { | |
| 508 | try zig_args.append("-ffunction-sections"); | |
| 509 | } | |
| 510 | if (compile.link_data_sections) { | |
| 511 | try zig_args.append("-fdata-sections"); | |
| 512 | } | |
| 513 | if (compile.link_gc_sections) |x| { | |
| 514 | try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); | |
| 515 | } | |
| 516 | if (!compile.linker_dynamicbase) { | |
| 517 | try zig_args.append("--no-dynamicbase"); | |
| 518 | } | |
| 519 | if (compile.linker_allow_shlib_undefined) |x| { | |
| 520 | try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); | |
| 521 | } | |
| 522 | if (compile.link_z_notext) { | |
| 523 | try zig_args.append("-z"); | |
| 524 | try zig_args.append("notext"); | |
| 525 | } | |
| 526 | if (!compile.link_z_relro) { | |
| 527 | try zig_args.append("-z"); | |
| 528 | try zig_args.append("norelro"); | |
| 529 | } | |
| 530 | if (compile.link_z_lazy) { | |
| 531 | try zig_args.append("-z"); | |
| 532 | try zig_args.append("lazy"); | |
| 533 | } | |
| 534 | if (compile.link_z_common_page_size) |size| { | |
| 535 | try zig_args.append("-z"); | |
| 536 | try zig_args.append(b.fmt("common-page-size={d}", .{size})); | |
| 537 | } | |
| 538 | if (compile.link_z_max_page_size) |size| { | |
| 539 | try zig_args.append("-z"); | |
| 540 | try zig_args.append(b.fmt("max-page-size={d}", .{size})); | |
| 541 | } | |
| 542 | if (compile.link_z_defs) { | |
| 543 | try zig_args.append("-z"); | |
| 544 | try zig_args.append("defs"); | |
| 545 | } | |
| 546 | ||
| 547 | if (compile.libc_file) |libc_file| { | |
| 548 | try zig_args.append("--libc"); | |
| 549 | try zig_args.append(libc_file.getPath2(b, step)); | |
| 550 | } else if (b.libc_file) |libc_file| { | |
| 551 | try zig_args.append("--libc"); | |
| 552 | try zig_args.append(libc_file); | |
| 553 | } | |
| 554 | ||
| 555 | try zig_args.append("--cache-dir"); | |
| 556 | try zig_args.append(b.cache_root.path orelse "."); | |
| 557 | ||
| 558 | try zig_args.append("--global-cache-dir"); | |
| 559 | try zig_args.append(b.graph.global_cache_root.path orelse "."); | |
| 560 | ||
| 561 | if (b.graph.debug_compiler_runtime_libs) |mode| | |
| 562 | try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); | |
| 563 | ||
| 564 | try zig_args.append("--name"); | |
| 565 | try zig_args.append(compile.name); | |
| 566 | ||
| 567 | if (compile.linkage) |some| switch (some) { | |
| 568 | .dynamic => try zig_args.append("-dynamic"), | |
| 569 | .static => try zig_args.append("-static"), | |
| 570 | }; | |
| 571 | if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { | |
| 572 | if (compile.version) |version| { | |
| 573 | try zig_args.append("--version"); | |
| 574 | try zig_args.append(b.fmt("{f}", .{version})); | |
| 575 | } | |
| 576 | ||
| 577 | if (compile.rootModuleTarget().os.tag.isDarwin()) { | |
| 578 | const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ | |
| 579 | compile.rootModuleTarget().libPrefix(), | |
| 580 | compile.name, | |
| 581 | compile.rootModuleTarget().dynamicLibSuffix(), | |
| 582 | }); | |
| 583 | try zig_args.append("-install_name"); | |
| 584 | try zig_args.append(install_name); | |
| 585 | } | |
| 586 | } | |
| 587 | ||
| 588 | if (compile.entitlements) |entitlements| { | |
| 589 | try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); | |
| 590 | } | |
| 591 | if (compile.pagezero_size) |pagezero_size| { | |
| 592 | const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); | |
| 593 | try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); | |
| 594 | } | |
| 595 | if (compile.headerpad_size) |headerpad_size| { | |
| 596 | const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); | |
| 597 | try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); | |
| 598 | } | |
| 599 | if (compile.headerpad_max_install_names) { | |
| 600 | try zig_args.append("-headerpad_max_install_names"); | |
| 601 | } | |
| 602 | if (compile.dead_strip_dylibs) { | |
| 603 | try zig_args.append("-dead_strip_dylibs"); | |
| 604 | } | |
| 605 | if (compile.force_load_objc) { | |
| 606 | try zig_args.append("-ObjC"); | |
| 607 | } | |
| 608 | if (compile.discard_local_symbols) { | |
| 609 | try zig_args.append("--discard-all"); | |
| 610 | } | |
| 611 | ||
| 612 | try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); | |
| 613 | try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); | |
| 614 | try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); | |
| 615 | if (compile.rdynamic) { | |
| 616 | try zig_args.append("-rdynamic"); | |
| 617 | } | |
| 618 | if (compile.import_memory) { | |
| 619 | try zig_args.append("--import-memory"); | |
| 620 | } | |
| 621 | if (compile.export_memory) { | |
| 622 | try zig_args.append("--export-memory"); | |
| 623 | } | |
| 624 | if (compile.import_symbols) { | |
| 625 | try zig_args.append("--import-symbols"); | |
| 626 | } | |
| 627 | if (compile.import_table) { | |
| 628 | try zig_args.append("--import-table"); | |
| 629 | } | |
| 630 | if (compile.export_table) { | |
| 631 | try zig_args.append("--export-table"); | |
| 632 | } | |
| 633 | if (compile.initial_memory) |initial_memory| { | |
| 634 | try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); | |
| 635 | } | |
| 636 | if (compile.max_memory) |max_memory| { | |
| 637 | try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); | |
| 638 | } | |
| 639 | if (compile.shared_memory) { | |
| 640 | try zig_args.append("--shared-memory"); | |
| 641 | } | |
| 642 | if (compile.global_base) |global_base| { | |
| 643 | try zig_args.append(b.fmt("--global-base={d}", .{global_base})); | |
| 644 | } | |
| 645 | ||
| 646 | if (compile.wasi_exec_model) |model| { | |
| 647 | try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); | |
| 648 | } | |
| 649 | if (compile.linker_script) |linker_script| { | |
| 650 | try zig_args.append("--script"); | |
| 651 | try zig_args.append(linker_script.getPath2(b, step)); | |
| 652 | } | |
| 653 | ||
| 654 | if (compile.version_script) |version_script| { | |
| 655 | try zig_args.append("--version-script"); | |
| 656 | try zig_args.append(version_script.getPath2(b, step)); | |
| 657 | } | |
| 658 | if (compile.linker_allow_undefined_version) |x| { | |
| 659 | try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); | |
| 660 | } | |
| 661 | ||
| 662 | if (compile.linker_enable_new_dtags) |enabled| { | |
| 663 | try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); | |
| 664 | } | |
| 665 | ||
| 666 | if (compile.kind == .@"test") { | |
| 667 | if (compile.exec_cmd_args) |exec_cmd_args| { | |
| 668 | for (exec_cmd_args) |cmd_arg| { | |
| 669 | if (cmd_arg) |arg| { | |
| 670 | try zig_args.append("--test-cmd"); | |
| 671 | try zig_args.append(arg); | |
| 672 | } else { | |
| 673 | try zig_args.append("--test-cmd-bin"); | |
| 674 | } | |
| 675 | } | |
| 676 | } | |
| 677 | } | |
| 678 | ||
| 679 | if (b.sysroot) |sysroot| { | |
| 680 | try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); | |
| 681 | } | |
| 682 | ||
| 683 | // -I and -L arguments that appear after the last --mod argument apply to all modules. | |
| 684 | const cwd: Io.Dir = .cwd(); | |
| 685 | const io = b.graph.io; | |
| 686 | ||
| 687 | for (b.search_prefixes.items) |search_prefix| { | |
| 688 | var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { | |
| 689 | return step.fail("unable to open prefix directory '{s}': {s}", .{ | |
| 690 | search_prefix, @errorName(err), | |
| 691 | }); | |
| 692 | }; | |
| 693 | defer prefix_dir.close(io); | |
| 694 | ||
| 695 | // Avoid passing -L and -I flags for nonexistent directories. | |
| 696 | // This prevents a warning, that should probably be upgraded to an error in Zig's | |
| 697 | // CLI parsing code, when the linker sees an -L directory that does not exist. | |
| 698 | ||
| 699 | if (prefix_dir.access(io, "lib", .{})) |_| { | |
| 700 | try zig_args.appendSlice(&.{ | |
| 701 | "-L", b.pathJoin(&.{ search_prefix, "lib" }), | |
| 702 | }); | |
| 703 | } else |err| switch (err) { | |
| 704 | error.FileNotFound => {}, | |
| 705 | else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ | |
| 706 | search_prefix, @errorName(e), | |
| 707 | }), | |
| 708 | } | |
| 709 | ||
| 710 | if (prefix_dir.access(io, "include", .{})) |_| { | |
| 711 | try zig_args.appendSlice(&.{ | |
| 712 | "-I", b.pathJoin(&.{ search_prefix, "include" }), | |
| 713 | }); | |
| 714 | } else |err| switch (err) { | |
| 715 | error.FileNotFound => {}, | |
| 716 | else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ | |
| 717 | search_prefix, @errorName(e), | |
| 718 | }), | |
| 719 | } | |
| 720 | } | |
| 721 | ||
| 722 | if (compile.rc_includes != .any) { | |
| 723 | try zig_args.append("-rcincludes"); | |
| 724 | try zig_args.append(@tagName(compile.rc_includes)); | |
| 725 | } | |
| 726 | ||
| 727 | try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); | |
| 728 | ||
| 729 | if (compile.build_id orelse b.build_id) |build_id| { | |
| 730 | try zig_args.append(switch (build_id) { | |
| 731 | .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), | |
| 732 | .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), | |
| 733 | }); | |
| 734 | } | |
| 735 | ||
| 736 | const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| | |
| 737 | dir.getPath2(b, step) | |
| 738 | else if (b.graph.zig_lib_directory.path) |_| | |
| 739 | b.fmt("{f}", .{b.graph.zig_lib_directory}) | |
| 740 | else | |
| 741 | null; | |
| 742 | ||
| 743 | if (opt_zig_lib_dir) |zig_lib_dir| { | |
| 744 | try zig_args.append("--zig-lib-dir"); | |
| 745 | try zig_args.append(zig_lib_dir); | |
| 746 | } | |
| 747 | ||
| 748 | try addFlag(&zig_args, "PIE", compile.pie); | |
| 749 | ||
| 750 | if (compile.lto) |lto| { | |
| 751 | try zig_args.append(switch (lto) { | |
| 752 | .full => "-flto=full", | |
| 753 | .thin => "-flto=thin", | |
| 754 | .none => "-fno-lto", | |
| 755 | }); | |
| 756 | } | |
| 757 | ||
| 758 | try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); | |
| 759 | ||
| 760 | if (compile.subsystem) |subsystem| { | |
| 761 | try zig_args.append("--subsystem"); | |
| 762 | try zig_args.append(@tagName(subsystem)); | |
| 763 | } | |
| 764 | ||
| 765 | if (compile.mingw_unicode_entry_point) { | |
| 766 | try zig_args.append("-municode"); | |
| 767 | } | |
| 768 | ||
| 769 | if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ | |
| 770 | "--error-limit", b.fmt("{d}", .{err_limit}), | |
| 771 | }); | |
| 772 | ||
| 773 | try addFlag(&zig_args, "incremental", b.graph.incremental); | |
| 774 | ||
| 775 | try zig_args.append("--listen=-"); | |
| 776 | ||
| 777 | // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux | |
| 778 | // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and | |
| 779 | // pass that to zig, e.g. via 'zig build-lib @args.rsp' | |
| 780 | // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html | |
| 781 | var args_length: usize = 0; | |
| 782 | for (zig_args.items) |arg| { | |
| 783 | args_length += arg.len + 1; // +1 to account for null terminator | |
| 784 | } | |
| 785 | if (args_length >= 30 * 1024) { | |
| 786 | try b.cache_root.handle.createDirPath(io, "args"); | |
| 787 | ||
| 788 | const args_to_escape = zig_args.items[2..]; | |
| 789 | var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); | |
| 790 | arg_blk: for (args_to_escape) |arg| { | |
| 791 | for (arg, 0..) |c, arg_idx| { | |
| 792 | if (c == '\\' or c == '"') { | |
| 793 | // Slow path for arguments that need to be escaped. We'll need to allocate and copy | |
| 794 | var escaped: std.ArrayList(u8) = .empty; | |
| 795 | try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1); | |
| 796 | try escaped.appendSlice(arena, arg[0..arg_idx]); | |
| 797 | for (arg[arg_idx..]) |to_escape| { | |
| 798 | if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\'); | |
| 799 | try escaped.append(arena, to_escape); | |
| 800 | } | |
| 801 | escaped_args.appendAssumeCapacity(escaped.items); | |
| 802 | continue :arg_blk; | |
| 803 | } | |
| 804 | } | |
| 805 | escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument | |
| 806 | } | |
| 807 | ||
| 808 | // Write the args to zig-cache/args/<SHA256 hash of args> to avoid conflicts with | |
| 809 | // other zig build commands running in parallel. | |
| 810 | const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); | |
| 811 | const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); | |
| 812 | ||
| 813 | var args_hash: [Sha256.digest_length]u8 = undefined; | |
| 814 | Sha256.hash(args, &args_hash, .{}); | |
| 815 | var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; | |
| 816 | _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); | |
| 817 | ||
| 818 | const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; | |
| 819 | if (b.cache_root.handle.access(io, args_file, .{})) |_| { | |
| 820 | // The args file is already present from a previous run. | |
| 821 | } else |err| switch (err) { | |
| 822 | error.FileNotFound => { | |
| 823 | var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ | |
| 824 | .replace = false, | |
| 825 | .make_path = true, | |
| 826 | }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ | |
| 827 | b.cache_root, args_file, e, | |
| 828 | }); | |
| 829 | defer af.deinit(io); | |
| 830 | ||
| 831 | af.file.writeStreamingAll(io, args) catch |e| { | |
| 832 | return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ | |
| 833 | b.cache_root, args_file, e, | |
| 834 | }); | |
| 835 | }; | |
| 836 | // Note we can't clean up this file, not even after build | |
| 837 | // success, because that might interfere with another build | |
| 838 | // process that needs the same file. | |
| 839 | af.link(io) catch |e| switch (e) { | |
| 840 | error.PathAlreadyExists => { | |
| 841 | // The args file was created by another concurrent build process. | |
| 842 | }, | |
| 843 | else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ | |
| 844 | b.cache_root, args_file, other_err, | |
| 845 | }), | |
| 846 | }; | |
| 847 | }, | |
| 848 | else => |other_err| return other_err, | |
| 849 | } | |
| 850 | ||
| 851 | const resolved_args_file = try mem.concat(arena, u8, &.{ | |
| 852 | "@", | |
| 853 | try b.cache_root.join(arena, &.{args_file}), | |
| 854 | }); | |
| 855 | ||
| 856 | zig_args.shrinkRetainingCapacity(2); | |
| 857 | try zig_args.append(resolved_args_file); | |
| 858 | } | |
| 859 | ||
| 860 | return try zig_args.toOwnedSlice(); | |
| 861 | } | |
| 862 | ||
| 863 | pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { | |
| 864 | c.step.result_error_msgs.clearRetainingCapacity(); | |
| 865 | c.step.result_stderr = ""; | |
| 866 | ||
| 867 | c.step.result_error_bundle.deinit(gpa); | |
| 868 | c.step.result_error_bundle = std.zig.ErrorBundle.empty; | |
| 869 | ||
| 870 | if (c.step.result_failed_command) |cmd| { | |
| 871 | gpa.free(cmd); | |
| 872 | c.step.result_failed_command = null; | |
| 873 | } | |
| 874 | ||
| 875 | const zig_args = try getZigArgs(c, true); | |
| 876 | const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); | |
| 877 | return maybe_output_bin_path.?; | |
| 878 | } | |
| 879 | ||
| 880 | pub fn doAtomicSymLinks( | |
| 881 | step: *Step, | |
| 882 | output_path: []const u8, | |
| 883 | filename_major_only: []const u8, | |
| 884 | filename_name_only: []const u8, | |
| 885 | ) !void { | |
| 886 | const b = step.owner; | |
| 887 | const io = b.graph.io; | |
| 888 | const out_dir = fs.path.dirname(output_path) orelse "."; | |
| 889 | const out_basename = fs.path.basename(output_path); | |
| 890 | // sym link for libfoo.so.1 to libfoo.so.1.2.3 | |
| 891 | const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); | |
| 892 | const cwd: Io.Dir = .cwd(); | |
| 893 | cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { | |
| 894 | return step.fail("unable to symlink {s} -> {s}: {s}", .{ | |
| 895 | major_only_path, out_basename, @errorName(err), | |
| 896 | }); | |
| 897 | }; | |
| 898 | // sym link for libfoo.so to libfoo.so.1 | |
| 899 | const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); | |
| 900 | cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { | |
| 901 | return step.fail("Unable to symlink {s} -> {s}: {s}", .{ | |
| 902 | name_only_path, filename_major_only, @errorName(err), | |
| 903 | }); | |
| 904 | }; | |
| 905 | } | |
| 906 | ||
| 907 | fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { | |
| 908 | const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; | |
| 909 | const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); | |
| 910 | var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); | |
| 911 | errdefer list.deinit(); | |
| 912 | var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); | |
| 913 | while (line_it.next()) |line| { | |
| 914 | if (mem.trim(u8, line, " \t").len == 0) continue; | |
| 915 | var tok_it = mem.tokenizeAny(u8, line, " \t"); | |
| 916 | try list.append(PkgConfigPkg{ | |
| 917 | .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, | |
| 918 | .desc = tok_it.rest(), | |
| 919 | }); | |
| 920 | } | |
| 921 | return list.toOwnedSlice(); | |
| 922 | } | |
| 923 | ||
| 924 | fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { | |
| 925 | if (b.pkg_config_pkg_list) |res| { | |
| 926 | return res; | |
| 927 | } | |
| 928 | var code: u8 = undefined; | |
| 929 | if (execPkgConfigList(b, &code)) |list| { | |
| 930 | b.pkg_config_pkg_list = list; | |
| 931 | return list; | |
| 932 | } else |err| { | |
| 933 | const result = switch (err) { | |
| 934 | error.ProcessTerminated => error.PkgConfigCrashed, | |
| 935 | error.ExecNotSupported => error.PkgConfigFailed, | |
| 936 | error.ExitCodeFailure => error.PkgConfigFailed, | |
| 937 | error.FileNotFound => error.PkgConfigNotInstalled, | |
| 938 | error.InvalidName => error.PkgConfigNotInstalled, | |
| 939 | error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, | |
| 940 | else => return err, | |
| 941 | }; | |
| 942 | b.pkg_config_pkg_list = result; | |
| 943 | return result; | |
| 944 | } | |
| 945 | } | |
| 946 | ||
| 947 | fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { | |
| 948 | const cond = opt orelse return; | |
| 949 | try args.ensureUnusedCapacity(1); | |
| 950 | if (cond) { | |
| 951 | args.appendAssumeCapacity("-f" ++ name); | |
| 952 | } else { | |
| 953 | args.appendAssumeCapacity("-fno-" ++ name); | |
| 954 | } | |
| 955 | } | |
| 956 | ||
| 957 | const PkgConfigResult = struct { | |
| 958 | cflags: []const []const u8, | |
| 959 | libs: []const []const u8, | |
| 960 | }; | |
| 961 | ||
| 962 | /// Run pkg-config for the given library name and parse the output, returning the arguments | |
| 963 | /// that should be passed to zig to link the given library. | |
| 964 | fn runPkgConfig(compile: *Compile, lib_name: []const u8) !PkgConfigResult { | |
| 965 | const wl_rpath_prefix = "-Wl,-rpath,"; | |
| 966 | ||
| 967 | const b = compile.step.owner; | |
| 968 | const arena = b.allocator; | |
| 969 | const pkg_name = match: { | |
| 970 | // First we have to map the library name to pkg config name. Unfortunately, | |
| 971 | // there are several examples where this is not straightforward: | |
| 972 | // -lSDL2 -> pkg-config sdl2 | |
| 973 | // -lgdk-3 -> pkg-config gdk-3.0 | |
| 974 | // -latk-1.0 -> pkg-config atk | |
| 975 | // -lpulse -> pkg-config libpulse | |
| 976 | const pkgs = try getPkgConfigList(b); | |
| 977 | ||
| 978 | // Exact match means instant winner. | |
| 979 | for (pkgs) |pkg| { | |
| 980 | if (mem.eql(u8, pkg.name, lib_name)) { | |
| 981 | break :match pkg.name; | |
| 982 | } | |
| 983 | } | |
| 984 | ||
| 985 | // Next we'll try ignoring case. | |
| 986 | for (pkgs) |pkg| { | |
| 987 | if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { | |
| 988 | break :match pkg.name; | |
| 989 | } | |
| 990 | } | |
| 991 | ||
| 992 | // Prefixed "lib" or suffixed ".0". | |
| 993 | for (pkgs) |pkg| { | |
| 994 | if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { | |
| 995 | const prefix = pkg.name[0..pos]; | |
| 996 | const suffix = pkg.name[pos + lib_name.len ..]; | |
| 997 | if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; | |
| 998 | if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; | |
| 999 | break :match pkg.name; | |
| 1000 | } | |
| 1001 | } | |
| 1002 | ||
| 1003 | // Trimming "-1.0". | |
| 1004 | if (mem.endsWith(u8, lib_name, "-1.0")) { | |
| 1005 | const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; | |
| 1006 | for (pkgs) |pkg| { | |
| 1007 | if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { | |
| 1008 | break :match pkg.name; | |
| 1009 | } | |
| 1010 | } | |
| 1011 | } | |
| 1012 | ||
| 1013 | return error.PackageNotFound; | |
| 1014 | }; | |
| 1015 | ||
| 1016 | var code: u8 = undefined; | |
| 1017 | const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; | |
| 1018 | const stdout = if (b.runAllowFail(&[_][]const u8{ | |
| 1019 | pkg_config_exe, | |
| 1020 | pkg_name, | |
| 1021 | "--cflags", | |
| 1022 | "--libs", | |
| 1023 | }, &code, .ignore)) |stdout| stdout else |err| switch (err) { | |
| 1024 | error.ProcessTerminated => return error.PkgConfigCrashed, | |
| 1025 | error.ExecNotSupported => return error.PkgConfigFailed, | |
| 1026 | error.ExitCodeFailure => return error.PkgConfigFailed, | |
| 1027 | error.FileNotFound => return error.PkgConfigNotInstalled, | |
| 1028 | else => return err, | |
| 1029 | }; | |
| 1030 | ||
| 1031 | var zig_cflags: std.ArrayList([]const u8) = .empty; | |
| 1032 | defer zig_cflags.deinit(arena); | |
| 1033 | var zig_libs: std.ArrayList([]const u8) = .empty; | |
| 1034 | defer zig_libs.deinit(arena); | |
| 1035 | ||
| 1036 | var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); | |
| 1037 | while (arg_it.next()) |arg| { | |
| 1038 | if (mem.eql(u8, arg, "-I")) { | |
| 1039 | const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; | |
| 1040 | try zig_cflags.appendSlice(arena, &.{ "-I", dir }); | |
| 1041 | } else if (mem.startsWith(u8, arg, "-I")) { | |
| 1042 | try zig_cflags.append(arena, arg); | |
| 1043 | } else if (mem.eql(u8, arg, "-L")) { | |
| 1044 | const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; | |
| 1045 | try zig_libs.appendSlice(arena, &.{ "-L", dir }); | |
| 1046 | } else if (mem.startsWith(u8, arg, "-L")) { | |
| 1047 | try zig_libs.append(arena, arg); | |
| 1048 | } else if (mem.eql(u8, arg, "-l")) { | |
| 1049 | const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; | |
| 1050 | try zig_libs.appendSlice(arena, &.{ "-l", lib }); | |
| 1051 | } else if (mem.startsWith(u8, arg, "-l")) { | |
| 1052 | try zig_libs.append(arena, arg); | |
| 1053 | } else if (mem.eql(u8, arg, "-D")) { | |
| 1054 | const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; | |
| 1055 | try zig_cflags.appendSlice(arena, &.{ "-D", macro }); | |
| 1056 | } else if (mem.startsWith(u8, arg, "-D")) { | |
| 1057 | try zig_cflags.append(arena, arg); | |
| 1058 | } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { | |
| 1059 | try zig_cflags.appendSlice(arena, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); | |
| 1060 | } else if (b.debug_pkg_config) { | |
| 1061 | return compile.step.fail("unknown pkg-config flag '{s}'", .{arg}); | |
| 1062 | } | |
| 1063 | } | |
| 1064 | ||
| 1065 | try zig_cflags.shrinkToLen(arena); | |
| 1066 | try zig_libs.shrinkToLen(arena); | |
| 1067 | ||
| 1068 | return .{ | |
| 1069 | .cflags = zig_cflags.toOwnedSliceAssert(), | |
| 1070 | .libs = zig_libs.toOwnedSliceAssert(), | |
| 1071 | }; | |
| 1072 | } | |
| 1073 | ||
| 1074 |
lib/compiler/maker/Step/InstallArtifact.zig created+96
| ... | ... | @@ -0,0 +1,96 @@ |
| 1 | ||
| 2 | fn make(step: *Step, options: Step.MakeOptions) !void { | |
| 3 | _ = options; | |
| 4 | const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); | |
| 5 | const b = step.owner; | |
| 6 | const io = b.graph.io; | |
| 7 | ||
| 8 | var all_cached = true; | |
| 9 | ||
| 10 | if (install_artifact.dest_dir) |dest_dir| { | |
| 11 | const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); | |
| 12 | const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); | |
| 13 | all_cached = all_cached and p == .fresh; | |
| 14 | ||
| 15 | if (install_artifact.dylib_symlinks) |dls| { | |
| 16 | try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); | |
| 17 | } | |
| 18 | ||
| 19 | install_artifact.artifact.installed_path = full_dest_path; | |
| 20 | } | |
| 21 | ||
| 22 | if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { | |
| 23 | const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); | |
| 24 | const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); | |
| 25 | all_cached = all_cached and p == .fresh; | |
| 26 | } | |
| 27 | ||
| 28 | if (install_artifact.implib_dir) |implib_dir| { | |
| 29 | const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); | |
| 30 | const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); | |
| 31 | all_cached = all_cached and p == .fresh; | |
| 32 | } | |
| 33 | ||
| 34 | if (install_artifact.pdb_dir) |pdb_dir| { | |
| 35 | const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); | |
| 36 | const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); | |
| 37 | all_cached = all_cached and p == .fresh; | |
| 38 | } | |
| 39 | ||
| 40 | if (install_artifact.h_dir) |h_dir| { | |
| 41 | if (install_artifact.emitted_h) |emitted_h| { | |
| 42 | const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); | |
| 43 | const p = try step.installFile(emitted_h, full_h_path); | |
| 44 | all_cached = all_cached and p == .fresh; | |
| 45 | } | |
| 46 | ||
| 47 | for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { | |
| 48 | .file => |file| { | |
| 49 | const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); | |
| 50 | const p = try step.installFile(file.source, full_h_path); | |
| 51 | all_cached = all_cached and p == .fresh; | |
| 52 | }, | |
| 53 | .directory => |dir| { | |
| 54 | const src_dir_path = dir.source.getPath3(b, step); | |
| 55 | const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); | |
| 56 | ||
| 57 | var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { | |
| 58 | return step.fail("unable to open source directory '{f}': {s}", .{ | |
| 59 | src_dir_path, @errorName(err), | |
| 60 | }); | |
| 61 | }; | |
| 62 | defer src_dir.close(io); | |
| 63 | ||
| 64 | var it = try src_dir.walk(b.allocator); | |
| 65 | next_entry: while (try it.next(io)) |entry| { | |
| 66 | for (dir.options.exclude_extensions) |ext| { | |
| 67 | if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; | |
| 68 | } | |
| 69 | if (dir.options.include_extensions) |incs| { | |
| 70 | for (incs) |inc| { | |
| 71 | if (std.mem.endsWith(u8, entry.path, inc)) break; | |
| 72 | } else { | |
| 73 | continue :next_entry; | |
| 74 | } | |
| 75 | } | |
| 76 | ||
| 77 | const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); | |
| 78 | switch (entry.kind) { | |
| 79 | .directory => { | |
| 80 | try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); | |
| 81 | const p = try step.installDir(full_dest_path); | |
| 82 | all_cached = all_cached and p == .existed; | |
| 83 | }, | |
| 84 | .file => { | |
| 85 | const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); | |
| 86 | all_cached = all_cached and p == .fresh; | |
| 87 | }, | |
| 88 | else => continue, | |
| 89 | } | |
| 90 | } | |
| 91 | }, | |
| 92 | }; | |
| 93 | } | |
| 94 | ||
| 95 | step.result_cached = all_cached; | |
| 96 | } |
lib/compiler/maker/Step/Run.zig created+2127
| ... | ... | @@ -0,0 +1,2127 @@ |
| 1 | const Run = @This(); | |
| 2 | ||
| 3 | const builtin = @import("builtin"); | |
| 4 | ||
| 5 | const std = @import("std"); | |
| 6 | const Io = std.Io; | |
| 7 | const Dir = std.Io.Dir; | |
| 8 | const mem = std.mem; | |
| 9 | const process = std.process; | |
| 10 | const EnvMap = std.process.Environ.Map; | |
| 11 | const assert = std.debug.assert; | |
| 12 | const Cache = std.Build.Cache; | |
| 13 | const Path = std.Build.Cache.Path; | |
| 14 | ||
| 15 | const Step = @import("../Step.zig"); | |
| 16 | ||
| 17 | /// If this is a Zig unit test binary, this tracks the names of the unit | |
| 18 | /// tests that are also fuzz tests. Indexes cannot be used as they may | |
| 19 | /// change between reruns. | |
| 20 | fuzz_tests: std.ArrayList([]const u8), | |
| 21 | cached_test_metadata: ?CachedTestMetadata = null, | |
| 22 | ||
| 23 | ||
| 24 | fn make(step: *Step, options: Step.MakeOptions) !void { | |
| 25 | const b = step.owner; | |
| 26 | const io = b.graph.io; | |
| 27 | const arena = b.allocator; | |
| 28 | const run: *Run = @fieldParentPtr("step", step); | |
| 29 | const has_side_effects = run.hasSideEffects(); | |
| 30 | ||
| 31 | var argv_list = std.array_list.Managed([]const u8).init(arena); | |
| 32 | var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); | |
| 33 | ||
| 34 | var man = b.graph.cache.obtain(); | |
| 35 | defer man.deinit(); | |
| 36 | ||
| 37 | if (run.environ_map) |environ_map| { | |
| 38 | for (environ_map.keys(), environ_map.values()) |key, value| { | |
| 39 | man.hash.addBytes(key); | |
| 40 | man.hash.addBytes(value); | |
| 41 | } | |
| 42 | } | |
| 43 | ||
| 44 | man.hash.add(run.color); | |
| 45 | man.hash.add(run.disable_zig_progress); | |
| 46 | ||
| 47 | for (run.argv.items) |arg| { | |
| 48 | switch (arg) { | |
| 49 | .bytes => |bytes| { | |
| 50 | try argv_list.append(bytes); | |
| 51 | man.hash.addBytes(bytes); | |
| 52 | }, | |
| 53 | .lazy_path => |file| { | |
| 54 | const file_path = file.lazy_path.getPath3(b, step); | |
| 55 | try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); | |
| 56 | man.hash.addBytes(file.prefix); | |
| 57 | _ = try man.addFilePath(file_path, null); | |
| 58 | }, | |
| 59 | .decorated_directory => |dd| { | |
| 60 | const file_path = dd.lazy_path.getPath3(b, step); | |
| 61 | const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); | |
| 62 | try argv_list.append(resolved_arg); | |
| 63 | man.hash.addBytes(resolved_arg); | |
| 64 | }, | |
| 65 | .file_content => |file_plp| { | |
| 66 | const file_path = file_plp.lazy_path.getPath3(b, step); | |
| 67 | ||
| 68 | var result: std.Io.Writer.Allocating = .init(arena); | |
| 69 | errdefer result.deinit(); | |
| 70 | result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; | |
| 71 | ||
| 72 | const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { | |
| 73 | return step.fail( | |
| 74 | "unable to open input file '{f}': {t}", | |
| 75 | .{ file_path, err }, | |
| 76 | ); | |
| 77 | }; | |
| 78 | defer file.close(io); | |
| 79 | ||
| 80 | var buf: [1024]u8 = undefined; | |
| 81 | var file_reader = file.reader(io, &buf); | |
| 82 | _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { | |
| 83 | error.ReadFailed => return step.fail( | |
| 84 | "failed to read from '{f}': {t}", | |
| 85 | .{ file_path, file_reader.err.? }, | |
| 86 | ), | |
| 87 | error.WriteFailed => return error.OutOfMemory, | |
| 88 | }; | |
| 89 | ||
| 90 | try argv_list.append(result.written()); | |
| 91 | man.hash.addBytes(file_plp.prefix); | |
| 92 | _ = try man.addFilePath(file_path, null); | |
| 93 | }, | |
| 94 | .artifact => |pa| { | |
| 95 | const artifact = pa.artifact; | |
| 96 | ||
| 97 | if (artifact.rootModuleTarget().os.tag == .windows) { | |
| 98 | // On Windows we don't have rpaths so we have to add .dll search paths to PATH | |
| 99 | run.addPathForDynLibs(artifact); | |
| 100 | } | |
| 101 | const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; | |
| 102 | ||
| 103 | try argv_list.append(b.fmt("{s}{s}", .{ | |
| 104 | pa.prefix, | |
| 105 | run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), | |
| 106 | })); | |
| 107 | ||
| 108 | _ = try man.addFile(file_path, null); | |
| 109 | }, | |
| 110 | .output_file, .output_directory => |output| { | |
| 111 | man.hash.addBytes(output.prefix); | |
| 112 | man.hash.addBytes(output.basename); | |
| 113 | // Add a placeholder into the argument list because we need the | |
| 114 | // manifest hash to be updated with all arguments before the | |
| 115 | // object directory is computed. | |
| 116 | try output_placeholders.append(.{ | |
| 117 | .index = argv_list.items.len, | |
| 118 | .tag = arg, | |
| 119 | .output = output, | |
| 120 | }); | |
| 121 | _ = try argv_list.addOne(); | |
| 122 | }, | |
| 123 | } | |
| 124 | } | |
| 125 | ||
| 126 | switch (run.stdin) { | |
| 127 | .bytes => |bytes| { | |
| 128 | man.hash.addBytes(bytes); | |
| 129 | }, | |
| 130 | .lazy_path => |lazy_path| { | |
| 131 | const file_path = lazy_path.getPath2(b, step); | |
| 132 | _ = try man.addFile(file_path, null); | |
| 133 | }, | |
| 134 | .none => {}, | |
| 135 | } | |
| 136 | ||
| 137 | if (run.captured_stdout) |captured| { | |
| 138 | man.hash.addBytes(captured.output.basename); | |
| 139 | man.hash.add(captured.trim_whitespace); | |
| 140 | } | |
| 141 | ||
| 142 | if (run.captured_stderr) |captured| { | |
| 143 | man.hash.addBytes(captured.output.basename); | |
| 144 | man.hash.add(captured.trim_whitespace); | |
| 145 | } | |
| 146 | ||
| 147 | hashStdIo(&man.hash, run.stdio); | |
| 148 | ||
| 149 | for (run.file_inputs.items) |lazy_path| { | |
| 150 | _ = try man.addFile(lazy_path.getPath2(b, step), null); | |
| 151 | } | |
| 152 | ||
| 153 | if (run.cwd) |cwd| { | |
| 154 | const cwd_path = cwd.getPath3(b, step); | |
| 155 | _ = man.hash.addBytes(try cwd_path.toString(arena)); | |
| 156 | } | |
| 157 | ||
| 158 | if (!has_side_effects and try step.cacheHitAndWatch(&man)) { | |
| 159 | // cache hit, skip running command | |
| 160 | const digest = man.final(); | |
| 161 | ||
| 162 | try populateGeneratedPaths( | |
| 163 | arena, | |
| 164 | output_placeholders.items, | |
| 165 | run.captured_stdout, | |
| 166 | run.captured_stderr, | |
| 167 | b.cache_root, | |
| 168 | &digest, | |
| 169 | ); | |
| 170 | ||
| 171 | step.result_cached = true; | |
| 172 | return; | |
| 173 | } | |
| 174 | ||
| 175 | const dep_output_file = run.dep_output_file orelse { | |
| 176 | // We already know the final output paths, use them directly. | |
| 177 | const digest = if (has_side_effects) | |
| 178 | man.hash.final() | |
| 179 | else | |
| 180 | man.final(); | |
| 181 | ||
| 182 | try populateGeneratedPaths( | |
| 183 | arena, | |
| 184 | output_placeholders.items, | |
| 185 | run.captured_stdout, | |
| 186 | run.captured_stderr, | |
| 187 | b.cache_root, | |
| 188 | &digest, | |
| 189 | ); | |
| 190 | ||
| 191 | const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; | |
| 192 | for (output_placeholders.items) |placeholder| { | |
| 193 | const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); | |
| 194 | const output_sub_dir_path = switch (placeholder.tag) { | |
| 195 | .output_file => Dir.path.dirname(output_sub_path).?, | |
| 196 | .output_directory => output_sub_path, | |
| 197 | else => unreachable, | |
| 198 | }; | |
| 199 | b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { | |
| 200 | return step.fail("unable to make path '{f}{s}': {s}", .{ | |
| 201 | b.cache_root, output_sub_dir_path, @errorName(err), | |
| 202 | }); | |
| 203 | }; | |
| 204 | const arg_output_path = run.convertPathArg(.{ | |
| 205 | .root_dir = .cwd(), | |
| 206 | .sub_path = placeholder.output.generated_file.getPath(), | |
| 207 | }); | |
| 208 | argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) | |
| 209 | arg_output_path | |
| 210 | else | |
| 211 | b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); | |
| 212 | } | |
| 213 | ||
| 214 | try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); | |
| 215 | if (!has_side_effects) try step.writeManifestAndWatch(&man); | |
| 216 | return; | |
| 217 | }; | |
| 218 | ||
| 219 | // We do not know the final output paths yet, use temp paths to run the command. | |
| 220 | var rand_int: u64 = undefined; | |
| 221 | io.random(@ptrCast(&rand_int)); | |
| 222 | const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); | |
| 223 | ||
| 224 | for (output_placeholders.items) |placeholder| { | |
| 225 | const output_components = .{ tmp_dir_path, placeholder.output.basename }; | |
| 226 | const output_sub_path = b.pathJoin(&output_components); | |
| 227 | const output_sub_dir_path = switch (placeholder.tag) { | |
| 228 | .output_file => Dir.path.dirname(output_sub_path).?, | |
| 229 | .output_directory => output_sub_path, | |
| 230 | else => unreachable, | |
| 231 | }; | |
| 232 | b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { | |
| 233 | return step.fail("unable to make path '{f}{s}': {s}", .{ | |
| 234 | b.cache_root, output_sub_dir_path, @errorName(err), | |
| 235 | }); | |
| 236 | }; | |
| 237 | const raw_output_path: Cache.Path = .{ | |
| 238 | .root_dir = b.cache_root, | |
| 239 | .sub_path = b.pathJoin(&output_components), | |
| 240 | }; | |
| 241 | placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); | |
| 242 | argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ | |
| 243 | placeholder.output.prefix, | |
| 244 | run.convertPathArg(raw_output_path), | |
| 245 | }); | |
| 246 | } | |
| 247 | ||
| 248 | try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); | |
| 249 | ||
| 250 | const dep_file_dir = Dir.cwd(); | |
| 251 | const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); | |
| 252 | if (has_side_effects) | |
| 253 | try man.addDepFile(dep_file_dir, dep_file_basename) | |
| 254 | else | |
| 255 | try man.addDepFilePost(dep_file_dir, dep_file_basename); | |
| 256 | ||
| 257 | const digest = if (has_side_effects) | |
| 258 | man.hash.final() | |
| 259 | else | |
| 260 | man.final(); | |
| 261 | ||
| 262 | const any_output = output_placeholders.items.len > 0 or | |
| 263 | run.captured_stdout != null or run.captured_stderr != null; | |
| 264 | ||
| 265 | // Rename into place | |
| 266 | if (any_output) { | |
| 267 | const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; | |
| 268 | ||
| 269 | b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { | |
| 270 | Dir.RenameError.DirNotEmpty => { | |
| 271 | b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { | |
| 272 | return step.fail("unable to remove dir '{f}'{s}: {t}", .{ | |
| 273 | b.cache_root, tmp_dir_path, del_err, | |
| 274 | }); | |
| 275 | }; | |
| 276 | b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { | |
| 277 | return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ | |
| 278 | b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, | |
| 279 | }); | |
| 280 | }; | |
| 281 | }, | |
| 282 | else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ | |
| 283 | b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, | |
| 284 | }), | |
| 285 | }; | |
| 286 | } | |
| 287 | ||
| 288 | if (!has_side_effects) try step.writeManifestAndWatch(&man); | |
| 289 | ||
| 290 | try populateGeneratedPaths( | |
| 291 | arena, | |
| 292 | output_placeholders.items, | |
| 293 | run.captured_stdout, | |
| 294 | run.captured_stderr, | |
| 295 | b.cache_root, | |
| 296 | &digest, | |
| 297 | ); | |
| 298 | } | |
| 299 | ||
| 300 | /// Reads stdout of a Zig test process until a termination condition is reached: | |
| 301 | /// * A write fails, indicating the child unexpectedly closed stdin | |
| 302 | /// * A test (or a response from the test runner) times out | |
| 303 | /// * The wait fails, indicating the child closed stdout and stderr | |
| 304 | fn waitZigTest( | |
| 305 | run: *Run, | |
| 306 | child: *process.Child, | |
| 307 | options: Step.MakeOptions, | |
| 308 | multi_reader: *Io.File.MultiReader, | |
| 309 | opt_metadata: *?TestMetadata, | |
| 310 | results: *Step.TestResults, | |
| 311 | ) !union(enum) { | |
| 312 | write_failed: anyerror, | |
| 313 | no_poll: struct { | |
| 314 | active_test_index: ?u32, | |
| 315 | ns_elapsed: u64, | |
| 316 | }, | |
| 317 | timeout: struct { | |
| 318 | active_test_index: ?u32, | |
| 319 | ns_elapsed: u64, | |
| 320 | }, | |
| 321 | } { | |
| 322 | const gpa = run.step.owner.allocator; | |
| 323 | const arena = run.step.owner.allocator; | |
| 324 | const io = run.step.owner.graph.io; | |
| 325 | ||
| 326 | var sub_prog_node: ?std.Progress.Node = null; | |
| 327 | defer if (sub_prog_node) |n| n.end(); | |
| 328 | ||
| 329 | if (opt_metadata.*) |*md| { | |
| 330 | // Previous unit test process died or was killed; we're continuing where it left off | |
| 331 | requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; | |
| 332 | } else { | |
| 333 | // Running unit tests normally | |
| 334 | run.fuzz_tests.clearRetainingCapacity(); | |
| 335 | sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err }; | |
| 336 | } | |
| 337 | ||
| 338 | var active_test_index: ?u32 = null; | |
| 339 | ||
| 340 | var last_update: Io.Clock.Timestamp = .now(io, .awake); | |
| 341 | ||
| 342 | // This timeout is used when we're waiting on the test runner itself rather than a user-specified | |
| 343 | // test. For instance, if the test runner leaves this much time between us requesting a test to | |
| 344 | // start and it acknowledging the test starting, we terminate the child and raise an error. This | |
| 345 | // *should* never happen, but could in theory be caused by some very unlucky IB in a test. | |
| 346 | const response_timeout: Io.Clock.Duration = t: { | |
| 347 | if (fuzz_context != null) break :t null; // don't timeout fuzz tests | |
| 348 | const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); | |
| 349 | break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; | |
| 350 | }; | |
| 351 | const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ | |
| 352 | .clock = .awake, | |
| 353 | .raw = .fromNanoseconds(ns), | |
| 354 | } else null; | |
| 355 | ||
| 356 | const stdout = multi_reader.reader(0); | |
| 357 | const stderr = multi_reader.reader(1); | |
| 358 | const Header = std.zig.Server.Message.Header; | |
| 359 | ||
| 360 | while (true) { | |
| 361 | const timeout: Io.Timeout = t: { | |
| 362 | const opt_duration = if (active_test_index == null) response_timeout else test_timeout; | |
| 363 | const duration = opt_duration orelse break :t .none; | |
| 364 | break :t .{ .deadline = last_update.addDuration(duration) }; | |
| 365 | }; | |
| 366 | ||
| 367 | // This block is exited when `stdout` contains enough bytes for a `Header`. | |
| 368 | header_ready: { | |
| 369 | if (stdout.buffered().len >= @sizeOf(Header)) { | |
| 370 | // We already have one, no need to poll! | |
| 371 | break :header_ready; | |
| 372 | } | |
| 373 | ||
| 374 | multi_reader.fill(64, timeout) catch |err| switch (err) { | |
| 375 | error.Timeout => return .{ .timeout = .{ | |
| 376 | .active_test_index = active_test_index, | |
| 377 | .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), | |
| 378 | } }, | |
| 379 | error.EndOfStream => return .{ .no_poll = .{ | |
| 380 | .active_test_index = active_test_index, | |
| 381 | .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), | |
| 382 | } }, | |
| 383 | else => |e| return e, | |
| 384 | }; | |
| 385 | ||
| 386 | continue; | |
| 387 | } | |
| 388 | // There is definitely a header available now -- read it. | |
| 389 | const header = stdout.takeStruct(Header, .little) catch unreachable; | |
| 390 | ||
| 391 | while (stdout.buffered().len < header.bytes_len) { | |
| 392 | multi_reader.fill(64, timeout) catch |err| switch (err) { | |
| 393 | error.Timeout => return .{ .timeout = .{ | |
| 394 | .active_test_index = active_test_index, | |
| 395 | .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), | |
| 396 | } }, | |
| 397 | error.EndOfStream => return .{ .no_poll = .{ | |
| 398 | .active_test_index = active_test_index, | |
| 399 | .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), | |
| 400 | } }, | |
| 401 | else => |e| return e, | |
| 402 | }; | |
| 403 | } | |
| 404 | ||
| 405 | const body = stdout.take(header.bytes_len) catch unreachable; | |
| 406 | var body_r: std.Io.Reader = .fixed(body); | |
| 407 | switch (header.tag) { | |
| 408 | .zig_version => { | |
| 409 | if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( | |
| 410 | "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", | |
| 411 | .{ builtin.zig_version_string, body }, | |
| 412 | ); | |
| 413 | }, | |
| 414 | .test_metadata => { | |
| 415 | // `metadata` would only be populated if we'd already seen a `test_metadata`, but we | |
| 416 | // only request it once (and importantly, we don't re-request it if we kill and | |
| 417 | // restart the test runner). | |
| 418 | assert(opt_metadata.* == null); | |
| 419 | ||
| 420 | const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable; | |
| 421 | results.test_count = tm_hdr.tests_len; | |
| 422 | ||
| 423 | const names = try arena.alloc(u32, results.test_count); | |
| 424 | for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; | |
| 425 | ||
| 426 | const expected_panic_msgs = try arena.alloc(u32, results.test_count); | |
| 427 | for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; | |
| 428 | ||
| 429 | const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; | |
| 430 | ||
| 431 | options.progress_node.setEstimatedTotalItems(names.len); | |
| 432 | opt_metadata.* = .{ | |
| 433 | .string_bytes = try arena.dupe(u8, string_bytes), | |
| 434 | .ns_per_test = try arena.alloc(u64, results.test_count), | |
| 435 | .names = names, | |
| 436 | .expected_panic_msgs = expected_panic_msgs, | |
| 437 | .next_index = 0, | |
| 438 | .prog_node = options.progress_node, | |
| 439 | }; | |
| 440 | @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); | |
| 441 | ||
| 442 | active_test_index = null; | |
| 443 | last_update = .now(io, .awake); | |
| 444 | ||
| 445 | requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err }; | |
| 446 | }, | |
| 447 | .test_started => { | |
| 448 | active_test_index = opt_metadata.*.?.next_index - 1; | |
| 449 | last_update = .now(io, .awake); | |
| 450 | }, | |
| 451 | .test_results => { | |
| 452 | const md = &opt_metadata.*.?; | |
| 453 | ||
| 454 | const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable; | |
| 455 | assert(tr_hdr.index == active_test_index); | |
| 456 | ||
| 457 | switch (tr_hdr.flags.status) { | |
| 458 | .pass => {}, | |
| 459 | .skip => results.skip_count +|= 1, | |
| 460 | .fail => results.fail_count +|= 1, | |
| 461 | } | |
| 462 | const leak_count = tr_hdr.flags.leak_count; | |
| 463 | const log_err_count = tr_hdr.flags.log_err_count; | |
| 464 | results.leak_count +|= leak_count; | |
| 465 | results.log_err_count +|= log_err_count; | |
| 466 | ||
| 467 | if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index)); | |
| 468 | ||
| 469 | if (tr_hdr.flags.status == .fail) { | |
| 470 | const name = md.testName(tr_hdr.index); | |
| 471 | const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); | |
| 472 | stderr.tossBuffered(); | |
| 473 | if (stderr_bytes.len == 0) { | |
| 474 | try run.step.addError("'{s}' failed without output", .{name}); | |
| 475 | } else { | |
| 476 | try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); | |
| 477 | } | |
| 478 | } else if (leak_count > 0) { | |
| 479 | const name = md.testName(tr_hdr.index); | |
| 480 | const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); | |
| 481 | stderr.tossBuffered(); | |
| 482 | try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); | |
| 483 | } else if (log_err_count > 0) { | |
| 484 | const name = md.testName(tr_hdr.index); | |
| 485 | const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); | |
| 486 | stderr.tossBuffered(); | |
| 487 | try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); | |
| 488 | } | |
| 489 | ||
| 490 | active_test_index = null; | |
| 491 | ||
| 492 | const now: Io.Clock.Timestamp = .now(io, .awake); | |
| 493 | md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds); | |
| 494 | last_update = now; | |
| 495 | ||
| 496 | requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; | |
| 497 | }, | |
| 498 | else => {}, // ignore other messages | |
| 499 | } | |
| 500 | } | |
| 501 | } | |
| 502 | ||
| 503 | const FuzzTestRunner = struct { | |
| 504 | run: *Run, | |
| 505 | ctx: FuzzContext, | |
| 506 | coverage_id: ?u64, | |
| 507 | ||
| 508 | instances: []Instance, | |
| 509 | /// The indexes of this are layed out such that it is effectively an array | |
| 510 | /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr. | |
| 511 | batch: Io.Batch, | |
| 512 | /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter. | |
| 513 | pending_broadcasts: std.ArrayList(u8), | |
| 514 | broadcast: std.ArrayList(u8), | |
| 515 | broadcast_undelivered: u32, | |
| 516 | ||
| 517 | const Instance = struct { | |
| 518 | child: process.Child, | |
| 519 | message: std.ArrayListAligned(u8, .@"4"), | |
| 520 | broadcast_written: usize, | |
| 521 | stderr: std.ArrayList(u8), | |
| 522 | stdin_vec: [1][]u8, | |
| 523 | stdout_vec: [1][]u8, | |
| 524 | stderr_vec: [1][]u8, | |
| 525 | progress_node: std.Progress.Node, | |
| 526 | ||
| 527 | fn messageHeader(instance: *Instance) InHeader { | |
| 528 | assert(instance.message.items.len >= @sizeOf(InHeader)); | |
| 529 | const header_ptr: *InHeader = @ptrCast(instance.message.items); | |
| 530 | var header = header_ptr.*; | |
| 531 | if (std.builtin.Endian.native != .little) { | |
| 532 | std.mem.byteSwapAllFields(InHeader, &header); | |
| 533 | } | |
| 534 | return header; | |
| 535 | } | |
| 536 | }; | |
| 537 | ||
| 538 | const PendingBroadcastFooter = struct { | |
| 539 | from_id: u32, | |
| 540 | body_len: u32, | |
| 541 | }; | |
| 542 | ||
| 543 | const InHeader = std.zig.Server.Message.Header; | |
| 544 | const OutHeader = std.zig.Client.Message.Header; | |
| 545 | ||
| 546 | const stdin_i = 0; | |
| 547 | const stdout_i = 1; | |
| 548 | const stderr_i = 2; | |
| 549 | ||
| 550 | fn init( | |
| 551 | run: *Run, | |
| 552 | ctx: FuzzContext, | |
| 553 | progress_node: std.Progress.Node, | |
| 554 | spawn_options: process.SpawnOptions, | |
| 555 | ) !FuzzTestRunner { | |
| 556 | const step_owner = run.step.owner; | |
| 557 | const gpa = step_owner.allocator; | |
| 558 | const io = step_owner.graph.io; | |
| 559 | ||
| 560 | const n_instances = switch (ctx.fuzz.mode) { | |
| 561 | .forever => step_owner.graph.max_jobs orelse @min( | |
| 562 | std.Thread.getCpuCount() catch 1, | |
| 563 | (std.math.maxInt(u32) - 2) / 3, | |
| 564 | ), | |
| 565 | .limit => 1, | |
| 566 | }; | |
| 567 | const instances = try gpa.alloc(Instance, n_instances); | |
| 568 | errdefer gpa.free(instances); | |
| 569 | const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3); | |
| 570 | errdefer gpa.free(batch_storage); | |
| 571 | ||
| 572 | @memset(instances, .{ | |
| 573 | .child = undefined, | |
| 574 | .message = .empty, | |
| 575 | .broadcast_written = undefined, | |
| 576 | .stderr = .empty, | |
| 577 | .stdin_vec = undefined, | |
| 578 | .stdout_vec = undefined, | |
| 579 | .stderr_vec = undefined, | |
| 580 | .progress_node = undefined, | |
| 581 | }); | |
| 582 | for (0.., instances) |id, *instance| { | |
| 583 | errdefer for (instances[0..id]) |*spawned| { | |
| 584 | spawned.child.kill(io); | |
| 585 | spawned.progress_node.end(); | |
| 586 | }; | |
| 587 | instance.child = try process.spawn(io, spawn_options); | |
| 588 | instance.progress_node = progress_node.start("starting fuzzer", 0); | |
| 589 | } | |
| 590 | ||
| 591 | return .{ | |
| 592 | .run = run, | |
| 593 | .ctx = ctx, | |
| 594 | .coverage_id = null, | |
| 595 | ||
| 596 | .instances = instances, | |
| 597 | .batch = .init(batch_storage), | |
| 598 | .pending_broadcasts = .empty, | |
| 599 | .broadcast = .empty, | |
| 600 | .broadcast_undelivered = 0, | |
| 601 | }; | |
| 602 | } | |
| 603 | ||
| 604 | fn deinit(f: *FuzzTestRunner) void { | |
| 605 | const step_owner = f.run.step.owner; | |
| 606 | const gpa = step_owner.allocator; | |
| 607 | const io = step_owner.graph.io; | |
| 608 | ||
| 609 | f.batch.cancel(io); | |
| 610 | gpa.free(f.batch.storage); | |
| 611 | var total_rss: usize = 0; | |
| 612 | for (f.instances) |*instance| { | |
| 613 | instance.child.kill(io); | |
| 614 | instance.message.deinit(gpa); | |
| 615 | instance.stderr.deinit(gpa); | |
| 616 | instance.progress_node.end(); | |
| 617 | total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; | |
| 618 | } | |
| 619 | f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); | |
| 620 | gpa.free(f.instances); | |
| 621 | } | |
| 622 | ||
| 623 | fn startInstances(f: *FuzzTestRunner) !void { | |
| 624 | const step_owner = f.run.step.owner; | |
| 625 | const io = step_owner.graph.io; | |
| 626 | ||
| 627 | for (0.., f.instances) |id, *instance| { | |
| 628 | const id32: u32 = @intCast(id); | |
| 629 | (switch (f.ctx.fuzz.mode) { | |
| 630 | .forever => sendRunFuzzTestMessage( | |
| 631 | io, | |
| 632 | instance.child.stdin.?, | |
| 633 | f.run.fuzz_tests.items, | |
| 634 | .forever, | |
| 635 | id32, | |
| 636 | ), | |
| 637 | .limit => |limit| sendRunFuzzTestMessage( | |
| 638 | io, | |
| 639 | instance.child.stdin.?, | |
| 640 | f.run.fuzz_tests.items, | |
| 641 | .iterations, | |
| 642 | limit.amount, | |
| 643 | ), | |
| 644 | }) catch |write_err| { | |
| 645 | // The runner unexpectedly closed stdin, which means it crashed during initialization. | |
| 646 | // Clean up everything and wait for the child to exit. | |
| 647 | instance.child.stdin.?.close(io); | |
| 648 | instance.child.stdin = null; | |
| 649 | const term = try instance.child.wait(io); | |
| 650 | return f.run.step.fail( | |
| 651 | "unable to write stdin ({t}); test process unexpectedly {f}", | |
| 652 | .{ write_err, fmtTerm(term) }, | |
| 653 | ); | |
| 654 | }; | |
| 655 | ||
| 656 | try f.addStdoutRead(id32, @sizeOf(InHeader)); | |
| 657 | try f.addStderrRead(id32); | |
| 658 | } | |
| 659 | } | |
| 660 | ||
| 661 | fn listen(f: *FuzzTestRunner) !void { | |
| 662 | const step_owner = f.run.step.owner; | |
| 663 | const io = step_owner.graph.io; | |
| 664 | ||
| 665 | while (true) { | |
| 666 | try f.batch.awaitConcurrent(io, .none); | |
| 667 | while (f.batch.next()) |completion| { | |
| 668 | const id = completion.index / 3; | |
| 669 | const result = completion.result; | |
| 670 | switch (completion.index % 3) { | |
| 671 | 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) { | |
| 672 | // Avoid calling `instanceEos` until EndOfStream is seen with stderr so | |
| 673 | // that all stderr is collected. | |
| 674 | error.BrokenPipe => continue, | |
| 675 | else => |write_e| return write_e, | |
| 676 | }), | |
| 677 | 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) { | |
| 678 | // Avoid calling `instanceEos` until EndOfStream is seen with stderr so | |
| 679 | // that all stderr is collected. | |
| 680 | error.EndOfStream => continue, | |
| 681 | else => |read_e| return read_e, | |
| 682 | }), | |
| 683 | 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { | |
| 684 | error.EndOfStream => return f.instanceEos(id), | |
| 685 | else => |read_e| return read_e, | |
| 686 | }), | |
| 687 | else => unreachable, | |
| 688 | } | |
| 689 | } | |
| 690 | } | |
| 691 | } | |
| 692 | ||
| 693 | fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { | |
| 694 | const step_owner = f.run.step.owner; | |
| 695 | const gpa = step_owner.allocator; | |
| 696 | const io = step_owner.graph.io; | |
| 697 | const instance = &f.instances[id]; | |
| 698 | ||
| 699 | instance.message.items.len += n; | |
| 700 | const total_read = instance.message.items.len; | |
| 701 | if (total_read < @sizeOf(InHeader)) { | |
| 702 | try f.addStdoutRead(id, @sizeOf(InHeader)); | |
| 703 | return; | |
| 704 | } | |
| 705 | ||
| 706 | const header = instance.messageHeader(); | |
| 707 | const body = instance.message.items[@sizeOf(InHeader)..]; | |
| 708 | if (body.len != header.bytes_len) { | |
| 709 | try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len); | |
| 710 | return; | |
| 711 | } | |
| 712 | ||
| 713 | switch (header.tag) { | |
| 714 | .zig_version => { | |
| 715 | if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( | |
| 716 | "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", | |
| 717 | .{ builtin.zig_version_string, body }, | |
| 718 | ); | |
| 719 | }, | |
| 720 | .coverage_id => { | |
| 721 | var body_r: Io.Reader = .fixed(body); | |
| 722 | f.coverage_id = body_r.takeInt(u64, .little) catch unreachable; | |
| 723 | const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable; | |
| 724 | const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable; | |
| 725 | const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable; | |
| 726 | ||
| 727 | const fuzz = f.ctx.fuzz; | |
| 728 | fuzz.queue_mutex.lockUncancelable(io); | |
| 729 | defer fuzz.queue_mutex.unlock(io); | |
| 730 | try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ | |
| 731 | .id = f.coverage_id.?, | |
| 732 | .cumulative = .{ | |
| 733 | .runs = cumulative_runs, | |
| 734 | .unique = cumulative_unique, | |
| 735 | .coverage = cumulative_coverage, | |
| 736 | }, | |
| 737 | .run = f.run, | |
| 738 | } }); | |
| 739 | fuzz.queue_cond.signal(io); | |
| 740 | }, | |
| 741 | .fuzz_start_addr => { | |
| 742 | var body_r: Io.Reader = .fixed(body); | |
| 743 | const fuzz = f.ctx.fuzz; | |
| 744 | const addr = body_r.takeInt(u64, .little) catch unreachable; | |
| 745 | ||
| 746 | fuzz.queue_mutex.lockUncancelable(io); | |
| 747 | defer fuzz.queue_mutex.unlock(io); | |
| 748 | try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ | |
| 749 | .addr = addr, | |
| 750 | .coverage_id = f.coverage_id.?, | |
| 751 | } }); | |
| 752 | fuzz.queue_cond.signal(io); | |
| 753 | }, | |
| 754 | .fuzz_test_change => { | |
| 755 | const test_i = std.mem.readInt(u32, body[0..4], .little); | |
| 756 | instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); | |
| 757 | }, | |
| 758 | .broadcast_fuzz_input => { | |
| 759 | if (f.instances.len == 1) { | |
| 760 | // No other processes to broadcast to. | |
| 761 | } else if (f.broadcast_undelivered == 0) { | |
| 762 | try f.instanceBroadcast(id, body); | |
| 763 | } else { | |
| 764 | const footer: PendingBroadcastFooter = .{ | |
| 765 | .from_id = id, | |
| 766 | .body_len = @intCast(body.len), | |
| 767 | }; | |
| 768 | // There is another broadcast in progress so add this one to the queue. | |
| 769 | const size = @sizeOf(PendingBroadcastFooter) + body.len; | |
| 770 | try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); | |
| 771 | f.pending_broadcasts.appendSliceAssumeCapacity(body); | |
| 772 | f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); | |
| 773 | } | |
| 774 | }, | |
| 775 | else => {}, // ignore other messages | |
| 776 | } | |
| 777 | ||
| 778 | instance.message.clearRetainingCapacity(); | |
| 779 | try f.addStdoutRead(id, @sizeOf(InHeader)); | |
| 780 | } | |
| 781 | ||
| 782 | fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void { | |
| 783 | const instance = &f.instances[id]; | |
| 784 | instance.stderr.items.len += n; | |
| 785 | try f.addStderrRead(id); | |
| 786 | } | |
| 787 | ||
| 788 | fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void { | |
| 789 | const instance = &f.instances[id]; | |
| 790 | ||
| 791 | instance.broadcast_written += n; | |
| 792 | if (instance.broadcast_written == f.broadcast.items.len) { | |
| 793 | f.broadcast_undelivered -= 1; | |
| 794 | if (f.broadcast_undelivered == 0) { | |
| 795 | try f.broadcastComplete(); | |
| 796 | } | |
| 797 | } else { | |
| 798 | f.addStdinWrite(id); | |
| 799 | } | |
| 800 | } | |
| 801 | ||
| 802 | fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { | |
| 803 | const step_owner = f.run.step.owner; | |
| 804 | const gpa = step_owner.allocator; | |
| 805 | const instance = &f.instances[id]; | |
| 806 | ||
| 807 | try instance.message.ensureTotalCapacity(gpa, end); | |
| 808 | const start = instance.message.items.len; | |
| 809 | instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]}; | |
| 810 | f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{ | |
| 811 | .file = instance.child.stdout.?, | |
| 812 | .data = &instance.stdout_vec, | |
| 813 | } }); | |
| 814 | } | |
| 815 | ||
| 816 | fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { | |
| 817 | const step_owner = f.run.step.owner; | |
| 818 | const gpa = step_owner.allocator; | |
| 819 | const instance = &f.instances[id]; | |
| 820 | ||
| 821 | try instance.stderr.ensureUnusedCapacity(gpa, 1); | |
| 822 | instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()}; | |
| 823 | f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{ | |
| 824 | .file = instance.child.stderr.?, | |
| 825 | .data = &instance.stderr_vec, | |
| 826 | } }); | |
| 827 | } | |
| 828 | ||
| 829 | fn addStdinWrite(f: *FuzzTestRunner, id: u32) void { | |
| 830 | const instance = &f.instances[id]; | |
| 831 | ||
| 832 | assert(f.broadcast.items.len != instance.broadcast_written); | |
| 833 | instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]}; | |
| 834 | f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{ | |
| 835 | .file = instance.child.stdin.?, | |
| 836 | .data = &instance.stdin_vec, | |
| 837 | } }); | |
| 838 | } | |
| 839 | ||
| 840 | fn instanceEos(f: *FuzzTestRunner, id: u32) !void { | |
| 841 | const step_owner = f.run.step.owner; | |
| 842 | const io = step_owner.graph.io; | |
| 843 | const instance = &f.instances[id]; | |
| 844 | ||
| 845 | instance.child.stdin.?.close(io); | |
| 846 | instance.child.stdin = null; | |
| 847 | const term = try instance.child.wait(io); | |
| 848 | if (!termMatches(.{ .exited = 0 }, term)) { | |
| 849 | f.run.step.result_stderr = try f.mergedStderr(); | |
| 850 | try f.saveCrash(id, term); | |
| 851 | return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); | |
| 852 | } | |
| 853 | } | |
| 854 | ||
| 855 | fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { | |
| 856 | const step = &f.run.step; | |
| 857 | const b = step.owner; | |
| 858 | const io = b.graph.io; | |
| 859 | ||
| 860 | if (f.coverage_id == null) return; | |
| 861 | ||
| 862 | // Search for the input file corresponding to the instance | |
| 863 | const InputHeader = Build.abi.fuzz.MmapInputHeader; | |
| 864 | var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; | |
| 865 | var in_r: Io.File.Reader = undefined; | |
| 866 | var in_f: Io.File = undefined; | |
| 867 | var in_name_buf: [12]u8 = undefined; | |
| 868 | var in_name: []const u8 = undefined; | |
| 869 | var i: u32 = 0; | |
| 870 | const header: InputHeader = while (true) : ({ | |
| 871 | if (i == std.math.maxInt(u32)) return; | |
| 872 | i += 1; | |
| 873 | }) { | |
| 874 | const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; | |
| 875 | in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; | |
| 876 | in_f = b.cache_root.handle.openFile(io, in_name, .{ | |
| 877 | .lock = .exclusive, | |
| 878 | .lock_nonblocking = true, | |
| 879 | }) catch |e| switch (e) { | |
| 880 | error.FileNotFound => return, | |
| 881 | error.WouldBlock => continue, // Can not be from | |
| 882 | // the crashed instance since it is still locked. | |
| 883 | else => return step.fail("failed to open file '{f}{s}': {t}", .{ | |
| 884 | b.cache_root, in_name, e, | |
| 885 | }), | |
| 886 | }; | |
| 887 | ||
| 888 | in_r = in_f.readerStreaming(io, &in_r_buf); | |
| 889 | const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { | |
| 890 | in_f.close(io); | |
| 891 | switch (e) { | |
| 892 | error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ | |
| 893 | b.cache_root, in_name, in_r.err.?, | |
| 894 | }), | |
| 895 | error.EndOfStream => continue, | |
| 896 | } | |
| 897 | }; | |
| 898 | ||
| 899 | if (header.pc_digest == f.coverage_id.? and | |
| 900 | header.instance_id == id and | |
| 901 | header.test_i < f.run.fuzz_tests.items.len) | |
| 902 | { | |
| 903 | break header; | |
| 904 | } | |
| 905 | ||
| 906 | in_f.close(io); | |
| 907 | }; | |
| 908 | defer in_f.close(io); | |
| 909 | ||
| 910 | // Save it to a seperate file | |
| 911 | const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; | |
| 912 | const out = b.cache_root.handle.createFile(io, crash_name, .{ | |
| 913 | .lock = .exclusive, // Multiple run steps could have found a crash at the same time | |
| 914 | }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ | |
| 915 | b.cache_root, crash_name, e, | |
| 916 | }); | |
| 917 | defer out.close(io); | |
| 918 | ||
| 919 | var out_w_buf: [512]u8 = undefined; | |
| 920 | var out_w = out.writerStreaming(io, &out_w_buf); | |
| 921 | _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { | |
| 922 | error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ | |
| 923 | b.cache_root, in_name, in_r.err.?, | |
| 924 | }), | |
| 925 | error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ | |
| 926 | b.cache_root, crash_name, out_w.err.?, | |
| 927 | }), | |
| 928 | }; | |
| 929 | ||
| 930 | return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ | |
| 931 | f.run.fuzz_tests.items[header.test_i], | |
| 932 | fmtTerm(term), | |
| 933 | b.cache_root, | |
| 934 | crash_name, | |
| 935 | }); | |
| 936 | } | |
| 937 | ||
| 938 | fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void { | |
| 939 | assert(f.instances.len > 1); | |
| 940 | assert(f.broadcast_undelivered == 0); // no other broadcast is progress | |
| 941 | assert(f.broadcast.items.len == 0); | |
| 942 | assert(from_id < f.instances.len); | |
| 943 | ||
| 944 | const step_owner = f.run.step.owner; | |
| 945 | const gpa = step_owner.allocator; | |
| 946 | ||
| 947 | var out_header: OutHeader = .{ | |
| 948 | .tag = .new_fuzz_input, | |
| 949 | .bytes_len = @intCast(bytes.len), | |
| 950 | }; | |
| 951 | if (std.builtin.Endian.native != .little) { | |
| 952 | std.mem.byteSwapAllFields(OutHeader, &out_header); | |
| 953 | } | |
| 954 | try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len); | |
| 955 | f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header)); | |
| 956 | f.broadcast.appendSliceAssumeCapacity(bytes); | |
| 957 | ||
| 958 | f.broadcast_undelivered = @intCast(f.instances.len - 1); | |
| 959 | for (0.., f.instances) |to_id, *instance| { | |
| 960 | if (to_id == from_id) continue; | |
| 961 | instance.broadcast_written = 0; | |
| 962 | f.addStdinWrite(@intCast(to_id)); | |
| 963 | } | |
| 964 | } | |
| 965 | ||
| 966 | fn broadcastComplete(f: *FuzzTestRunner) !void { | |
| 967 | assert(f.instances.len > 1); | |
| 968 | assert(f.broadcast_undelivered == 0); | |
| 969 | f.broadcast.clearRetainingCapacity(); | |
| 970 | ||
| 971 | const pending = &f.pending_broadcasts; | |
| 972 | if (pending.items.len != 0) { | |
| 973 | // Another broadcast is pending; copy it over to `broadcast` | |
| 974 | ||
| 975 | const footer_len = @sizeOf(PendingBroadcastFooter); | |
| 976 | const footer_bytes = pending.items[pending.items.len - footer_len ..]; | |
| 977 | const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes); | |
| 978 | pending.items.len -= footer_len; | |
| 979 | ||
| 980 | const body = pending.items[pending.items.len - footer.body_len ..]; | |
| 981 | try f.instanceBroadcast(footer.from_id, body); | |
| 982 | pending.items.len -= body.len; | |
| 983 | } | |
| 984 | } | |
| 985 | ||
| 986 | fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { | |
| 987 | const step_owner = f.run.step.owner; | |
| 988 | const arena = step_owner.allocator; | |
| 989 | ||
| 990 | // Collect any available stderr | |
| 991 | while (f.batch.next()) |completion| { | |
| 992 | if (completion.index % 3 != 2) continue; | |
| 993 | const len = completion.result.file_read_streaming catch continue; | |
| 994 | f.instances[completion.index / 3].stderr.items.len += len; | |
| 995 | } | |
| 996 | ||
| 997 | var stderr_len: usize = 0; | |
| 998 | for (f.instances) |*instance| stderr_len += instance.stderr.items.len; | |
| 999 | const stderr = try arena.alloc(u8, stderr_len); | |
| 1000 | ||
| 1001 | stderr_len = 0; | |
| 1002 | for (f.instances) |*instance| { | |
| 1003 | @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items); | |
| 1004 | stderr_len += instance.stderr.items.len; | |
| 1005 | } | |
| 1006 | return stderr; | |
| 1007 | } | |
| 1008 | }; | |
| 1009 | ||
| 1010 | fn evalFuzzTest( | |
| 1011 | run: *Run, | |
| 1012 | spawn_options: process.SpawnOptions, | |
| 1013 | options: Step.MakeOptions, | |
| 1014 | fuzz_context: FuzzContext, | |
| 1015 | ) !void { | |
| 1016 | var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); | |
| 1017 | defer f.deinit(); | |
| 1018 | try f.startInstances(); | |
| 1019 | try f.listen(); | |
| 1020 | } | |
| 1021 | ||
| 1022 | const StdioPollEnum = enum { stdout, stderr }; | |
| 1023 | ||
| 1024 | fn evalZigTest( | |
| 1025 | run: *Run, | |
| 1026 | spawn_options: process.SpawnOptions, | |
| 1027 | options: Step.MakeOptions, | |
| 1028 | fuzz_context: ?FuzzContext, | |
| 1029 | ) !void { | |
| 1030 | if (fuzz_context != null) { | |
| 1031 | try evalFuzzTest(run, spawn_options, options, fuzz_context.?); | |
| 1032 | return; | |
| 1033 | } | |
| 1034 | ||
| 1035 | const step_owner = run.step.owner; | |
| 1036 | const gpa = step_owner.allocator; | |
| 1037 | const arena = step_owner.allocator; | |
| 1038 | const io = step_owner.graph.io; | |
| 1039 | ||
| 1040 | // We will update this every time a child runs. | |
| 1041 | run.step.result_peak_rss = 0; | |
| 1042 | ||
| 1043 | var test_results: Step.TestResults = .{ | |
| 1044 | .test_count = 0, | |
| 1045 | .skip_count = 0, | |
| 1046 | .fail_count = 0, | |
| 1047 | .crash_count = 0, | |
| 1048 | .timeout_count = 0, | |
| 1049 | .leak_count = 0, | |
| 1050 | .log_err_count = 0, | |
| 1051 | }; | |
| 1052 | var test_metadata: ?TestMetadata = null; | |
| 1053 | ||
| 1054 | while (true) { | |
| 1055 | var child = try process.spawn(io, spawn_options); | |
| 1056 | var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; | |
| 1057 | var multi_reader: Io.File.MultiReader = undefined; | |
| 1058 | multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); | |
| 1059 | var child_killed = false; | |
| 1060 | defer if (!child_killed) { | |
| 1061 | child.kill(io); | |
| 1062 | multi_reader.deinit(); | |
| 1063 | run.step.result_peak_rss = @max( | |
| 1064 | run.step.result_peak_rss, | |
| 1065 | child.resource_usage_statistics.getMaxRss() orelse 0, | |
| 1066 | ); | |
| 1067 | }; | |
| 1068 | ||
| 1069 | switch (try waitZigTest( | |
| 1070 | run, | |
| 1071 | &child, | |
| 1072 | options, | |
| 1073 | &multi_reader, | |
| 1074 | &test_metadata, | |
| 1075 | &test_results, | |
| 1076 | )) { | |
| 1077 | .write_failed => |err| { | |
| 1078 | // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured | |
| 1079 | // all available stderr to make our error output as useful as possible. | |
| 1080 | const stderr_fr = multi_reader.fileReader(1); | |
| 1081 | while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) { | |
| 1082 | error.ReadFailed => return stderr_fr.err.?, | |
| 1083 | error.EndOfStream => {}, | |
| 1084 | } | |
| 1085 | run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); | |
| 1086 | ||
| 1087 | // Clean up everything and wait for the child to exit. | |
| 1088 | child.stdin.?.close(io); | |
| 1089 | child.stdin = null; | |
| 1090 | multi_reader.deinit(); | |
| 1091 | child_killed = true; | |
| 1092 | const term = try child.wait(io); | |
| 1093 | run.step.result_peak_rss = @max( | |
| 1094 | run.step.result_peak_rss, | |
| 1095 | child.resource_usage_statistics.getMaxRss() orelse 0, | |
| 1096 | ); | |
| 1097 | ||
| 1098 | // The individual unit test results are irrelevant: the test runner itself broke! | |
| 1099 | // Fail immediately without populating `s.test_results`. | |
| 1100 | return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); | |
| 1101 | }, | |
| 1102 | .no_poll => |no_poll| { | |
| 1103 | // This might be a success (we requested exit and the child dutifully closed stdout) or | |
| 1104 | // a crash of some kind. Either way, the child will terminate by itself -- wait for it. | |
| 1105 | const stderr_reader = multi_reader.reader(1); | |
| 1106 | const stderr_owned = try arena.dupe(u8, stderr_reader.buffered()); | |
| 1107 | ||
| 1108 | // Clean up everything and wait for the child to exit. | |
| 1109 | child.stdin.?.close(io); | |
| 1110 | child.stdin = null; | |
| 1111 | multi_reader.deinit(); | |
| 1112 | child_killed = true; | |
| 1113 | const term = try child.wait(io); | |
| 1114 | run.step.result_peak_rss = @max( | |
| 1115 | run.step.result_peak_rss, | |
| 1116 | child.resource_usage_statistics.getMaxRss() orelse 0, | |
| 1117 | ); | |
| 1118 | ||
| 1119 | if (no_poll.active_test_index) |test_index| { | |
| 1120 | // A test was running, so this is definitely a crash. Report it against that | |
| 1121 | // test, and continue to the next test. | |
| 1122 | test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; | |
| 1123 | test_results.crash_count += 1; | |
| 1124 | try run.step.addError("'{s}' {f}{s}{s}", .{ | |
| 1125 | test_metadata.?.testName(test_index), | |
| 1126 | fmtTerm(term), | |
| 1127 | if (stderr_owned.len != 0) " with stderr:\n" else "", | |
| 1128 | std.mem.trim(u8, stderr_owned, "\n"), | |
| 1129 | }); | |
| 1130 | continue; | |
| 1131 | } | |
| 1132 | ||
| 1133 | // Report an error if the child terminated uncleanly or if we were still trying to run more tests. | |
| 1134 | run.step.result_stderr = stderr_owned; | |
| 1135 | const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); | |
| 1136 | if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { | |
| 1137 | // The individual unit test results are irrelevant: the test runner itself broke! | |
| 1138 | // Fail immediately without populating `s.test_results`. | |
| 1139 | return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); | |
| 1140 | } | |
| 1141 | ||
| 1142 | // We're done with all of the tests! Commit the test results and return. | |
| 1143 | run.step.test_results = test_results; | |
| 1144 | if (test_metadata) |tm| { | |
| 1145 | run.cached_test_metadata = tm.toCachedTestMetadata(); | |
| 1146 | if (options.web_server) |ws| { | |
| 1147 | if (run.step.owner.graph.time_report) { | |
| 1148 | ws.updateTimeReportRunTest( | |
| 1149 | run, | |
| 1150 | &run.cached_test_metadata.?, | |
| 1151 | tm.ns_per_test, | |
| 1152 | ); | |
| 1153 | } | |
| 1154 | } | |
| 1155 | } | |
| 1156 | return; | |
| 1157 | }, | |
| 1158 | .timeout => |timeout| { | |
| 1159 | const stderr_reader = multi_reader.reader(1); | |
| 1160 | const stderr = stderr_reader.buffered(); | |
| 1161 | stderr_reader.tossBuffered(); | |
| 1162 | if (timeout.active_test_index) |test_index| { | |
| 1163 | // A test was running. Report the timeout against that test, and continue on to | |
| 1164 | // the next test. | |
| 1165 | test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; | |
| 1166 | test_results.timeout_count += 1; | |
| 1167 | try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ | |
| 1168 | test_metadata.?.testName(test_index), | |
| 1169 | Io.Duration{ .nanoseconds = timeout.ns_elapsed }, | |
| 1170 | if (stderr.len != 0) " with stderr:\n" else "", | |
| 1171 | std.mem.trim(u8, stderr, "\n"), | |
| 1172 | }); | |
| 1173 | continue; | |
| 1174 | } | |
| 1175 | // Just log an error and let the child be killed. | |
| 1176 | run.step.result_stderr = try arena.dupe(u8, stderr); | |
| 1177 | // The individual unit test results in `results` are irrelevant: the test runner | |
| 1178 | // is broken! Fail immediately without populating `s.test_results`. | |
| 1179 | return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); | |
| 1180 | }, | |
| 1181 | } | |
| 1182 | comptime unreachable; | |
| 1183 | } | |
| 1184 | } | |
| 1185 | ||
| 1186 | const TestMetadata = struct { | |
| 1187 | names: []const u32, | |
| 1188 | ns_per_test: []u64, | |
| 1189 | expected_panic_msgs: []const u32, | |
| 1190 | string_bytes: []const u8, | |
| 1191 | next_index: u32, | |
| 1192 | prog_node: std.Progress.Node, | |
| 1193 | ||
| 1194 | fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { | |
| 1195 | return .{ | |
| 1196 | .names = tm.names, | |
| 1197 | .string_bytes = tm.string_bytes, | |
| 1198 | }; | |
| 1199 | } | |
| 1200 | ||
| 1201 | fn testName(tm: TestMetadata, index: u32) []const u8 { | |
| 1202 | return tm.toCachedTestMetadata().testName(index); | |
| 1203 | } | |
| 1204 | }; | |
| 1205 | ||
| 1206 | pub const CachedTestMetadata = struct { | |
| 1207 | names: []const u32, | |
| 1208 | string_bytes: []const u8, | |
| 1209 | ||
| 1210 | pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { | |
| 1211 | return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); | |
| 1212 | } | |
| 1213 | }; | |
| 1214 | ||
| 1215 | fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { | |
| 1216 | while (metadata.next_index < metadata.names.len) { | |
| 1217 | const i = metadata.next_index; | |
| 1218 | metadata.next_index += 1; | |
| 1219 | ||
| 1220 | if (metadata.expected_panic_msgs[i] != 0) continue; | |
| 1221 | ||
| 1222 | const name = metadata.testName(i); | |
| 1223 | if (sub_prog_node.*) |n| n.end(); | |
| 1224 | sub_prog_node.* = metadata.prog_node.start(name, 0); | |
| 1225 | ||
| 1226 | try sendRunTestMessage(io, in, .run_test, i); | |
| 1227 | return; | |
| 1228 | } else { | |
| 1229 | metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done | |
| 1230 | try sendMessage(io, in, .exit); | |
| 1231 | } | |
| 1232 | } | |
| 1233 | ||
| 1234 | fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { | |
| 1235 | const header: std.zig.Client.Message.Header = .{ | |
| 1236 | .tag = tag, | |
| 1237 | .bytes_len = 0, | |
| 1238 | }; | |
| 1239 | var w = file.writerStreaming(io, &.{}); | |
| 1240 | w.interface.writeStruct(header, .little) catch |err| switch (err) { | |
| 1241 | error.WriteFailed => return w.err.?, | |
| 1242 | }; | |
| 1243 | } | |
| 1244 | ||
| 1245 | fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void { | |
| 1246 | const header: std.zig.Client.Message.Header = .{ | |
| 1247 | .tag = tag, | |
| 1248 | .bytes_len = 4, | |
| 1249 | }; | |
| 1250 | var w = file.writerStreaming(io, &.{}); | |
| 1251 | w.interface.writeStruct(header, .little) catch |err| switch (err) { | |
| 1252 | error.WriteFailed => return w.err.?, | |
| 1253 | }; | |
| 1254 | w.interface.writeInt(u32, index, .little) catch |err| switch (err) { | |
| 1255 | error.WriteFailed => return w.err.?, | |
| 1256 | }; | |
| 1257 | } | |
| 1258 | ||
| 1259 | fn sendRunFuzzTestMessage( | |
| 1260 | io: Io, | |
| 1261 | file: Io.File, | |
| 1262 | test_names: []const []const u8, | |
| 1263 | kind: std.Build.abi.fuzz.LimitKind, | |
| 1264 | amount_or_instance: u64, | |
| 1265 | ) !void { | |
| 1266 | const header: std.zig.Client.Message.Header = .{ | |
| 1267 | .tag = .start_fuzzing, | |
| 1268 | .bytes_len = 1 + 8 + 4 + count: { | |
| 1269 | var c: u32 = @intCast(test_names.len * 4); | |
| 1270 | for (test_names) |name| { | |
| 1271 | c += @intCast(name.len); | |
| 1272 | } | |
| 1273 | break :count c; | |
| 1274 | }, | |
| 1275 | }; | |
| 1276 | var w = file.writerStreaming(io, &.{}); | |
| 1277 | w.interface.writeStruct(header, .little) catch |err| switch (err) { | |
| 1278 | error.WriteFailed => return w.err.?, | |
| 1279 | }; | |
| 1280 | w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) { | |
| 1281 | error.WriteFailed => return w.err.?, | |
| 1282 | }; | |
| 1283 | w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) { | |
| 1284 | error.WriteFailed => return w.err.?, | |
| 1285 | }; | |
| 1286 | w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) { | |
| 1287 | error.WriteFailed => return w.err.?, | |
| 1288 | }; | |
| 1289 | for (test_names) |test_name| { | |
| 1290 | w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) { | |
| 1291 | error.WriteFailed => return w.err.?, | |
| 1292 | }; | |
| 1293 | w.interface.writeAll(test_name) catch |err| switch (err) { | |
| 1294 | error.WriteFailed => return w.err.?, | |
| 1295 | }; | |
| 1296 | } | |
| 1297 | } | |
| 1298 | ||
| 1299 | fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { | |
| 1300 | const b = run.step.owner; | |
| 1301 | const io = b.graph.io; | |
| 1302 | const arena = b.allocator; | |
| 1303 | const gpa = b.allocator; | |
| 1304 | ||
| 1305 | var child = try process.spawn(io, spawn_options); | |
| 1306 | defer child.kill(io); | |
| 1307 | ||
| 1308 | switch (run.stdin) { | |
| 1309 | .bytes => |bytes| { | |
| 1310 | child.stdin.?.writeStreamingAll(io, bytes) catch |err| { | |
| 1311 | return run.step.fail("unable to write stdin: {t}", .{err}); | |
| 1312 | }; | |
| 1313 | child.stdin.?.close(io); | |
| 1314 | child.stdin = null; | |
| 1315 | }, | |
| 1316 | .lazy_path => |lazy_path| { | |
| 1317 | const path = lazy_path.getPath3(b, &run.step); | |
| 1318 | const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { | |
| 1319 | return run.step.fail("unable to open stdin file: {t}", .{err}); | |
| 1320 | }; | |
| 1321 | defer file.close(io); | |
| 1322 | // TODO https://github.com/ziglang/zig/issues/23955 | |
| 1323 | var read_buffer: [1024]u8 = undefined; | |
| 1324 | var file_reader = file.reader(io, &read_buffer); | |
| 1325 | var write_buffer: [1024]u8 = undefined; | |
| 1326 | var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); | |
| 1327 | _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { | |
| 1328 | error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ | |
| 1329 | path, file_reader.err.?, | |
| 1330 | }), | |
| 1331 | error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ | |
| 1332 | stdin_writer.err.?, | |
| 1333 | }), | |
| 1334 | }; | |
| 1335 | stdin_writer.interface.flush() catch |err| switch (err) { | |
| 1336 | error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ | |
| 1337 | stdin_writer.err.?, | |
| 1338 | }), | |
| 1339 | }; | |
| 1340 | child.stdin.?.close(io); | |
| 1341 | child.stdin = null; | |
| 1342 | }, | |
| 1343 | .none => {}, | |
| 1344 | } | |
| 1345 | ||
| 1346 | var stdout_bytes: ?[]const u8 = null; | |
| 1347 | var stderr_bytes: ?[]const u8 = null; | |
| 1348 | ||
| 1349 | if (child.stdout) |stdout| { | |
| 1350 | if (child.stderr) |stderr| { | |
| 1351 | var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; | |
| 1352 | var multi_reader: Io.File.MultiReader = undefined; | |
| 1353 | multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); | |
| 1354 | defer multi_reader.deinit(); | |
| 1355 | ||
| 1356 | const stdout_reader = multi_reader.reader(0); | |
| 1357 | const stderr_reader = multi_reader.reader(1); | |
| 1358 | ||
| 1359 | while (multi_reader.fill(64, .none)) |_| { | |
| 1360 | if (run.stdio_limit.toInt()) |limit| { | |
| 1361 | if (stdout_reader.buffered().len > limit) | |
| 1362 | return error.StdoutStreamTooLong; | |
| 1363 | if (stderr_reader.buffered().len > limit) | |
| 1364 | return error.StderrStreamTooLong; | |
| 1365 | } | |
| 1366 | } else |err| switch (err) { | |
| 1367 | error.Timeout => unreachable, | |
| 1368 | error.EndOfStream => {}, | |
| 1369 | else => |e| return e, | |
| 1370 | } | |
| 1371 | ||
| 1372 | try multi_reader.checkAnyError(); | |
| 1373 | ||
| 1374 | // TODO: this string can leak since alloc below can return error. | |
| 1375 | stdout_bytes = try multi_reader.toOwnedSlice(0); | |
| 1376 | // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. | |
| 1377 | stderr_bytes = try multi_reader.toOwnedSlice(1); | |
| 1378 | } else { | |
| 1379 | var stdout_reader = stdout.readerStreaming(io, &.{}); | |
| 1380 | stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { | |
| 1381 | error.OutOfMemory => |e| return e, | |
| 1382 | error.ReadFailed => return stdout_reader.err.?, | |
| 1383 | error.StreamTooLong => return error.StdoutStreamTooLong, | |
| 1384 | }; | |
| 1385 | } | |
| 1386 | } else if (child.stderr) |stderr| { | |
| 1387 | var stderr_reader = stderr.readerStreaming(io, &.{}); | |
| 1388 | stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { | |
| 1389 | error.OutOfMemory => |e| return e, | |
| 1390 | error.ReadFailed => return stderr_reader.err.?, | |
| 1391 | error.StreamTooLong => return error.StderrStreamTooLong, | |
| 1392 | }; | |
| 1393 | } | |
| 1394 | ||
| 1395 | if (stderr_bytes) |bytes| if (bytes.len > 0) { | |
| 1396 | // Treat stderr as an error message. | |
| 1397 | const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { | |
| 1398 | .check => |checks| !checksContainStderr(checks.items), | |
| 1399 | else => true, | |
| 1400 | }; | |
| 1401 | if (stderr_is_diagnostic) { | |
| 1402 | run.step.result_stderr = bytes; | |
| 1403 | } | |
| 1404 | }; | |
| 1405 | ||
| 1406 | run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; | |
| 1407 | ||
| 1408 | return .{ | |
| 1409 | .term = try child.wait(io), | |
| 1410 | .stdout = stdout_bytes, | |
| 1411 | .stderr = stderr_bytes, | |
| 1412 | }; | |
| 1413 | } | |
| 1414 | ||
| 1415 | const IndexedOutput = struct { | |
| 1416 | index: usize, | |
| 1417 | tag: @typeInfo(Arg).@"union".tag_type.?, | |
| 1418 | output: *Output, | |
| 1419 | }; | |
| 1420 | ||
| 1421 | pub fn rerunInFuzzMode( | |
| 1422 | run: *Run, | |
| 1423 | fuzz: *std.Build.Fuzz, | |
| 1424 | prog_node: std.Progress.Node, | |
| 1425 | ) !void { | |
| 1426 | const step = &run.step; | |
| 1427 | const b = step.owner; | |
| 1428 | const io = b.graph.io; | |
| 1429 | const arena = b.allocator; | |
| 1430 | var argv_list: std.ArrayList([]const u8) = .empty; | |
| 1431 | for (run.argv.items) |arg| { | |
| 1432 | switch (arg) { | |
| 1433 | .bytes => |bytes| { | |
| 1434 | try argv_list.append(arena, bytes); | |
| 1435 | }, | |
| 1436 | .lazy_path => |file| { | |
| 1437 | const file_path = file.lazy_path.getPath3(b, step); | |
| 1438 | try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); | |
| 1439 | }, | |
| 1440 | .decorated_directory => |dd| { | |
| 1441 | const file_path = dd.lazy_path.getPath3(b, step); | |
| 1442 | try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); | |
| 1443 | }, | |
| 1444 | .file_content => |file_plp| { | |
| 1445 | const file_path = file_plp.lazy_path.getPath3(b, step); | |
| 1446 | ||
| 1447 | var result: std.Io.Writer.Allocating = .init(arena); | |
| 1448 | errdefer result.deinit(); | |
| 1449 | result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; | |
| 1450 | ||
| 1451 | const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); | |
| 1452 | defer file.close(io); | |
| 1453 | ||
| 1454 | var buf: [1024]u8 = undefined; | |
| 1455 | var file_reader = file.reader(io, &buf); | |
| 1456 | _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { | |
| 1457 | error.ReadFailed => return file_reader.err.?, | |
| 1458 | error.WriteFailed => return error.OutOfMemory, | |
| 1459 | }; | |
| 1460 | ||
| 1461 | try argv_list.append(arena, result.written()); | |
| 1462 | }, | |
| 1463 | .artifact => |pa| { | |
| 1464 | const artifact = pa.artifact; | |
| 1465 | const file_path: []const u8 = p: { | |
| 1466 | if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); | |
| 1467 | break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; | |
| 1468 | }; | |
| 1469 | try argv_list.append(arena, b.fmt("{s}{s}", .{ | |
| 1470 | pa.prefix, | |
| 1471 | run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), | |
| 1472 | })); | |
| 1473 | }, | |
| 1474 | .output_file, .output_directory => unreachable, | |
| 1475 | } | |
| 1476 | } | |
| 1477 | ||
| 1478 | if (run.step.result_failed_command) |cmd| { | |
| 1479 | fuzz.gpa.free(cmd); | |
| 1480 | run.step.result_failed_command = null; | |
| 1481 | } | |
| 1482 | ||
| 1483 | const has_side_effects = false; | |
| 1484 | var rand_int: u64 = undefined; | |
| 1485 | io.random(@ptrCast(&rand_int)); | |
| 1486 | const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); | |
| 1487 | try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ | |
| 1488 | .progress_node = prog_node, | |
| 1489 | .watch = undefined, // not used by `runCommand` | |
| 1490 | .web_server = null, // only needed for time reports | |
| 1491 | .unit_test_timeout_ns = null, // don't time out fuzz tests for now | |
| 1492 | .gpa = fuzz.gpa, | |
| 1493 | }, .{ | |
| 1494 | .fuzz = fuzz, | |
| 1495 | }); | |
| 1496 | } | |
| 1497 | ||
| 1498 | fn populateGeneratedPaths( | |
| 1499 | arena: std.mem.Allocator, | |
| 1500 | output_placeholders: []const IndexedOutput, | |
| 1501 | captured_stdout: ?*CapturedStdIo, | |
| 1502 | captured_stderr: ?*CapturedStdIo, | |
| 1503 | cache_root: Cache.Directory, | |
| 1504 | digest: *const Cache.HexDigest, | |
| 1505 | ) !void { | |
| 1506 | for (output_placeholders) |placeholder| { | |
| 1507 | placeholder.output.generated_file.path = try cache_root.join(arena, &.{ | |
| 1508 | "o", digest, placeholder.output.basename, | |
| 1509 | }); | |
| 1510 | } | |
| 1511 | ||
| 1512 | if (captured_stdout) |captured| { | |
| 1513 | captured.output.generated_file.path = try cache_root.join(arena, &.{ | |
| 1514 | "o", digest, captured.output.basename, | |
| 1515 | }); | |
| 1516 | } | |
| 1517 | ||
| 1518 | if (captured_stderr) |captured| { | |
| 1519 | captured.output.generated_file.path = try cache_root.join(arena, &.{ | |
| 1520 | "o", digest, captured.output.basename, | |
| 1521 | }); | |
| 1522 | } | |
| 1523 | } | |
| 1524 | ||
| 1525 | fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void { | |
| 1526 | if (term) |t| switch (t) { | |
| 1527 | .exited => |code| try w.print("exited with code {d}", .{code}), | |
| 1528 | .signal => |sig| try w.print("terminated with signal {t}", .{sig}), | |
| 1529 | .stopped => |sig| try w.print("stopped with signal {t}", .{sig}), | |
| 1530 | .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}), | |
| 1531 | } else { | |
| 1532 | try w.writeAll("exited with any code"); | |
| 1533 | } | |
| 1534 | } | |
| 1535 | fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) { | |
| 1536 | return .{ .data = term }; | |
| 1537 | } | |
| 1538 | ||
| 1539 | const FuzzContext = struct { | |
| 1540 | fuzz: *std.Build.Fuzz, | |
| 1541 | }; | |
| 1542 | ||
| 1543 | fn runCommand( | |
| 1544 | run: *Run, | |
| 1545 | argv: []const []const u8, | |
| 1546 | has_side_effects: bool, | |
| 1547 | output_dir_path: []const u8, | |
| 1548 | options: Step.MakeOptions, | |
| 1549 | fuzz_context: ?FuzzContext, | |
| 1550 | ) !void { | |
| 1551 | const step = &run.step; | |
| 1552 | const b = step.owner; | |
| 1553 | const arena = b.allocator; | |
| 1554 | const gpa = options.gpa; | |
| 1555 | const io = b.graph.io; | |
| 1556 | ||
| 1557 | const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; | |
| 1558 | ||
| 1559 | try step.handleChildProcUnsupported(); | |
| 1560 | try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); | |
| 1561 | ||
| 1562 | const allow_skip = switch (run.stdio) { | |
| 1563 | .check, .zig_test => run.skip_foreign_checks, | |
| 1564 | else => false, | |
| 1565 | }; | |
| 1566 | ||
| 1567 | var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); | |
| 1568 | defer interp_argv.deinit(); | |
| 1569 | ||
| 1570 | var environ_map: EnvMap = env: { | |
| 1571 | const orig = run.environ_map orelse &b.graph.environ_map; | |
| 1572 | break :env try orig.clone(gpa); | |
| 1573 | }; | |
| 1574 | defer environ_map.deinit(); | |
| 1575 | ||
| 1576 | const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { | |
| 1577 | // InvalidExe: cpu arch mismatch | |
| 1578 | // FileNotFound: can happen with a wrong dynamic linker path | |
| 1579 | if (err == error.InvalidExe or err == error.FileNotFound) interpret: { | |
| 1580 | // TODO: learn the target from the binary directly rather than from | |
| 1581 | // relying on it being a Compile step. This will make this logic | |
| 1582 | // work even for the edge case that the binary was produced by a | |
| 1583 | // third party. | |
| 1584 | const exe = switch (run.argv.items[0]) { | |
| 1585 | .artifact => |exe| exe.artifact, | |
| 1586 | else => break :interpret, | |
| 1587 | }; | |
| 1588 | switch (exe.kind) { | |
| 1589 | .exe, .@"test" => {}, | |
| 1590 | else => break :interpret, | |
| 1591 | } | |
| 1592 | ||
| 1593 | const root_target = exe.rootModuleTarget(); | |
| 1594 | const need_cross_libc = exe.is_linking_libc and | |
| 1595 | (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); | |
| 1596 | const other_target = exe.root_module.resolved_target.?.result; | |
| 1597 | switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ | |
| 1598 | .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, | |
| 1599 | .link_libc = exe.is_linking_libc, | |
| 1600 | })) { | |
| 1601 | .native, .rosetta => { | |
| 1602 | if (allow_skip) return error.MakeSkipped; | |
| 1603 | break :interpret; | |
| 1604 | }, | |
| 1605 | .wine => |bin_name| { | |
| 1606 | if (b.enable_wine) { | |
| 1607 | try interp_argv.append(bin_name); | |
| 1608 | try interp_argv.appendSlice(argv); | |
| 1609 | ||
| 1610 | // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but | |
| 1611 | // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. | |
| 1612 | if (environ_map.get("WINEDEBUG") == null) { | |
| 1613 | try environ_map.put("WINEDEBUG", "-all"); | |
| 1614 | } | |
| 1615 | } else { | |
| 1616 | return failForeign(run, "-fwine", argv[0], exe); | |
| 1617 | } | |
| 1618 | }, | |
| 1619 | .qemu => |bin_name| { | |
| 1620 | if (b.enable_qemu) { | |
| 1621 | try interp_argv.append(bin_name); | |
| 1622 | ||
| 1623 | if (need_cross_libc) { | |
| 1624 | if (b.libc_runtimes_dir) |dir| { | |
| 1625 | try interp_argv.append("-L"); | |
| 1626 | try interp_argv.append(b.pathJoin(&.{ | |
| 1627 | dir, | |
| 1628 | try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( | |
| 1629 | b.allocator, | |
| 1630 | root_target.cpu.arch, | |
| 1631 | root_target.os.tag, | |
| 1632 | root_target.abi, | |
| 1633 | ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( | |
| 1634 | b.allocator, | |
| 1635 | root_target.cpu.arch, | |
| 1636 | root_target.abi, | |
| 1637 | ) else unreachable, | |
| 1638 | })); | |
| 1639 | } else return failForeign(run, "--libc-runtimes", argv[0], exe); | |
| 1640 | } | |
| 1641 | ||
| 1642 | try interp_argv.appendSlice(argv); | |
| 1643 | } else return failForeign(run, "-fqemu", argv[0], exe); | |
| 1644 | }, | |
| 1645 | .darling => |bin_name| { | |
| 1646 | if (b.enable_darling) { | |
| 1647 | try interp_argv.append(bin_name); | |
| 1648 | try interp_argv.appendSlice(argv); | |
| 1649 | } else { | |
| 1650 | return failForeign(run, "-fdarling", argv[0], exe); | |
| 1651 | } | |
| 1652 | }, | |
| 1653 | .wasmtime => |bin_name| { | |
| 1654 | if (b.enable_wasmtime) { | |
| 1655 | try interp_argv.append(bin_name); | |
| 1656 | try interp_argv.append("--dir=."); | |
| 1657 | // Wasmtime doeesn't inherit environment variables from the parent process | |
| 1658 | // by default. '-S inherit-env' was added in Wasmtime version 20. | |
| 1659 | try interp_argv.append("-Sinherit-env"); | |
| 1660 | try interp_argv.append(argv[0]); | |
| 1661 | try interp_argv.appendSlice(argv[1..]); | |
| 1662 | } else { | |
| 1663 | return failForeign(run, "-fwasmtime", argv[0], exe); | |
| 1664 | } | |
| 1665 | }, | |
| 1666 | .bad_dl => |foreign_dl| { | |
| 1667 | if (allow_skip) return error.MakeSkipped; | |
| 1668 | ||
| 1669 | const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; | |
| 1670 | ||
| 1671 | return step.fail( | |
| 1672 | \\the host system is unable to execute binaries from the target | |
| 1673 | \\ because the host dynamic linker is '{s}', | |
| 1674 | \\ while the target dynamic linker is '{s}'. | |
| 1675 | \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step | |
| 1676 | , .{ host_dl, foreign_dl }); | |
| 1677 | }, | |
| 1678 | .bad_os_or_cpu => { | |
| 1679 | if (allow_skip) return error.MakeSkipped; | |
| 1680 | ||
| 1681 | const host_name = try b.graph.host.result.zigTriple(b.allocator); | |
| 1682 | const foreign_name = try root_target.zigTriple(b.allocator); | |
| 1683 | ||
| 1684 | return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ | |
| 1685 | host_name, foreign_name, | |
| 1686 | }); | |
| 1687 | }, | |
| 1688 | } | |
| 1689 | ||
| 1690 | if (root_target.os.tag == .windows) { | |
| 1691 | // On Windows we don't have rpaths so we have to add .dll search paths to PATH | |
| 1692 | run.addPathForDynLibs(exe); | |
| 1693 | } | |
| 1694 | ||
| 1695 | gpa.free(step.result_failed_command.?); | |
| 1696 | step.result_failed_command = null; | |
| 1697 | try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); | |
| 1698 | ||
| 1699 | break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { | |
| 1700 | if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; | |
| 1701 | if (e == error.MakeFailed) return error.MakeFailed; // error already reported | |
| 1702 | return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); | |
| 1703 | }; | |
| 1704 | } | |
| 1705 | if (err == error.MakeFailed) return error.MakeFailed; // error already reported | |
| 1706 | ||
| 1707 | return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); | |
| 1708 | }; | |
| 1709 | ||
| 1710 | const generic_result = opt_generic_result orelse { | |
| 1711 | assert(run.stdio == .zig_test); | |
| 1712 | // Specific errors have already been reported, and test results are populated. All we need | |
| 1713 | // to do is report step failure if any test failed. | |
| 1714 | if (!step.test_results.isSuccess()) return error.MakeFailed; | |
| 1715 | return; | |
| 1716 | }; | |
| 1717 | ||
| 1718 | assert(fuzz_context == null); | |
| 1719 | assert(run.stdio != .zig_test); | |
| 1720 | ||
| 1721 | // Capture stdout and stderr to GeneratedFile objects. | |
| 1722 | const Stream = struct { | |
| 1723 | captured: ?*CapturedStdIo, | |
| 1724 | bytes: ?[]const u8, | |
| 1725 | }; | |
| 1726 | for ([_]Stream{ | |
| 1727 | .{ | |
| 1728 | .captured = run.captured_stdout, | |
| 1729 | .bytes = generic_result.stdout, | |
| 1730 | }, | |
| 1731 | .{ | |
| 1732 | .captured = run.captured_stderr, | |
| 1733 | .bytes = generic_result.stderr, | |
| 1734 | }, | |
| 1735 | }) |stream| { | |
| 1736 | if (stream.captured) |captured| { | |
| 1737 | const output_components = .{ output_dir_path, captured.output.basename }; | |
| 1738 | const output_path = try b.cache_root.join(arena, &output_components); | |
| 1739 | captured.output.generated_file.path = output_path; | |
| 1740 | ||
| 1741 | const sub_path = b.pathJoin(&output_components); | |
| 1742 | const sub_path_dirname = Dir.path.dirname(sub_path).?; | |
| 1743 | b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { | |
| 1744 | return step.fail("unable to make path '{f}{s}': {s}", .{ | |
| 1745 | b.cache_root, sub_path_dirname, @errorName(err), | |
| 1746 | }); | |
| 1747 | }; | |
| 1748 | const data = switch (captured.trim_whitespace) { | |
| 1749 | .none => stream.bytes.?, | |
| 1750 | .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), | |
| 1751 | .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), | |
| 1752 | .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), | |
| 1753 | }; | |
| 1754 | b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { | |
| 1755 | return step.fail("unable to write file '{f}{s}': {s}", .{ | |
| 1756 | b.cache_root, sub_path, @errorName(err), | |
| 1757 | }); | |
| 1758 | }; | |
| 1759 | } | |
| 1760 | } | |
| 1761 | ||
| 1762 | switch (run.stdio) { | |
| 1763 | .zig_test => unreachable, | |
| 1764 | .check => |checks| for (checks.items) |check| switch (check) { | |
| 1765 | .expect_stderr_exact => |expected_bytes| { | |
| 1766 | if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { | |
| 1767 | return step.fail( | |
| 1768 | \\========= expected this stderr: ========= | |
| 1769 | \\{s} | |
| 1770 | \\========= but found: ==================== | |
| 1771 | \\{s} | |
| 1772 | , .{ | |
| 1773 | expected_bytes, | |
| 1774 | generic_result.stderr.?, | |
| 1775 | }); | |
| 1776 | } | |
| 1777 | }, | |
| 1778 | .expect_stderr_match => |match| { | |
| 1779 | if (mem.find(u8, generic_result.stderr.?, match) == null) { | |
| 1780 | return step.fail( | |
| 1781 | \\========= expected to find in stderr: ========= | |
| 1782 | \\{s} | |
| 1783 | \\========= but stderr does not contain it: ===== | |
| 1784 | \\{s} | |
| 1785 | , .{ | |
| 1786 | match, | |
| 1787 | generic_result.stderr.?, | |
| 1788 | }); | |
| 1789 | } | |
| 1790 | }, | |
| 1791 | .expect_stdout_exact => |expected_bytes| { | |
| 1792 | if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { | |
| 1793 | return step.fail( | |
| 1794 | \\========= expected this stdout: ========= | |
| 1795 | \\{s} | |
| 1796 | \\========= but found: ==================== | |
| 1797 | \\{s} | |
| 1798 | , .{ | |
| 1799 | expected_bytes, | |
| 1800 | generic_result.stdout.?, | |
| 1801 | }); | |
| 1802 | } | |
| 1803 | }, | |
| 1804 | .expect_stdout_match => |match| { | |
| 1805 | if (mem.find(u8, generic_result.stdout.?, match) == null) { | |
| 1806 | return step.fail( | |
| 1807 | \\========= expected to find in stdout: ========= | |
| 1808 | \\{s} | |
| 1809 | \\========= but stdout does not contain it: ===== | |
| 1810 | \\{s} | |
| 1811 | , .{ | |
| 1812 | match, | |
| 1813 | generic_result.stdout.?, | |
| 1814 | }); | |
| 1815 | } | |
| 1816 | }, | |
| 1817 | .expect_term => |expected_term| { | |
| 1818 | if (!termMatches(expected_term, generic_result.term)) { | |
| 1819 | return step.fail("process {f} (expected {f})", .{ | |
| 1820 | fmtTerm(generic_result.term), | |
| 1821 | fmtTerm(expected_term), | |
| 1822 | }); | |
| 1823 | } | |
| 1824 | }, | |
| 1825 | }, | |
| 1826 | else => { | |
| 1827 | // On failure, report captured stderr like normal standard error output. | |
| 1828 | const bad_exit = switch (generic_result.term) { | |
| 1829 | .exited => |code| code != 0, | |
| 1830 | .signal, .stopped, .unknown => true, | |
| 1831 | }; | |
| 1832 | if (bad_exit) { | |
| 1833 | if (generic_result.stderr) |bytes| { | |
| 1834 | run.step.result_stderr = bytes; | |
| 1835 | } | |
| 1836 | } | |
| 1837 | ||
| 1838 | try step.handleChildProcessTerm(generic_result.term); | |
| 1839 | }, | |
| 1840 | } | |
| 1841 | } | |
| 1842 | ||
| 1843 | const EvalGenericResult = struct { | |
| 1844 | term: process.Child.Term, | |
| 1845 | stdout: ?[]const u8, | |
| 1846 | stderr: ?[]const u8, | |
| 1847 | }; | |
| 1848 | ||
| 1849 | fn spawnChildAndCollect( | |
| 1850 | run: *Run, | |
| 1851 | argv: []const []const u8, | |
| 1852 | environ_map: *EnvMap, | |
| 1853 | has_side_effects: bool, | |
| 1854 | options: Step.MakeOptions, | |
| 1855 | fuzz_context: ?FuzzContext, | |
| 1856 | ) !?EvalGenericResult { | |
| 1857 | const b = run.step.owner; | |
| 1858 | const graph = b.graph; | |
| 1859 | const io = graph.io; | |
| 1860 | ||
| 1861 | if (fuzz_context != null) { | |
| 1862 | assert(!has_side_effects); | |
| 1863 | assert(run.stdio == .zig_test); | |
| 1864 | } | |
| 1865 | ||
| 1866 | const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; | |
| 1867 | ||
| 1868 | // If an error occurs, it's caused by this command: | |
| 1869 | assert(run.step.result_failed_command == null); | |
| 1870 | run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ | |
| 1871 | .child = environ_map, | |
| 1872 | .parent = &graph.environ_map, | |
| 1873 | }, argv); | |
| 1874 | ||
| 1875 | var spawn_options: process.SpawnOptions = .{ | |
| 1876 | .argv = argv, | |
| 1877 | .cwd = child_cwd, | |
| 1878 | .environ_map = environ_map, | |
| 1879 | .request_resource_usage_statistics = true, | |
| 1880 | .stdin = if (run.stdin != .none) s: { | |
| 1881 | assert(run.stdio != .inherit); | |
| 1882 | break :s .pipe; | |
| 1883 | } else switch (run.stdio) { | |
| 1884 | .infer_from_args => if (has_side_effects) .inherit else .ignore, | |
| 1885 | .inherit => .inherit, | |
| 1886 | .check => .ignore, | |
| 1887 | .zig_test => .pipe, | |
| 1888 | }, | |
| 1889 | .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { | |
| 1890 | .infer_from_args => if (has_side_effects) .inherit else .ignore, | |
| 1891 | .inherit => .inherit, | |
| 1892 | .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, | |
| 1893 | .zig_test => .pipe, | |
| 1894 | }, | |
| 1895 | .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { | |
| 1896 | .infer_from_args => if (has_side_effects) .inherit else .pipe, | |
| 1897 | .inherit => .inherit, | |
| 1898 | .check => .pipe, | |
| 1899 | .zig_test => .pipe, | |
| 1900 | }, | |
| 1901 | }; | |
| 1902 | ||
| 1903 | if (run.stdio == .zig_test) { | |
| 1904 | const started: Io.Clock.Timestamp = .now(io, .awake); | |
| 1905 | const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { | |
| 1906 | error.Canceled => |e| return e, | |
| 1907 | else => |e| e, | |
| 1908 | }; | |
| 1909 | run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); | |
| 1910 | try result; | |
| 1911 | return null; | |
| 1912 | } else { | |
| 1913 | const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; | |
| 1914 | if (!run.disable_zig_progress and !inherit) { | |
| 1915 | spawn_options.progress_node = options.progress_node; | |
| 1916 | } | |
| 1917 | const terminal_mode: Io.Terminal.Mode = if (inherit) m: { | |
| 1918 | const stderr = try io.lockStderr(&.{}, graph.stderr_mode); | |
| 1919 | break :m stderr.terminal_mode; | |
| 1920 | } else .no_color; | |
| 1921 | defer if (inherit) io.unlockStderr(); | |
| 1922 | try setColorEnvironmentVariables(run, environ_map, terminal_mode); | |
| 1923 | ||
| 1924 | const started: Io.Clock.Timestamp = .now(io, .awake); | |
| 1925 | const result = evalGeneric(run, spawn_options) catch |err| switch (err) { | |
| 1926 | error.Canceled => |e| return e, | |
| 1927 | else => |e| e, | |
| 1928 | }; | |
| 1929 | run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); | |
| 1930 | return try result; | |
| 1931 | } | |
| 1932 | } | |
| 1933 | ||
| 1934 | fn hashStdIo(hh: *Cache.HashHelper, stdio: StdIo) void { | |
| 1935 | switch (stdio) { | |
| 1936 | .infer_from_args, .inherit, .zig_test => {}, | |
| 1937 | .check => |checks| for (checks.items) |check| { | |
| 1938 | hh.add(@as(std.meta.Tag(StdIo.Check), check)); | |
| 1939 | switch (check) { | |
| 1940 | .expect_stderr_exact, | |
| 1941 | .expect_stderr_match, | |
| 1942 | .expect_stdout_exact, | |
| 1943 | .expect_stdout_match, | |
| 1944 | => |s| hh.addBytes(s), | |
| 1945 | ||
| 1946 | .expect_term => |term| { | |
| 1947 | hh.add(@as(std.meta.Tag(process.Child.Term), term)); | |
| 1948 | switch (term) { | |
| 1949 | inline .exited, .signal, .stopped => |x| hh.add(x), | |
| 1950 | .unknown => |x| hh.add(x), | |
| 1951 | } | |
| 1952 | }, | |
| 1953 | } | |
| 1954 | }, | |
| 1955 | } | |
| 1956 | } | |
| 1957 | fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { | |
| 1958 | return if (expected) |e| switch (e) { | |
| 1959 | .exited => |expected_code| switch (actual) { | |
| 1960 | .exited => |actual_code| expected_code == actual_code, | |
| 1961 | else => false, | |
| 1962 | }, | |
| 1963 | .signal => |expected_sig| switch (actual) { | |
| 1964 | .signal => |actual_sig| expected_sig == actual_sig, | |
| 1965 | else => false, | |
| 1966 | }, | |
| 1967 | .stopped => |expected_sig| switch (actual) { | |
| 1968 | .stopped => |actual_sig| expected_sig == actual_sig, | |
| 1969 | else => false, | |
| 1970 | }, | |
| 1971 | .unknown => |expected_code| switch (actual) { | |
| 1972 | .unknown => |actual_code| expected_code == actual_code, | |
| 1973 | else => false, | |
| 1974 | }, | |
| 1975 | } else switch (actual) { | |
| 1976 | .exited => true, | |
| 1977 | else => false, | |
| 1978 | }; | |
| 1979 | } | |
| 1980 | ||
| 1981 | fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { | |
| 1982 | color: switch (run.color) { | |
| 1983 | .manual => {}, | |
| 1984 | .enable => { | |
| 1985 | try environ_map.put("CLICOLOR_FORCE", "1"); | |
| 1986 | _ = environ_map.swapRemove("NO_COLOR"); | |
| 1987 | }, | |
| 1988 | .disable => { | |
| 1989 | try environ_map.put("NO_COLOR", "1"); | |
| 1990 | _ = environ_map.swapRemove("CLICOLOR_FORCE"); | |
| 1991 | }, | |
| 1992 | .inherit => switch (terminal_mode) { | |
| 1993 | .no_color, .windows_api => continue :color .disable, | |
| 1994 | .escape_codes => continue :color .enable, | |
| 1995 | }, | |
| 1996 | .auto => { | |
| 1997 | const capture_stderr = run.captured_stderr != null or switch (run.stdio) { | |
| 1998 | .check => |checks| checksContainStderr(checks.items), | |
| 1999 | .infer_from_args, .inherit, .zig_test => false, | |
| 2000 | }; | |
| 2001 | if (capture_stderr) { | |
| 2002 | continue :color .disable; | |
| 2003 | } else { | |
| 2004 | continue :color .inherit; | |
| 2005 | } | |
| 2006 | }, | |
| 2007 | } | |
| 2008 | } | |
| 2009 | ||
| 2010 | fn checksContainStdout(checks: []const StdIo.Check) bool { | |
| 2011 | for (checks) |check| switch (check) { | |
| 2012 | .expect_stderr_exact, | |
| 2013 | .expect_stderr_match, | |
| 2014 | .expect_term, | |
| 2015 | => continue, | |
| 2016 | ||
| 2017 | .expect_stdout_exact, | |
| 2018 | .expect_stdout_match, | |
| 2019 | => return true, | |
| 2020 | }; | |
| 2021 | return false; | |
| 2022 | } | |
| 2023 | ||
| 2024 | fn checksContainStderr(checks: []const StdIo.Check) bool { | |
| 2025 | for (checks) |check| switch (check) { | |
| 2026 | .expect_stdout_exact, | |
| 2027 | .expect_stdout_match, | |
| 2028 | .expect_term, | |
| 2029 | => continue, | |
| 2030 | ||
| 2031 | .expect_stderr_exact, | |
| 2032 | .expect_stderr_match, | |
| 2033 | => return true, | |
| 2034 | }; | |
| 2035 | return false; | |
| 2036 | } | |
| 2037 | ||
| 2038 | /// Returns whether the Run step has side effects *other than* updating the output arguments. | |
| 2039 | fn hasSideEffects(run: Run) bool { | |
| 2040 | if (run.has_side_effects) return true; | |
| 2041 | return switch (run.stdio) { | |
| 2042 | .infer_from_args => !run.hasAnyOutputArgs(), | |
| 2043 | .inherit => true, | |
| 2044 | .check => false, | |
| 2045 | .zig_test => false, | |
| 2046 | }; | |
| 2047 | } | |
| 2048 | ||
| 2049 | fn hasAnyOutputArgs(run: Run) bool { | |
| 2050 | if (run.captured_stdout != null) return true; | |
| 2051 | if (run.captured_stderr != null) return true; | |
| 2052 | for (run.argv.items) |arg| switch (arg) { | |
| 2053 | .output_file, .output_directory => return true, | |
| 2054 | else => continue, | |
| 2055 | }; | |
| 2056 | return false; | |
| 2057 | } | |
| 2058 | ||
| 2059 | /// If `path` is cwd-relative, make it relative to the cwd of the child instead. | |
| 2060 | /// | |
| 2061 | /// Whenever a path is included in the argv of a child, it should be put through this function first | |
| 2062 | /// to make sure the child doesn't see paths relative to a cwd other than its own. | |
| 2063 | fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { | |
| 2064 | const b = run.step.owner; | |
| 2065 | const graph = b.graph; | |
| 2066 | const arena = graph.arena; | |
| 2067 | ||
| 2068 | const path_str = path.toString(arena) catch @panic("OOM"); | |
| 2069 | if (Dir.path.isAbsolute(path_str)) { | |
| 2070 | // Absolute paths don't need changing. | |
| 2071 | return path_str; | |
| 2072 | } | |
| 2073 | const child_cwd_rel: []const u8 = rel: { | |
| 2074 | const child_lazy_cwd = run.cwd orelse break :rel path_str; | |
| 2075 | const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); | |
| 2076 | // Convert it from relative to *our* cwd, to relative to the *child's* cwd. | |
| 2077 | break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); | |
| 2078 | }; | |
| 2079 | // Not every path can be made relative, e.g. if the path and the child cwd are on different | |
| 2080 | // disk designators on Windows. In that case, `relative` will return an absolute path which we can | |
| 2081 | // just return. | |
| 2082 | if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel; | |
| 2083 | ||
| 2084 | // We're not done yet. In some cases this path must be prefixed with './': | |
| 2085 | // * On POSIX, the executable name cannot be a single component like 'foo' | |
| 2086 | // * Some executables might treat a leading '-' like a flag, which we must avoid | |
| 2087 | // There's no harm in it, so just *always* apply this prefix. | |
| 2088 | return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); | |
| 2089 | } | |
| 2090 | ||
| 2091 | fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { | |
| 2092 | const b = run.step.owner; | |
| 2093 | const compiles = artifact.getCompileDependencies(true); | |
| 2094 | for (compiles) |compile| { | |
| 2095 | if (compile.root_module.resolved_target.?.result.os.tag == .windows and | |
| 2096 | compile.isDynamicLibrary()) | |
| 2097 | { | |
| 2098 | addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); | |
| 2099 | } | |
| 2100 | } | |
| 2101 | } | |
| 2102 | ||
| 2103 | fn failForeign( | |
| 2104 | run: *Run, | |
| 2105 | suggested_flag: []const u8, | |
| 2106 | argv0: []const u8, | |
| 2107 | exe: *Step.Compile, | |
| 2108 | ) error{ MakeFailed, MakeSkipped, OutOfMemory } { | |
| 2109 | switch (run.stdio) { | |
| 2110 | .check, .zig_test => { | |
| 2111 | if (run.skip_foreign_checks) | |
| 2112 | return error.MakeSkipped; | |
| 2113 | ||
| 2114 | const b = run.step.owner; | |
| 2115 | const host_name = try b.graph.host.result.zigTriple(b.allocator); | |
| 2116 | const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); | |
| 2117 | ||
| 2118 | return run.step.fail( | |
| 2119 | \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) | |
| 2120 | \\ consider using {s} or enabling skip_foreign_checks in the Run step | |
| 2121 | , .{ argv0, foreign_name, host_name, suggested_flag }); | |
| 2122 | }, | |
| 2123 | else => { | |
| 2124 | return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); | |
| 2125 | }, | |
| 2126 | } | |
| 2127 | } |
lib/compiler/maker/Step/WriteFile.zig created+206
| ... | ... | @@ -0,0 +1,206 @@ |
| 1 | ||
| 2 | fn make(step: *Step, options: Step.MakeOptions) !void { | |
| 3 | _ = options; | |
| 4 | const b = step.owner; | |
| 5 | const graph = b.graph; | |
| 6 | const io = graph.io; | |
| 7 | const arena = b.allocator; | |
| 8 | const gpa = graph.cache.gpa; | |
| 9 | const write_file: *WriteFile = @fieldParentPtr("step", step); | |
| 10 | ||
| 11 | const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); | |
| 12 | var open_dirs_count: usize = 0; | |
| 13 | defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); | |
| 14 | ||
| 15 | switch (write_file.mode) { | |
| 16 | .whole_cached => { | |
| 17 | step.clearWatchInputs(); | |
| 18 | ||
| 19 | // The cache is used here not really as a way to speed things up - because writing | |
| 20 | // the data to a file would probably be very fast - but as a way to find a canonical | |
| 21 | // location to put build artifacts. | |
| 22 | ||
| 23 | // If, for example, a hard-coded path was used as the location to put WriteFile | |
| 24 | // files, then two WriteFiles executing in parallel might clobber each other. | |
| 25 | ||
| 26 | var man = b.graph.cache.obtain(); | |
| 27 | defer man.deinit(); | |
| 28 | ||
| 29 | for (write_file.files.items) |file| { | |
| 30 | man.hash.addBytes(file.sub_path); | |
| 31 | ||
| 32 | switch (file.contents) { | |
| 33 | .bytes => |bytes| { | |
| 34 | man.hash.addBytes(bytes); | |
| 35 | }, | |
| 36 | .copy => |lazy_path| { | |
| 37 | const path = lazy_path.getPath3(b, step); | |
| 38 | _ = try man.addFilePath(path, null); | |
| 39 | try step.addWatchInput(lazy_path); | |
| 40 | }, | |
| 41 | } | |
| 42 | } | |
| 43 | ||
| 44 | for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { | |
| 45 | man.hash.addBytes(dir.sub_path); | |
| 46 | for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); | |
| 47 | if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); | |
| 48 | ||
| 49 | const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); | |
| 50 | const src_dir_path = dir.source.getPath3(b, step); | |
| 51 | ||
| 52 | var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { | |
| 53 | return step.fail("unable to open source directory '{f}': {s}", .{ | |
| 54 | src_dir_path, @errorName(err), | |
| 55 | }); | |
| 56 | }; | |
| 57 | open_dir_cache_elem.* = src_dir; | |
| 58 | open_dirs_count += 1; | |
| 59 | ||
| 60 | var it = try src_dir.walk(gpa); | |
| 61 | defer it.deinit(); | |
| 62 | while (try it.next(io)) |entry| { | |
| 63 | if (!dir.options.pathIncluded(entry.path)) continue; | |
| 64 | ||
| 65 | switch (entry.kind) { | |
| 66 | .directory => { | |
| 67 | if (need_derived_inputs) { | |
| 68 | const entry_path = try src_dir_path.join(arena, entry.path); | |
| 69 | try step.addDirectoryWatchInputFromPath(entry_path); | |
| 70 | } | |
| 71 | }, | |
| 72 | .file => { | |
| 73 | const entry_path = try src_dir_path.join(arena, entry.path); | |
| 74 | _ = try man.addFilePath(entry_path, null); | |
| 75 | }, | |
| 76 | else => continue, | |
| 77 | } | |
| 78 | } | |
| 79 | } | |
| 80 | ||
| 81 | if (try step.cacheHit(&man)) { | |
| 82 | const digest = man.final(); | |
| 83 | write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); | |
| 84 | assert(step.result_cached); | |
| 85 | return; | |
| 86 | } | |
| 87 | ||
| 88 | const digest = man.final(); | |
| 89 | const cache_path = "o" ++ Dir.path.sep_str ++ digest; | |
| 90 | ||
| 91 | write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); | |
| 92 | ||
| 93 | try operate(write_file, open_dir_cache, .{ | |
| 94 | .root_dir = b.cache_root, | |
| 95 | .sub_path = cache_path, | |
| 96 | }); | |
| 97 | ||
| 98 | try step.writeManifest(&man); | |
| 99 | }, | |
| 100 | .tmp => { | |
| 101 | step.result_cached = false; | |
| 102 | ||
| 103 | var rand_int: u64 = undefined; | |
| 104 | io.random(@ptrCast(&rand_int)); | |
| 105 | const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); | |
| 106 | ||
| 107 | write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); | |
| 108 | ||
| 109 | try operate(write_file, open_dir_cache, .{ | |
| 110 | .root_dir = b.cache_root, | |
| 111 | .sub_path = tmp_dir_sub_path, | |
| 112 | }); | |
| 113 | }, | |
| 114 | .mutate => |lp| { | |
| 115 | step.result_cached = false; | |
| 116 | const root_path = try lp.getPath4(b, step); | |
| 117 | write_file.generated_directory.path = try root_path.toString(arena); | |
| 118 | try operate(write_file, open_dir_cache, root_path); | |
| 119 | }, | |
| 120 | } | |
| 121 | } | |
| 122 | ||
| 123 | fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { | |
| 124 | const step = &write_file.step; | |
| 125 | const b = step.owner; | |
| 126 | const io = b.graph.io; | |
| 127 | const gpa = b.graph.cache.gpa; | |
| 128 | const arena = b.allocator; | |
| 129 | ||
| 130 | var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| | |
| 131 | return step.fail("unable to make path {f}: {t}", .{ root_path, err }); | |
| 132 | defer cache_dir.close(io); | |
| 133 | ||
| 134 | for (write_file.files.items) |file| { | |
| 135 | if (Dir.path.dirname(file.sub_path)) |dirname| { | |
| 136 | cache_dir.createDirPath(io, dirname) catch |err| { | |
| 137 | return step.fail("unable to make path '{f}{c}{s}': {t}", .{ | |
| 138 | root_path, Dir.path.sep, dirname, err, | |
| 139 | }); | |
| 140 | }; | |
| 141 | } | |
| 142 | switch (file.contents) { | |
| 143 | .bytes => |bytes| { | |
| 144 | cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { | |
| 145 | return step.fail("unable to write file '{f}{c}{s}': {t}", .{ | |
| 146 | root_path, Dir.path.sep, file.sub_path, err, | |
| 147 | }); | |
| 148 | }; | |
| 149 | }, | |
| 150 | .copy => |file_source| { | |
| 151 | const source_path = file_source.getPath2(b, step); | |
| 152 | const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { | |
| 153 | return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ | |
| 154 | source_path, root_path, Dir.path.sep, file.sub_path, err, | |
| 155 | }); | |
| 156 | }; | |
| 157 | // At this point we already will mark the step as a cache miss. | |
| 158 | // But this is kind of a partial cache hit since individual | |
| 159 | // file copies may be avoided. Oh well, this information is | |
| 160 | // discarded. | |
| 161 | _ = prev_status; | |
| 162 | }, | |
| 163 | } | |
| 164 | } | |
| 165 | ||
| 166 | for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { | |
| 167 | const src_dir_path = dir.source.getPath3(b, step); | |
| 168 | const dest_dirname = dir.sub_path; | |
| 169 | ||
| 170 | if (dest_dirname.len != 0) { | |
| 171 | cache_dir.createDirPath(io, dest_dirname) catch |err| { | |
| 172 | return step.fail("unable to make path '{f}{c}{s}': {t}", .{ | |
| 173 | root_path, Dir.path.sep, dest_dirname, err, | |
| 174 | }); | |
| 175 | }; | |
| 176 | } | |
| 177 | ||
| 178 | var it = try already_open_dir.walk(gpa); | |
| 179 | defer it.deinit(); | |
| 180 | while (try it.next(io)) |entry| { | |
| 181 | if (!dir.options.pathIncluded(entry.path)) continue; | |
| 182 | ||
| 183 | const src_entry_path = try src_dir_path.join(arena, entry.path); | |
| 184 | const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); | |
| 185 | switch (entry.kind) { | |
| 186 | .directory => try cache_dir.createDirPath(io, dest_path), | |
| 187 | .file => { | |
| 188 | const prev_status = Io.Dir.updateFile( | |
| 189 | src_entry_path.root_dir.handle, | |
| 190 | io, | |
| 191 | src_entry_path.sub_path, | |
| 192 | cache_dir, | |
| 193 | dest_path, | |
| 194 | .{}, | |
| 195 | ) catch |err| { | |
| 196 | return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ | |
| 197 | src_entry_path, root_path, Dir.path.sep, dest_path, err, | |
| 198 | }); | |
| 199 | }; | |
| 200 | _ = prev_status; | |
| 201 | }, | |
| 202 | else => continue, | |
| 203 | } | |
| 204 | } | |
| 205 | } | |
| 206 | } |
lib/compiler/maker/Watch.zig created+968
| ... | ... | @@ -0,0 +1,968 @@ |
| 1 | const builtin = @import("builtin"); | |
| 2 | ||
| 3 | const std = @import("../std.zig"); | |
| 4 | const Io = std.Io; | |
| 5 | const Step = std.Build.Step; | |
| 6 | const Allocator = std.mem.Allocator; | |
| 7 | const assert = std.debug.assert; | |
| 8 | const fatal = std.process.fatal; | |
| 9 | const Watch = @This(); | |
| 10 | const FsEvents = @import("Watch/FsEvents.zig"); | |
| 11 | ||
| 12 | os: Os, | |
| 13 | /// The number to show as the number of directories being watched. | |
| 14 | dir_count: usize, | |
| 15 | // These fields are common to most implementations so are kept here for simplicity. | |
| 16 | // They are `undefined` on implementations which do not utilize then. | |
| 17 | dir_table: DirTable, | |
| 18 | generation: Generation, | |
| 19 | ||
| 20 | pub const have_impl = Os != void; | |
| 21 | ||
| 22 | /// Key is the directory to watch which contains one or more files we are | |
| 23 | /// interested in noticing changes to. | |
| 24 | /// | |
| 25 | /// Value is generation. | |
| 26 | const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false); | |
| 27 | ||
| 28 | /// Special key of "." means any changes in this directory trigger the steps. | |
| 29 | const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); | |
| 30 | const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation); | |
| 31 | ||
| 32 | const Generation = u8; | |
| 33 | ||
| 34 | const Hash = std.hash.Wyhash; | |
| 35 | const Cache = std.Build.Cache; | |
| 36 | ||
| 37 | const Os = switch (builtin.os.tag) { | |
| 38 | .linux => struct { | |
| 39 | const posix = std.posix; | |
| 40 | ||
| 41 | /// Keyed differently but indexes correspond 1:1 with `dir_table`. | |
| 42 | handle_table: HandleTable, | |
| 43 | /// fanotify file descriptors are keyed by mount id since marks | |
| 44 | /// are limited to a single filesystem. | |
| 45 | poll_fds: std.AutoArrayHashMapUnmanaged(MountId, posix.pollfd), | |
| 46 | ||
| 47 | const MountId = i32; | |
| 48 | const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, struct { mount_id: MountId, reaction_set: ReactionSet }, FileHandle.Adapter, false); | |
| 49 | ||
| 50 | const fan_mask: std.os.linux.fanotify.MarkMask = .{ | |
| 51 | .CLOSE_WRITE = true, | |
| 52 | .CREATE = true, | |
| 53 | .DELETE = true, | |
| 54 | .DELETE_SELF = true, | |
| 55 | .EVENT_ON_CHILD = true, | |
| 56 | .MOVED_FROM = true, | |
| 57 | .MOVED_TO = true, | |
| 58 | .MOVE_SELF = true, | |
| 59 | .ONDIR = true, | |
| 60 | }; | |
| 61 | ||
| 62 | const FileHandle = struct { | |
| 63 | handle: *align(1) std.os.linux.file_handle, | |
| 64 | ||
| 65 | fn clone(lfh: FileHandle, gpa: Allocator) Allocator.Error!FileHandle { | |
| 66 | const bytes = lfh.slice(); | |
| 67 | const new_ptr = try gpa.alignedAlloc( | |
| 68 | u8, | |
| 69 | .of(std.os.linux.file_handle), | |
| 70 | @sizeOf(std.os.linux.file_handle) + bytes.len, | |
| 71 | ); | |
| 72 | const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); | |
| 73 | new_header.* = lfh.handle.*; | |
| 74 | const new: FileHandle = .{ .handle = new_header }; | |
| 75 | @memcpy(new.slice(), lfh.slice()); | |
| 76 | return new; | |
| 77 | } | |
| 78 | ||
| 79 | fn destroy(lfh: FileHandle, gpa: Allocator) void { | |
| 80 | const ptr: [*]u8 = @ptrCast(lfh.handle); | |
| 81 | const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; | |
| 82 | return gpa.free(allocated_slice); | |
| 83 | } | |
| 84 | ||
| 85 | fn slice(lfh: FileHandle) []u8 { | |
| 86 | const ptr: [*]u8 = &lfh.handle.f_handle; | |
| 87 | return ptr[0..lfh.handle.handle_bytes]; | |
| 88 | } | |
| 89 | ||
| 90 | const Adapter = struct { | |
| 91 | pub fn hash(self: Adapter, a: FileHandle) u32 { | |
| 92 | _ = self; | |
| 93 | const unsigned_type: u32 = @bitCast(a.handle.handle_type); | |
| 94 | return @truncate(Hash.hash(unsigned_type, a.slice())); | |
| 95 | } | |
| 96 | pub fn eql(self: Adapter, a: FileHandle, b: FileHandle, b_index: usize) bool { | |
| 97 | _ = self; | |
| 98 | _ = b_index; | |
| 99 | return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice()); | |
| 100 | } | |
| 101 | }; | |
| 102 | }; | |
| 103 | ||
| 104 | fn init(cwd_path: []const u8) !Watch { | |
| 105 | _ = cwd_path; | |
| 106 | return .{ | |
| 107 | .dir_table = .{}, | |
| 108 | .dir_count = 0, | |
| 109 | .os = switch (builtin.os.tag) { | |
| 110 | .linux => .{ | |
| 111 | .handle_table = .{}, | |
| 112 | .poll_fds = .{}, | |
| 113 | }, | |
| 114 | else => {}, | |
| 115 | }, | |
| 116 | .generation = 0, | |
| 117 | }; | |
| 118 | } | |
| 119 | ||
| 120 | fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path, mount_id: *MountId) !FileHandle { | |
| 121 | var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; | |
| 122 | var buf: [std.fs.max_path_bytes]u8 = undefined; | |
| 123 | const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{ | |
| 124 | path.sub_path, | |
| 125 | }) catch return error.NameTooLong; | |
| 126 | const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); | |
| 127 | stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); | |
| 128 | try posix.name_to_handle_at(path.root_dir.handle.handle, adjusted_path, stack_ptr, mount_id, std.os.linux.AT.HANDLE_FID); | |
| 129 | const stack_lfh: FileHandle = .{ .handle = stack_ptr }; | |
| 130 | return stack_lfh.clone(gpa); | |
| 131 | } | |
| 132 | ||
| 133 | fn markDirtySteps(w: *Watch, gpa: Allocator, fan_fd: posix.fd_t) !bool { | |
| 134 | const fanotify = std.os.linux.fanotify; | |
| 135 | const M = fanotify.event_metadata; | |
| 136 | var events_buf: [256 + 4096]u8 = undefined; | |
| 137 | var any_dirty = false; | |
| 138 | while (true) { | |
| 139 | var len = posix.read(fan_fd, &events_buf) catch |err| switch (err) { | |
| 140 | error.WouldBlock => return any_dirty, | |
| 141 | else => |e| return e, | |
| 142 | }; | |
| 143 | var meta: [*]align(1) M = @ptrCast(&events_buf); | |
| 144 | while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ | |
| 145 | len -= meta[0].event_len; | |
| 146 | meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); | |
| 147 | }) { | |
| 148 | assert(meta[0].vers == M.VERSION); | |
| 149 | if (meta[0].mask.Q_OVERFLOW) { | |
| 150 | any_dirty = true; | |
| 151 | std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); | |
| 152 | markAllFilesDirty(w, gpa); | |
| 153 | return true; | |
| 154 | } | |
| 155 | const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); | |
| 156 | switch (fid.hdr.info_type) { | |
| 157 | .DFID_NAME => { | |
| 158 | const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); | |
| 159 | const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); | |
| 160 | const file_name = std.mem.span(file_name_z); | |
| 161 | const lfh: FileHandle = .{ .handle = file_handle }; | |
| 162 | if (w.os.handle_table.getPtr(lfh)) |value| { | |
| 163 | if (value.reaction_set.getPtr(".")) |glob_set| | |
| 164 | any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); | |
| 165 | if (value.reaction_set.getPtr(file_name)) |step_set| | |
| 166 | any_dirty = markStepSetDirty(gpa, step_set, any_dirty); | |
| 167 | } | |
| 168 | }, | |
| 169 | else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), | |
| 170 | } | |
| 171 | } | |
| 172 | } | |
| 173 | } | |
| 174 | ||
| 175 | fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 176 | // Add missing marks and note persisted ones. | |
| 177 | for (steps) |step| { | |
| 178 | for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { | |
| 179 | const reaction_set = rs: { | |
| 180 | const gop = try w.dir_table.getOrPut(gpa, path); | |
| 181 | if (!gop.found_existing) { | |
| 182 | var mount_id: MountId = undefined; | |
| 183 | const dir_handle = getDirHandle(gpa, path, &mount_id) catch |err| switch (err) { | |
| 184 | error.FileNotFound => { | |
| 185 | std.debug.assert(w.dir_table.swapRemove(path)); | |
| 186 | continue; | |
| 187 | }, | |
| 188 | else => return err, | |
| 189 | }; | |
| 190 | const fan_fd = blk: { | |
| 191 | const fd_gop = try w.os.poll_fds.getOrPut(gpa, mount_id); | |
| 192 | if (!fd_gop.found_existing) { | |
| 193 | const fan_fd = std.posix.fanotify_init(.{ | |
| 194 | .CLASS = .NOTIF, | |
| 195 | .CLOEXEC = true, | |
| 196 | .NONBLOCK = true, | |
| 197 | .REPORT_NAME = true, | |
| 198 | .REPORT_DIR_FID = true, | |
| 199 | .REPORT_FID = true, | |
| 200 | .REPORT_TARGET_FID = true, | |
| 201 | }, 0) catch |err| switch (err) { | |
| 202 | error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}), | |
| 203 | else => |e| return e, | |
| 204 | }; | |
| 205 | fd_gop.value_ptr.* = .{ | |
| 206 | .fd = fan_fd, | |
| 207 | .events = std.posix.POLL.IN, | |
| 208 | .revents = undefined, | |
| 209 | }; | |
| 210 | } | |
| 211 | break :blk fd_gop.value_ptr.*.fd; | |
| 212 | }; | |
| 213 | // `dir_handle` may already be present in the table in | |
| 214 | // the case that we have multiple Cache.Path instances | |
| 215 | // that compare inequal but ultimately point to the same | |
| 216 | // directory on the file system. | |
| 217 | // In such case, we must revert adding this directory, but keep | |
| 218 | // the additions to the step set. | |
| 219 | const dh_gop = try w.os.handle_table.getOrPut(gpa, dir_handle); | |
| 220 | if (dh_gop.found_existing) { | |
| 221 | _ = w.dir_table.pop(); | |
| 222 | } else { | |
| 223 | assert(dh_gop.index == gop.index); | |
| 224 | dh_gop.value_ptr.* = .{ .mount_id = mount_id, .reaction_set = .{} }; | |
| 225 | posix.fanotify_mark(fan_fd, .{ | |
| 226 | .ADD = true, | |
| 227 | .ONLYDIR = true, | |
| 228 | }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| { | |
| 229 | fatal("unable to watch {f}: {s}", .{ path, @errorName(err) }); | |
| 230 | }; | |
| 231 | } | |
| 232 | break :rs &dh_gop.value_ptr.reaction_set; | |
| 233 | } | |
| 234 | break :rs &w.os.handle_table.values()[gop.index].reaction_set; | |
| 235 | }; | |
| 236 | for (files.items) |basename| { | |
| 237 | const gop = try reaction_set.getOrPut(gpa, basename); | |
| 238 | if (!gop.found_existing) gop.value_ptr.* = .{}; | |
| 239 | try gop.value_ptr.put(gpa, step, w.generation); | |
| 240 | } | |
| 241 | } | |
| 242 | } | |
| 243 | ||
| 244 | { | |
| 245 | // Remove marks for files that are no longer inputs. | |
| 246 | var i: usize = 0; | |
| 247 | while (i < w.os.handle_table.entries.len) { | |
| 248 | { | |
| 249 | const reaction_set = &w.os.handle_table.values()[i].reaction_set; | |
| 250 | var step_set_i: usize = 0; | |
| 251 | while (step_set_i < reaction_set.entries.len) { | |
| 252 | const step_set = &reaction_set.values()[step_set_i]; | |
| 253 | var dirent_i: usize = 0; | |
| 254 | while (dirent_i < step_set.entries.len) { | |
| 255 | const generations = step_set.values(); | |
| 256 | if (generations[dirent_i] == w.generation) { | |
| 257 | dirent_i += 1; | |
| 258 | continue; | |
| 259 | } | |
| 260 | step_set.swapRemoveAt(dirent_i); | |
| 261 | } | |
| 262 | if (step_set.entries.len > 0) { | |
| 263 | step_set_i += 1; | |
| 264 | continue; | |
| 265 | } | |
| 266 | reaction_set.swapRemoveAt(step_set_i); | |
| 267 | } | |
| 268 | if (reaction_set.entries.len > 0) { | |
| 269 | i += 1; | |
| 270 | continue; | |
| 271 | } | |
| 272 | } | |
| 273 | ||
| 274 | const path = w.dir_table.keys()[i]; | |
| 275 | ||
| 276 | const mount_id = w.os.handle_table.values()[i].mount_id; | |
| 277 | const fan_fd = w.os.poll_fds.getEntry(mount_id).?.value_ptr.fd; | |
| 278 | posix.fanotify_mark(fan_fd, .{ | |
| 279 | .REMOVE = true, | |
| 280 | .ONLYDIR = true, | |
| 281 | }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| switch (err) { | |
| 282 | error.FileNotFound => {}, // Expected, harmless. | |
| 283 | else => |e| std.log.warn("unable to unwatch '{f}': {s}", .{ path, @errorName(e) }), | |
| 284 | }; | |
| 285 | ||
| 286 | w.dir_table.swapRemoveAt(i); | |
| 287 | w.os.handle_table.swapRemoveAt(i); | |
| 288 | } | |
| 289 | w.generation +%= 1; | |
| 290 | } | |
| 291 | w.dir_count = w.dir_table.count(); | |
| 292 | } | |
| 293 | ||
| 294 | fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 295 | _ = io; | |
| 296 | const events_len = try std.posix.poll(w.os.poll_fds.values(), timeout.to_i32_ms()); | |
| 297 | if (events_len == 0) | |
| 298 | return .timeout; | |
| 299 | for (w.os.poll_fds.values()) |poll_fd| { | |
| 300 | if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try markDirtySteps(w, gpa, poll_fd.fd)) | |
| 301 | return .dirty; | |
| 302 | } | |
| 303 | return .clean; | |
| 304 | } | |
| 305 | }, | |
| 306 | .windows => struct { | |
| 307 | const windows = std.os.windows; | |
| 308 | ||
| 309 | /// Keyed differently but indexes correspond 1:1 with `dir_table`. | |
| 310 | handle_table: std.ArrayHashMapUnmanaged(*Directory, void, Directory.TableAdapter, false), | |
| 311 | ready_dirs: std.DoublyLinkedList, | |
| 312 | ||
| 313 | const FileId = struct { | |
| 314 | volumeSerialNumber: windows.ULONG, | |
| 315 | indexNumber: windows.LARGE_INTEGER, | |
| 316 | }; | |
| 317 | ||
| 318 | const Directory = struct { | |
| 319 | reaction_set: ReactionSet, | |
| 320 | id: FileId, | |
| 321 | file: Io.File, | |
| 322 | state: enum { idle, listening, ready }, | |
| 323 | iosb: windows.IO_STATUS_BLOCK, | |
| 324 | // 64 KB is the packet size limit when monitoring over a network. | |
| 325 | // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw#remarks | |
| 326 | buffer: [64 * 1024]u8 align(@alignOf(windows.FILE.NOTIFY.INFORMATION)), | |
| 327 | ready_node: std.DoublyLinkedList.Node, | |
| 328 | ||
| 329 | /// Start listening for events, buffer field will be overwritten eventually. | |
| 330 | fn startListening(dir: *Directory, w: *Watch) !void { | |
| 331 | assert(dir.file.flags.nonblocking); | |
| 332 | assert(dir.state == .idle); | |
| 333 | switch (windows.ntdll.NtNotifyChangeDirectoryFileEx( | |
| 334 | dir.file.handle, | |
| 335 | null, | |
| 336 | ¬ifyApc, | |
| 337 | w, | |
| 338 | &dir.iosb, | |
| 339 | &dir.buffer, | |
| 340 | dir.buffer.len, | |
| 341 | .{ | |
| 342 | .FILE_NAME = true, | |
| 343 | .DIR_NAME = true, | |
| 344 | .SIZE = true, | |
| 345 | .LAST_WRITE = true, | |
| 346 | .CREATION = true, | |
| 347 | }, | |
| 348 | .FALSE, | |
| 349 | .Notify, | |
| 350 | )) { | |
| 351 | .SUCCESS, .PENDING => dir.state = .listening, | |
| 352 | .ILLEGAL_FUNCTION => return error.ReadDirectoryChangesUnsupported, | |
| 353 | else => |status| return windows.unexpectedStatus(status), | |
| 354 | } | |
| 355 | } | |
| 356 | ||
| 357 | fn notifyApc(apc_context: ?*anyopaque, iosb: *windows.IO_STATUS_BLOCK, _: windows.ULONG) align(std.Io.Threaded.apc_align) callconv(.winapi) void { | |
| 358 | const w: *Watch = @ptrCast(@alignCast(apc_context)); | |
| 359 | const dir: *Directory = @fieldParentPtr("iosb", iosb); | |
| 360 | assert(iosb.u.Status != .PENDING); | |
| 361 | assert(dir.state == .listening); | |
| 362 | w.os.ready_dirs.append(&dir.ready_node); | |
| 363 | dir.state = .ready; | |
| 364 | } | |
| 365 | ||
| 366 | fn init(gpa: Allocator, path: Cache.Path) !*Directory { | |
| 367 | // The following code is a drawn out NtCreateFile call. (mostly adapted from Io.Dir.makeOpenDirAccessMaskW) | |
| 368 | // It's necessary in order to get the specific flags that are required when calling ReadDirectoryChangesW. | |
| 369 | var dir_handle: windows.HANDLE = undefined; | |
| 370 | const root_fd = path.root_dir.handle.handle; | |
| 371 | const sub_path = path.subPathOrDot(); | |
| 372 | const sub_path_w = try Io.Threaded.sliceToPrefixedFileW(root_fd, sub_path, .{}); // TODO eliminate this call | |
| 373 | var iosb: windows.IO_STATUS_BLOCK = undefined; | |
| 374 | switch (windows.ntdll.NtCreateFile( | |
| 375 | &dir_handle, | |
| 376 | .{ | |
| 377 | .SPECIFIC = .{ .FILE_DIRECTORY = .{ | |
| 378 | .LIST = true, | |
| 379 | } }, | |
| 380 | .STANDARD = .{ .SYNCHRONIZE = true }, | |
| 381 | .GENERIC = .{ .READ = true }, | |
| 382 | }, | |
| 383 | &.{ | |
| 384 | .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w.span())) null else root_fd, | |
| 385 | .ObjectName = @constCast(&sub_path_w.string()), | |
| 386 | }, | |
| 387 | &iosb, | |
| 388 | null, | |
| 389 | .{}, | |
| 390 | .VALID_FLAGS, | |
| 391 | .OPEN, | |
| 392 | .{ | |
| 393 | .DIRECTORY_FILE = true, | |
| 394 | .IO = .ASYNCHRONOUS, | |
| 395 | .OPEN_FOR_BACKUP_INTENT = true, | |
| 396 | }, | |
| 397 | null, | |
| 398 | 0, | |
| 399 | )) { | |
| 400 | .SUCCESS => {}, | |
| 401 | .OBJECT_NAME_INVALID => return error.BadPathName, | |
| 402 | .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, | |
| 403 | .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, | |
| 404 | .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, | |
| 405 | .NOT_A_DIRECTORY => return error.NotDir, | |
| 406 | // This can happen if the directory has 'List folder contents' permission set to 'Deny' | |
| 407 | .ACCESS_DENIED => return error.AccessDenied, | |
| 408 | .INVALID_PARAMETER => unreachable, | |
| 409 | else => |rc| return windows.unexpectedStatus(rc), | |
| 410 | } | |
| 411 | assert(dir_handle != windows.INVALID_HANDLE_VALUE); | |
| 412 | errdefer windows.CloseHandle(dir_handle); | |
| 413 | ||
| 414 | const dir_id = try getFileId(dir_handle); | |
| 415 | ||
| 416 | const dir = try gpa.create(Directory); | |
| 417 | dir.* = .{ | |
| 418 | .reaction_set = .empty, | |
| 419 | .id = dir_id, | |
| 420 | .file = .{ .handle = dir_handle, .flags = .{ .nonblocking = true } }, | |
| 421 | .state = .idle, | |
| 422 | .iosb = undefined, | |
| 423 | .buffer = undefined, | |
| 424 | .ready_node = undefined, | |
| 425 | }; | |
| 426 | return dir; | |
| 427 | } | |
| 428 | ||
| 429 | fn deinit(dir: *Directory, gpa: Allocator, w: *Watch) void { | |
| 430 | state: switch (dir.state) { | |
| 431 | .idle => {}, | |
| 432 | .listening => { | |
| 433 | var cancel_iosb: windows.IO_STATUS_BLOCK = undefined; | |
| 434 | _ = windows.ntdll.NtCancelIoFileEx(dir.file.handle, &dir.iosb, &cancel_iosb); | |
| 435 | while (switch (dir.state) { | |
| 436 | .idle => unreachable, | |
| 437 | .listening => true, | |
| 438 | .ready => false, | |
| 439 | }) Io.Threaded.waitForApcOrAlert(); | |
| 440 | continue :state .ready; | |
| 441 | }, | |
| 442 | .ready => w.os.ready_dirs.remove(&dir.ready_node), | |
| 443 | } | |
| 444 | windows.CloseHandle(dir.file.handle); | |
| 445 | gpa.destroy(dir); | |
| 446 | } | |
| 447 | ||
| 448 | /// Useful to make `*Directory` a key in `std.ArrayHashMap`. | |
| 449 | const TableAdapter = struct { | |
| 450 | pub fn hash(_: TableAdapter, lhs_dir: *Directory) u32 { | |
| 451 | return @truncate(Hash.hash(lhs_dir.id.volumeSerialNumber, @ptrCast(&lhs_dir.id.indexNumber))); | |
| 452 | } | |
| 453 | pub fn eql(_: TableAdapter, lhs_dir: *Directory, rhs_dir: *Directory, rhs_index: usize) bool { | |
| 454 | _ = rhs_index; | |
| 455 | return lhs_dir.id.volumeSerialNumber == rhs_dir.id.volumeSerialNumber and | |
| 456 | lhs_dir.id.indexNumber == rhs_dir.id.indexNumber; | |
| 457 | } | |
| 458 | }; | |
| 459 | }; | |
| 460 | ||
| 461 | fn init(cwd_path: []const u8) !Watch { | |
| 462 | _ = cwd_path; | |
| 463 | return .{ | |
| 464 | .dir_table = .{}, | |
| 465 | .dir_count = 0, | |
| 466 | .os = switch (builtin.os.tag) { | |
| 467 | .windows => .{ | |
| 468 | .handle_table = .empty, | |
| 469 | .ready_dirs = .{}, | |
| 470 | }, | |
| 471 | else => {}, | |
| 472 | }, | |
| 473 | .generation = 0, | |
| 474 | }; | |
| 475 | } | |
| 476 | ||
| 477 | fn getFileId(handle: windows.HANDLE) !FileId { | |
| 478 | var file_id: FileId = undefined; | |
| 479 | var io_status: windows.IO_STATUS_BLOCK = undefined; | |
| 480 | var volume_info: windows.FILE.FS_VOLUME_INFORMATION = undefined; | |
| 481 | switch (windows.ntdll.NtQueryVolumeInformationFile( | |
| 482 | handle, | |
| 483 | &io_status, | |
| 484 | &volume_info, | |
| 485 | @sizeOf(windows.FILE.FS_VOLUME_INFORMATION), | |
| 486 | .Volume, | |
| 487 | )) { | |
| 488 | .SUCCESS => {}, | |
| 489 | // Buffer overflow here indicates that there is more information available than was able to be stored in the buffer | |
| 490 | // size provided. This is treated as success because the type of variable-length information that this would be relevant for | |
| 491 | // (name, volume name, etc) we don't care about. | |
| 492 | .BUFFER_OVERFLOW => {}, | |
| 493 | else => |rc| return windows.unexpectedStatus(rc), | |
| 494 | } | |
| 495 | file_id.volumeSerialNumber = volume_info.VolumeSerialNumber; | |
| 496 | var internal_info: windows.FILE.INTERNAL_INFORMATION = undefined; | |
| 497 | switch (windows.ntdll.NtQueryInformationFile( | |
| 498 | handle, | |
| 499 | &io_status, | |
| 500 | &internal_info, | |
| 501 | @sizeOf(windows.FILE.INTERNAL_INFORMATION), | |
| 502 | .Internal, | |
| 503 | )) { | |
| 504 | .SUCCESS => {}, | |
| 505 | else => |rc| return windows.unexpectedStatus(rc), | |
| 506 | } | |
| 507 | file_id.indexNumber = internal_info.IndexNumber; | |
| 508 | return file_id; | |
| 509 | } | |
| 510 | ||
| 511 | fn markDirtySteps(w: *Watch, gpa: Allocator, dir: *Directory) !bool { | |
| 512 | var any_dirty = false; | |
| 513 | const bytes_returned = dir.iosb.Information; | |
| 514 | if (bytes_returned == 0) { | |
| 515 | std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); | |
| 516 | markAllFilesDirty(w, gpa); | |
| 517 | try dir.startListening(w); | |
| 518 | return true; | |
| 519 | } | |
| 520 | var file_name_buf: [std.fs.max_path_bytes]u8 = undefined; | |
| 521 | var offset: usize = 0; | |
| 522 | while (true) { | |
| 523 | const notify: *windows.FILE.NOTIFY.INFORMATION = @ptrCast(@alignCast(&dir.buffer[offset])); | |
| 524 | const file_name = file_name_buf[0..std.unicode.wtf16LeToWtf8(&file_name_buf, notify.fileName())]; | |
| 525 | if (dir.reaction_set.getPtr(".")) |glob_set| | |
| 526 | any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); | |
| 527 | if (dir.reaction_set.getPtr(file_name)) |step_set| | |
| 528 | any_dirty = markStepSetDirty(gpa, step_set, any_dirty); | |
| 529 | if (notify.NextEntryOffset == 0) | |
| 530 | break; | |
| 531 | ||
| 532 | offset += notify.NextEntryOffset; | |
| 533 | } | |
| 534 | ||
| 535 | // We call this now since at this point we have finished reading dir.buffer. | |
| 536 | try dir.startListening(w); | |
| 537 | return any_dirty; | |
| 538 | } | |
| 539 | ||
| 540 | fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 541 | // Add missing marks and note persisted ones. | |
| 542 | for (steps) |step| { | |
| 543 | for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { | |
| 544 | const dir = dir: { | |
| 545 | const gop = try w.dir_table.getOrPut(gpa, path); | |
| 546 | if (!gop.found_existing) { | |
| 547 | const dir: *Directory = try .init(gpa, path); | |
| 548 | errdefer dir.deinit(gpa, w); | |
| 549 | // `dir.id` may already be present in the table in | |
| 550 | // the case that we have multiple Cache.Path instances | |
| 551 | // that compare inequal but ultimately point to the same | |
| 552 | // directory on the file system. | |
| 553 | // In such case, we must revert adding this directory, but keep | |
| 554 | // the additions to the step set. | |
| 555 | const dh_gop = try w.os.handle_table.getOrPut(gpa, dir); | |
| 556 | if (dh_gop.found_existing) { | |
| 557 | dir.deinit(gpa, w); | |
| 558 | _ = w.dir_table.pop(); | |
| 559 | break :dir w.os.handle_table.keys()[dh_gop.index]; | |
| 560 | } else { | |
| 561 | assert(dh_gop.index == gop.index); | |
| 562 | try dir.startListening(w); | |
| 563 | break :dir dir; | |
| 564 | } | |
| 565 | } | |
| 566 | break :dir w.os.handle_table.keys()[gop.index]; | |
| 567 | }; | |
| 568 | for (files.items) |basename| { | |
| 569 | const gop = try dir.reaction_set.getOrPut(gpa, basename); | |
| 570 | if (!gop.found_existing) gop.value_ptr.* = .{}; | |
| 571 | try gop.value_ptr.put(gpa, step, w.generation); | |
| 572 | } | |
| 573 | } | |
| 574 | } | |
| 575 | ||
| 576 | { | |
| 577 | // Remove marks for files that are no longer inputs. | |
| 578 | var i: usize = 0; | |
| 579 | while (i < w.os.handle_table.entries.len) { | |
| 580 | const dir = w.os.handle_table.keys()[i]; | |
| 581 | { | |
| 582 | var step_set_i: usize = 0; | |
| 583 | while (step_set_i < dir.reaction_set.entries.len) { | |
| 584 | const step_set = &dir.reaction_set.values()[step_set_i]; | |
| 585 | var dirent_i: usize = 0; | |
| 586 | while (dirent_i < step_set.entries.len) { | |
| 587 | const generations = step_set.values(); | |
| 588 | if (generations[dirent_i] == w.generation) { | |
| 589 | dirent_i += 1; | |
| 590 | continue; | |
| 591 | } | |
| 592 | step_set.swapRemoveAt(dirent_i); | |
| 593 | } | |
| 594 | if (step_set.entries.len > 0) { | |
| 595 | step_set_i += 1; | |
| 596 | continue; | |
| 597 | } | |
| 598 | dir.reaction_set.swapRemoveAt(step_set_i); | |
| 599 | } | |
| 600 | if (dir.reaction_set.entries.len > 0) { | |
| 601 | i += 1; | |
| 602 | continue; | |
| 603 | } | |
| 604 | } | |
| 605 | ||
| 606 | w.dir_table.swapRemoveAt(i); | |
| 607 | w.os.handle_table.swapRemoveAt(i); | |
| 608 | dir.deinit(gpa, w); | |
| 609 | } | |
| 610 | w.generation +%= 1; | |
| 611 | } | |
| 612 | w.dir_count = w.dir_table.count(); | |
| 613 | } | |
| 614 | ||
| 615 | fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 616 | for (0..2) |attempt| { | |
| 617 | while (w.os.ready_dirs.popFirst()) |ready_node| { | |
| 618 | const dir: *Directory = @fieldParentPtr("ready_node", ready_node); | |
| 619 | assert(dir.state == .ready); | |
| 620 | dir.state = .idle; | |
| 621 | switch (dir.iosb.u.Status) { | |
| 622 | .SUCCESS => return if (try markDirtySteps(w, gpa, dir)) .dirty else .clean, | |
| 623 | .PENDING => unreachable, | |
| 624 | .CANCELLED => {}, | |
| 625 | else => |status| return windows.unexpectedStatus(status), | |
| 626 | } | |
| 627 | try dir.startListening(w); | |
| 628 | } | |
| 629 | try io.checkCancel(); | |
| 630 | if (attempt == 1) return .timeout; | |
| 631 | const delay_interval: windows.LARGE_INTEGER = switch (timeout) { | |
| 632 | .none => std.math.minInt(windows.LARGE_INTEGER), | |
| 633 | .ms => |ms| -@as(windows.LARGE_INTEGER, ms) * (std.time.ns_per_ms / 100), | |
| 634 | }; | |
| 635 | _ = windows.ntdll.NtDelayExecution(.TRUE, &delay_interval); | |
| 636 | } else unreachable; | |
| 637 | } | |
| 638 | }, | |
| 639 | .dragonfly, .freebsd, .netbsd, .openbsd, .ios, .tvos, .visionos, .watchos => struct { | |
| 640 | const posix = std.posix; | |
| 641 | ||
| 642 | kq_fd: i32, | |
| 643 | /// Indexes correspond 1:1 with `dir_table`. | |
| 644 | handles: std.MultiArrayList(struct { | |
| 645 | rs: ReactionSet, | |
| 646 | /// If the corresponding dir_table Path has sub_path == "", then it | |
| 647 | /// suffices as the open directory handle, and this value will be | |
| 648 | /// -1. Otherwise, it needs to be opened in update(), and will be | |
| 649 | /// stored here. | |
| 650 | dir_fd: i32, | |
| 651 | }), | |
| 652 | ||
| 653 | const dir_open_flags: posix.O = f: { | |
| 654 | var f: posix.O = .{ | |
| 655 | .ACCMODE = .RDONLY, | |
| 656 | .NOFOLLOW = false, | |
| 657 | .DIRECTORY = true, | |
| 658 | .CLOEXEC = true, | |
| 659 | }; | |
| 660 | if (@hasField(posix.O, "EVTONLY")) f.EVTONLY = true; | |
| 661 | if (@hasField(posix.O, "PATH")) f.PATH = true; | |
| 662 | break :f f; | |
| 663 | }; | |
| 664 | ||
| 665 | const EV = std.c.EV; | |
| 666 | const NOTE = std.c.NOTE; | |
| 667 | ||
| 668 | fn init(cwd_path: []const u8) !Watch { | |
| 669 | _ = cwd_path; | |
| 670 | return .{ | |
| 671 | .dir_table = .{}, | |
| 672 | .dir_count = 0, | |
| 673 | .os = .{ | |
| 674 | .kq_fd = try Io.Kqueue.createFileDescriptor(), | |
| 675 | .handles = .empty, | |
| 676 | }, | |
| 677 | .generation = 0, | |
| 678 | }; | |
| 679 | } | |
| 680 | ||
| 681 | fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 682 | const handles = &w.os.handles; | |
| 683 | for (steps) |step| { | |
| 684 | for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { | |
| 685 | const reaction_set = rs: { | |
| 686 | const gop = try w.dir_table.getOrPut(gpa, path); | |
| 687 | if (!gop.found_existing) { | |
| 688 | const skip_open_dir = path.sub_path.len == 0; | |
| 689 | const dir_fd = if (skip_open_dir) | |
| 690 | path.root_dir.handle.handle | |
| 691 | else | |
| 692 | posix.openat(path.root_dir.handle.handle, path.sub_path, dir_open_flags, 0) catch |err| { | |
| 693 | fatal("failed to open directory {f}: {t}", .{ path, err }); | |
| 694 | }; | |
| 695 | // Empirically the dir has to stay open or else no events are triggered. | |
| 696 | errdefer if (!skip_open_dir) std.Io.Threaded.closeFd(dir_fd); | |
| 697 | const changes = [1]posix.Kevent{.{ | |
| 698 | .ident = @bitCast(@as(isize, dir_fd)), | |
| 699 | .filter = std.c.EVFILT.VNODE, | |
| 700 | .flags = EV.ADD | EV.ENABLE | EV.CLEAR, | |
| 701 | .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE, | |
| 702 | .data = 0, | |
| 703 | .udata = gop.index, | |
| 704 | }}; | |
| 705 | _ = try Io.Kqueue.kevent(w.os.kq_fd, &changes, &.{}, null); | |
| 706 | assert(handles.len == gop.index); | |
| 707 | try handles.append(gpa, .{ | |
| 708 | .rs = .{}, | |
| 709 | .dir_fd = if (skip_open_dir) -1 else dir_fd, | |
| 710 | }); | |
| 711 | } | |
| 712 | ||
| 713 | break :rs &handles.items(.rs)[gop.index]; | |
| 714 | }; | |
| 715 | for (files.items) |basename| { | |
| 716 | const gop = try reaction_set.getOrPut(gpa, basename); | |
| 717 | if (!gop.found_existing) gop.value_ptr.* = .{}; | |
| 718 | try gop.value_ptr.put(gpa, step, w.generation); | |
| 719 | } | |
| 720 | } | |
| 721 | } | |
| 722 | ||
| 723 | { | |
| 724 | // Remove marks for files that are no longer inputs. | |
| 725 | var i: usize = 0; | |
| 726 | while (i < handles.len) { | |
| 727 | { | |
| 728 | const reaction_set = &handles.items(.rs)[i]; | |
| 729 | var step_set_i: usize = 0; | |
| 730 | while (step_set_i < reaction_set.entries.len) { | |
| 731 | const step_set = &reaction_set.values()[step_set_i]; | |
| 732 | var dirent_i: usize = 0; | |
| 733 | while (dirent_i < step_set.entries.len) { | |
| 734 | const generations = step_set.values(); | |
| 735 | if (generations[dirent_i] == w.generation) { | |
| 736 | dirent_i += 1; | |
| 737 | continue; | |
| 738 | } | |
| 739 | step_set.swapRemoveAt(dirent_i); | |
| 740 | } | |
| 741 | if (step_set.entries.len > 0) { | |
| 742 | step_set_i += 1; | |
| 743 | continue; | |
| 744 | } | |
| 745 | reaction_set.swapRemoveAt(step_set_i); | |
| 746 | } | |
| 747 | if (reaction_set.entries.len > 0) { | |
| 748 | i += 1; | |
| 749 | continue; | |
| 750 | } | |
| 751 | } | |
| 752 | ||
| 753 | // If the sub_path == "" then this patch has already the | |
| 754 | // dir fd that we need to use as the ident to remove the | |
| 755 | // event. If it was opened above with openat() then we need | |
| 756 | // to access that data via the dir_fd field. | |
| 757 | const path = w.dir_table.keys()[i]; | |
| 758 | const dir_fd = if (path.sub_path.len == 0) | |
| 759 | path.root_dir.handle.handle | |
| 760 | else | |
| 761 | handles.items(.dir_fd)[i]; | |
| 762 | assert(dir_fd != -1); | |
| 763 | ||
| 764 | // The changelist also needs to update the udata field of the last | |
| 765 | // event, since we are doing a swap remove, and we store the dir_table | |
| 766 | // index in the udata field. | |
| 767 | const last_dir_fd = fd: { | |
| 768 | const last_path = w.dir_table.keys()[handles.len - 1]; | |
| 769 | const last_dir_fd = if (last_path.sub_path.len == 0) | |
| 770 | last_path.root_dir.handle.handle | |
| 771 | else | |
| 772 | handles.items(.dir_fd)[handles.len - 1]; | |
| 773 | assert(last_dir_fd != -1); | |
| 774 | break :fd last_dir_fd; | |
| 775 | }; | |
| 776 | const changes = [_]posix.Kevent{ | |
| 777 | .{ | |
| 778 | .ident = @bitCast(@as(isize, dir_fd)), | |
| 779 | .filter = std.c.EVFILT.VNODE, | |
| 780 | .flags = EV.DELETE, | |
| 781 | .fflags = 0, | |
| 782 | .data = 0, | |
| 783 | .udata = i, | |
| 784 | }, | |
| 785 | .{ | |
| 786 | .ident = @bitCast(@as(isize, last_dir_fd)), | |
| 787 | .filter = std.c.EVFILT.VNODE, | |
| 788 | .flags = EV.ADD, | |
| 789 | .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE, | |
| 790 | .data = 0, | |
| 791 | .udata = i, | |
| 792 | }, | |
| 793 | }; | |
| 794 | const filtered_changes = if (i == handles.len - 1) changes[0..1] else &changes; | |
| 795 | _ = try Io.Kqueue.kevent(w.os.kq_fd, filtered_changes, &.{}, null); | |
| 796 | if (path.sub_path.len != 0) std.Io.Threaded.closeFd(dir_fd); | |
| 797 | ||
| 798 | w.dir_table.swapRemoveAt(i); | |
| 799 | handles.swapRemove(i); | |
| 800 | } | |
| 801 | w.generation +%= 1; | |
| 802 | } | |
| 803 | w.dir_count = w.dir_table.count(); | |
| 804 | } | |
| 805 | ||
| 806 | fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 807 | _ = io; | |
| 808 | var timespec_buffer: posix.timespec = undefined; | |
| 809 | var event_buffer: [100]posix.Kevent = undefined; | |
| 810 | var n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, timeout.toTimespec(×pec_buffer)); | |
| 811 | if (n == 0) return .timeout; | |
| 812 | const reaction_sets = w.os.handles.items(.rs); | |
| 813 | var any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], false); | |
| 814 | timespec_buffer = .{ .sec = 0, .nsec = 0 }; | |
| 815 | while (n == event_buffer.len) { | |
| 816 | n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, ×pec_buffer); | |
| 817 | if (n == 0) break; | |
| 818 | any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], any_dirty); | |
| 819 | } | |
| 820 | return if (any_dirty) .dirty else .clean; | |
| 821 | } | |
| 822 | ||
| 823 | fn markDirtySteps( | |
| 824 | gpa: Allocator, | |
| 825 | reaction_sets: []ReactionSet, | |
| 826 | events: []const std.c.Kevent, | |
| 827 | start_any_dirty: bool, | |
| 828 | ) bool { | |
| 829 | var any_dirty = start_any_dirty; | |
| 830 | for (events) |event| { | |
| 831 | const index: usize = @intCast(event.udata); | |
| 832 | const reaction_set = &reaction_sets[index]; | |
| 833 | // If we knew the basename of the changed file, here we would | |
| 834 | // mark only the step set dirty, and possibly the glob set: | |
| 835 | //if (reaction_set.getPtr(".")) |glob_set| | |
| 836 | // any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); | |
| 837 | //if (reaction_set.getPtr(file_name)) |step_set| | |
| 838 | // any_dirty = markStepSetDirty(gpa, step_set, any_dirty); | |
| 839 | // However we don't know the file name so just mark all the | |
| 840 | // sets dirty for this directory. | |
| 841 | for (reaction_set.values()) |*step_set| { | |
| 842 | any_dirty = markStepSetDirty(gpa, step_set, any_dirty); | |
| 843 | } | |
| 844 | } | |
| 845 | return any_dirty; | |
| 846 | } | |
| 847 | }, | |
| 848 | .macos => struct { | |
| 849 | fse: FsEvents, | |
| 850 | ||
| 851 | fn init(cwd_path: []const u8) !Watch { | |
| 852 | return .{ | |
| 853 | .os = .{ .fse = try .init(cwd_path) }, | |
| 854 | .dir_count = 0, | |
| 855 | .dir_table = undefined, | |
| 856 | .generation = undefined, | |
| 857 | }; | |
| 858 | } | |
| 859 | fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 860 | try w.os.fse.setPaths(gpa, steps); | |
| 861 | w.dir_count = w.os.fse.watch_roots.len; | |
| 862 | } | |
| 863 | fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 864 | _ = io; | |
| 865 | return w.os.fse.wait(gpa, switch (timeout) { | |
| 866 | .none => null, | |
| 867 | .ms => |ms| @as(u64, ms) * std.time.ns_per_ms, | |
| 868 | }); | |
| 869 | } | |
| 870 | }, | |
| 871 | else => void, | |
| 872 | }; | |
| 873 | ||
| 874 | pub fn init(cwd_path: []const u8) !Watch { | |
| 875 | return Os.init(cwd_path); | |
| 876 | } | |
| 877 | ||
| 878 | pub const Match = struct { | |
| 879 | /// Relative to the watched directory, the file path that triggers this | |
| 880 | /// match. | |
| 881 | basename: []const u8, | |
| 882 | /// The step to re-run when file corresponding to `basename` is changed. | |
| 883 | step: *Step, | |
| 884 | ||
| 885 | pub const Context = struct { | |
| 886 | pub fn hash(self: Context, a: Match) u32 { | |
| 887 | _ = self; | |
| 888 | var hasher = Hash.init(0); | |
| 889 | std.hash.autoHash(&hasher, a.step); | |
| 890 | hasher.update(a.basename); | |
| 891 | return @truncate(hasher.final()); | |
| 892 | } | |
| 893 | pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { | |
| 894 | _ = self; | |
| 895 | _ = b_index; | |
| 896 | return a.step == b.step and std.mem.eql(u8, a.basename, b.basename); | |
| 897 | } | |
| 898 | }; | |
| 899 | }; | |
| 900 | ||
| 901 | fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { | |
| 902 | for (switch (builtin.os.tag) { | |
| 903 | .windows => w.os.handle_table.keys(), | |
| 904 | else => w.os.handle_table.values(), | |
| 905 | }) |item| { | |
| 906 | const reaction_set = switch (builtin.os.tag) { | |
| 907 | .linux, .windows => item.reaction_set, | |
| 908 | else => item, | |
| 909 | }; | |
| 910 | for (reaction_set.values()) |step_set| { | |
| 911 | for (step_set.keys()) |step| { | |
| 912 | _ = step.invalidateResult(gpa); | |
| 913 | } | |
| 914 | } | |
| 915 | } | |
| 916 | } | |
| 917 | ||
| 918 | fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool { | |
| 919 | var this_any_dirty = false; | |
| 920 | for (step_set.keys()) |step| { | |
| 921 | if (step.invalidateResult(gpa)) this_any_dirty = true; | |
| 922 | } | |
| 923 | return any_dirty or this_any_dirty; | |
| 924 | } | |
| 925 | ||
| 926 | pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 927 | return Os.update(w, gpa, steps); | |
| 928 | } | |
| 929 | ||
| 930 | pub const Timeout = union(enum) { | |
| 931 | none, | |
| 932 | ms: u16, | |
| 933 | ||
| 934 | pub fn to_i32_ms(t: Timeout) i32 { | |
| 935 | return switch (t) { | |
| 936 | .none => -1, | |
| 937 | .ms => |ms| ms, | |
| 938 | }; | |
| 939 | } | |
| 940 | ||
| 941 | pub fn toTimespec(t: Timeout, buf: *std.posix.timespec) ?*std.posix.timespec { | |
| 942 | return switch (t) { | |
| 943 | .none => null, | |
| 944 | .ms => |ms_u16| { | |
| 945 | const ms: isize = ms_u16; | |
| 946 | buf.* = .{ | |
| 947 | .sec = @divTrunc(ms, std.time.ms_per_s), | |
| 948 | .nsec = @rem(ms, std.time.ms_per_s) * std.time.ns_per_ms, | |
| 949 | }; | |
| 950 | return buf; | |
| 951 | }, | |
| 952 | }; | |
| 953 | } | |
| 954 | }; | |
| 955 | ||
| 956 | pub const WaitResult = enum { | |
| 957 | timeout, | |
| 958 | /// File system watching triggered on files that were marked as inputs to at least one Step. | |
| 959 | /// Relevant steps have been marked dirty. | |
| 960 | dirty, | |
| 961 | /// File system watching triggered but none of the events were relevant to | |
| 962 | /// what we are listening to. There is nothing to do. | |
| 963 | clean, | |
| 964 | }; | |
| 965 | ||
| 966 | pub fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 967 | return Os.wait(w, gpa, io, timeout); | |
| 968 | } |
lib/compiler/maker/Watch/FsEvents.zig created+479
| ... | ... | @@ -0,0 +1,479 @@ |
| 1 | //! An implementation of file-system watching based on the `FSEventStream` API in macOS. | |
| 2 | //! While macOS supports kqueue, it does not allow detecting changes to files without | |
| 3 | //! placing watches on each individual file, meaning FD limits are reached incredibly | |
| 4 | //! quickly. The File System Events API works differently: it implements *recursive* | |
| 5 | //! directory watches, managed by a system service. Rather than being in libc, the API is | |
| 6 | //! exposed by the CoreServices framework. To avoid a compile dependency on the framework | |
| 7 | //! bundle, we dynamically load CoreServices with `std.DynLib`. | |
| 8 | //! | |
| 9 | //! While the logic in this file *is* specialized to `std.Build.Watch`, efforts have been | |
| 10 | //! made to keep that specialization to a minimum. Other use cases could be served with | |
| 11 | //! relatively minimal modifications to the `watch_paths` field and its usages (in | |
| 12 | //! particular the `setPaths` function). We avoid using the global GCD dispatch queue in | |
| 13 | //! favour of creating our own and synchronizing with an explicit semaphore, meaning this | |
| 14 | //! logic is thread-safe and does not affect process-global state. | |
| 15 | //! | |
| 16 | //! In theory, this API is quite good at avoiding filesystem race conditions. In practice, | |
| 17 | //! the logic that would avoid them is currently disabled, because the build system kind | |
| 18 | //! of relies on them at the time of writing to avoid redundant work -- see the comment at | |
| 19 | //! the top of `wait` for details. | |
| 20 | ||
| 21 | const enable_debug_logs = false; | |
| 22 | ||
| 23 | core_services: std.DynLib, | |
| 24 | resolved_symbols: ResolvedSymbols, | |
| 25 | ||
| 26 | paths_arena: std.heap.ArenaAllocator.State, | |
| 27 | /// The roots of the recursive watches. FSEvents has relatively small limits on the number | |
| 28 | /// of watched paths, so this slice must not be too long. The paths themselves are allocated | |
| 29 | /// into `paths_arena`, but this slice is allocated into the GPA. | |
| 30 | watch_roots: [][:0]const u8, | |
| 31 | /// All of the paths being watched. Value is the set of steps which depend on the file/directory. | |
| 32 | /// Keys and values are in `paths_arena`, but this map is allocated into the GPA. | |
| 33 | watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step), | |
| 34 | ||
| 35 | /// The semaphore we use to block the thread calling `wait` until the callback determines a relevant | |
| 36 | /// event has occurred. This is retained across `wait` calls for simplicity and efficiency. | |
| 37 | waiting_semaphore: dispatch.semaphore_t, | |
| 38 | /// This dispatch queue is created by us and executes serially. It exists exclusively to trigger the | |
| 39 | /// callbacks of the FSEventStream we create. This is not in use outside of `wait`, but is retained | |
| 40 | /// across `wait` calls for simplicity and efficiency. | |
| 41 | dispatch_queue: dispatch.queue_t, | |
| 42 | /// In theory, this field avoids race conditions. In practice, it is essentially unused at the time | |
| 43 | /// of writing. See the comment at the start of `wait` for details. | |
| 44 | since_event: FSEventStreamEventId, | |
| 45 | ||
| 46 | cwd_path: []const u8, | |
| 47 | ||
| 48 | /// All of the symbols we pull from the `dlopen`ed CoreServices framework. If any of these symbols | |
| 49 | /// is not present, `init` will close the framework and return an error. | |
| 50 | const ResolvedSymbols = struct { | |
| 51 | FSEventStreamCreate: *const fn ( | |
| 52 | allocator: CFAllocatorRef, | |
| 53 | callback: FSEventStreamCallback, | |
| 54 | ctx: ?*const FSEventStreamContext, | |
| 55 | paths_to_watch: CFArrayRef, | |
| 56 | since_when: FSEventStreamEventId, | |
| 57 | latency: CFTimeInterval, | |
| 58 | flags: FSEventStreamCreateFlags, | |
| 59 | ) callconv(.c) FSEventStreamRef, | |
| 60 | FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch.queue_t) callconv(.c) void, | |
| 61 | FSEventStreamStart: *const fn (stream: FSEventStreamRef) callconv(.c) bool, | |
| 62 | FSEventStreamStop: *const fn (stream: FSEventStreamRef) callconv(.c) void, | |
| 63 | FSEventStreamInvalidate: *const fn (stream: FSEventStreamRef) callconv(.c) void, | |
| 64 | FSEventStreamRelease: *const fn (stream: FSEventStreamRef) callconv(.c) void, | |
| 65 | FSEventStreamGetLatestEventId: *const fn (stream: ConstFSEventStreamRef) callconv(.c) FSEventStreamEventId, | |
| 66 | FSEventsGetCurrentEventId: *const fn () callconv(.c) FSEventStreamEventId, | |
| 67 | CFRelease: *const fn (cf: *const anyopaque) callconv(.c) void, | |
| 68 | CFArrayCreate: *const fn ( | |
| 69 | allocator: CFAllocatorRef, | |
| 70 | values: [*]const usize, | |
| 71 | num_values: CFIndex, | |
| 72 | call_backs: ?*const CFArrayCallBacks, | |
| 73 | ) callconv(.c) CFArrayRef, | |
| 74 | CFStringCreateWithCString: *const fn ( | |
| 75 | alloc: CFAllocatorRef, | |
| 76 | c_str: [*:0]const u8, | |
| 77 | encoding: CFStringEncoding, | |
| 78 | ) callconv(.c) CFStringRef, | |
| 79 | CFAllocatorCreate: *const fn (allocator: CFAllocatorRef, context: *const CFAllocatorContext) callconv(.c) CFAllocatorRef, | |
| 80 | kCFAllocatorUseContext: *const CFAllocatorRef, | |
| 81 | }; | |
| 82 | ||
| 83 | pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol, SystemResources }!FsEvents { | |
| 84 | var core_services = std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/CoreServices") catch | |
| 85 | return error.OpenFrameworkFailed; | |
| 86 | errdefer core_services.close(); | |
| 87 | ||
| 88 | var resolved_symbols: ResolvedSymbols = undefined; | |
| 89 | inline for (@typeInfo(ResolvedSymbols).@"struct".fields) |f| { | |
| 90 | @field(resolved_symbols, f.name) = core_services.lookup(f.type, f.name) orelse return error.MissingCoreServicesSymbol; | |
| 91 | } | |
| 92 | ||
| 93 | return .{ | |
| 94 | .core_services = core_services, | |
| 95 | .resolved_symbols = resolved_symbols, | |
| 96 | .paths_arena = .{}, | |
| 97 | .watch_roots = &.{}, | |
| 98 | .watch_paths = .empty, | |
| 99 | .waiting_semaphore = dispatch.semaphore_create(0) orelse return error.SystemResources, | |
| 100 | .dispatch_queue = dispatch.queue_create("zig-watch", .SERIAL()) orelse return error.SystemResources, | |
| 101 | // Not `.since_now`, because this means we can init `FsEvents` *before* we do work in order | |
| 102 | // to notice any changes which happened during said work. | |
| 103 | .since_event = resolved_symbols.FSEventsGetCurrentEventId(), | |
| 104 | .cwd_path = cwd_path, | |
| 105 | }; | |
| 106 | } | |
| 107 | ||
| 108 | pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void { | |
| 109 | fse.waiting_semaphore.as_object().release(); | |
| 110 | fse.dispatch_queue.as_object().release(); | |
| 111 | fse.core_services.close(io); | |
| 112 | ||
| 113 | gpa.free(fse.watch_roots); | |
| 114 | fse.watch_paths.deinit(gpa); | |
| 115 | { | |
| 116 | var paths_arena = fse.paths_arena.promote(gpa); | |
| 117 | paths_arena.deinit(); | |
| 118 | } | |
| 119 | } | |
| 120 | ||
| 121 | pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) !void { | |
| 122 | var paths_arena_instance = fse.paths_arena.promote(gpa); | |
| 123 | defer fse.paths_arena = paths_arena_instance.state; | |
| 124 | const paths_arena = paths_arena_instance.allocator(); | |
| 125 | ||
| 126 | var need_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; | |
| 127 | defer need_dirs.deinit(gpa); | |
| 128 | ||
| 129 | fse.watch_paths.clearRetainingCapacity(); | |
| 130 | ||
| 131 | // We take `step` by pointer for a slight memory optimization in a moment. | |
| 132 | for (steps) |*step| { | |
| 133 | for (step.*.inputs.table.keys(), step.*.inputs.table.values()) |path, *files| { | |
| 134 | const resolved_dir = try std.fs.path.resolvePosix(paths_arena, &.{ | |
| 135 | fse.cwd_path, path.root_dir.path orelse ".", path.sub_path, | |
| 136 | }); | |
| 137 | try need_dirs.put(gpa, resolved_dir, {}); | |
| 138 | for (files.items) |file_name| { | |
| 139 | const watch_path = if (std.mem.eql(u8, file_name, ".")) | |
| 140 | resolved_dir | |
| 141 | else | |
| 142 | try std.fs.path.join(paths_arena, &.{ resolved_dir, file_name }); | |
| 143 | const gop = try fse.watch_paths.getOrPut(gpa, watch_path); | |
| 144 | if (gop.found_existing) { | |
| 145 | const old_steps = gop.value_ptr.*; | |
| 146 | const new_steps = try paths_arena.alloc(*std.Build.Step, old_steps.len + 1); | |
| 147 | @memcpy(new_steps[0..old_steps.len], old_steps); | |
| 148 | new_steps[old_steps.len] = step.*; | |
| 149 | gop.value_ptr.* = new_steps; | |
| 150 | } else { | |
| 151 | // This is why we captured `step` by pointer! We can avoid allocating a slice of one | |
| 152 | // step in the arena in the common case where a file is referenced by only one step. | |
| 153 | gop.value_ptr.* = step[0..1]; | |
| 154 | } | |
| 155 | } | |
| 156 | } | |
| 157 | } | |
| 158 | ||
| 159 | { | |
| 160 | // There's no point looking at directories inside other ones (e.g. "/foo" and "/foo/bar"). | |
| 161 | // To eliminate these, we'll re-add directories in order of path length with a redundancy check. | |
| 162 | const old_dirs = try gpa.dupe([]const u8, need_dirs.keys()); | |
| 163 | defer gpa.free(old_dirs); | |
| 164 | std.mem.sort([]const u8, old_dirs, {}, struct { | |
| 165 | fn lessThan(ctx: void, a: []const u8, b: []const u8) bool { | |
| 166 | ctx; | |
| 167 | return std.mem.lessThan(u8, a, b); | |
| 168 | } | |
| 169 | }.lessThan); | |
| 170 | need_dirs.clearRetainingCapacity(); | |
| 171 | for (old_dirs) |dir_path| { | |
| 172 | var it: std.fs.path.ComponentIterator(.posix, u8) = .init(dir_path); | |
| 173 | while (it.next()) |component| { | |
| 174 | if (need_dirs.contains(component.path)) { | |
| 175 | // this path is '/foo/bar/qux', but '/foo' or '/foo/bar' was already added | |
| 176 | break; | |
| 177 | } | |
| 178 | } else { | |
| 179 | need_dirs.putAssumeCapacityNoClobber(dir_path, {}); | |
| 180 | } | |
| 181 | } | |
| 182 | } | |
| 183 | ||
| 184 | // `need_dirs` is now a set of directories to watch with no redundancy. In practice, this is very | |
| 185 | // likely to have reduced it to a quite small set (e.g. it'll typically coalesce a full `src/` | |
| 186 | // directory into one entry). However, the FSEventStream API has a fairly low undocumented limit | |
| 187 | // on total watches (supposedly 4096), so we should handle the case where we exceed it. To be | |
| 188 | // safe, because this API can be a little unpredictable, we'll cap ourselves a little *below* | |
| 189 | // that known limit. | |
| 190 | if (need_dirs.count() > 2048) { | |
| 191 | // Fallback: watch the whole filesystem. This is excessive, but... it *works* :P | |
| 192 | if (enable_debug_logs) watch_log.debug("too many dirs; recursively watching root", .{}); | |
| 193 | fse.watch_roots = try gpa.realloc(fse.watch_roots, 1); | |
| 194 | fse.watch_roots[0] = "/"; | |
| 195 | } else { | |
| 196 | fse.watch_roots = try gpa.realloc(fse.watch_roots, need_dirs.count()); | |
| 197 | for (fse.watch_roots, need_dirs.keys()) |*out, in| { | |
| 198 | out.* = try paths_arena.dupeSentinel(u8, in, 0); | |
| 199 | } | |
| 200 | } | |
| 201 | if (enable_debug_logs) { | |
| 202 | watch_log.debug("watching {d} paths using {d} recursive watches:", .{ fse.watch_paths.count(), fse.watch_roots.len }); | |
| 203 | for (fse.watch_roots) |dir_path| { | |
| 204 | watch_log.debug("- '{s}'", .{dir_path}); | |
| 205 | } | |
| 206 | } | |
| 207 | } | |
| 208 | ||
| 209 | pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory, StartFailed }!std.Build.Watch.WaitResult { | |
| 210 | if (fse.watch_roots.len == 0) @panic("nothing to watch"); | |
| 211 | ||
| 212 | const rs = fse.resolved_symbols; | |
| 213 | ||
| 214 | // At the time of writing, using `since_event` in the obvious way causes redundant rebuilds | |
| 215 | // to occur, because one step modifies a file which is an input to another step. The solution | |
| 216 | // to this problem will probably be either: | |
| 217 | // | |
| 218 | // a) Don't include the output of one step as a watch input of another; only mark external | |
| 219 | // files as watch inputs. Or... | |
| 220 | // | |
| 221 | // b) Note the current event ID when a step begins, and disregard events preceding that ID | |
| 222 | // when considering whether to dirty that step in `eventCallback`. | |
| 223 | // | |
| 224 | // For now, to avoid the redundant rebuilds, we bypass this `since_event` mechanism. This does | |
| 225 | // introduce race conditions, but the other `std.Build.Watch` implementations suffer from those | |
| 226 | // too at the time of writing, so this is kind of expected. | |
| 227 | fse.since_event = .since_now; | |
| 228 | ||
| 229 | const cf_allocator = rs.CFAllocatorCreate(rs.kCFAllocatorUseContext.*, &.{ | |
| 230 | .version = 0, | |
| 231 | .info = @constCast(&gpa), | |
| 232 | .retain = null, | |
| 233 | .release = null, | |
| 234 | .copy_description = null, | |
| 235 | .allocate = &cf_alloc_callbacks.allocate, | |
| 236 | .reallocate = &cf_alloc_callbacks.reallocate, | |
| 237 | .deallocate = &cf_alloc_callbacks.deallocate, | |
| 238 | .preferred_size = null, | |
| 239 | }) orelse return error.OutOfMemory; | |
| 240 | defer rs.CFRelease(cf_allocator); | |
| 241 | ||
| 242 | const cf_paths = try gpa.alloc(?CFStringRef, fse.watch_roots.len); | |
| 243 | @memset(cf_paths, null); | |
| 244 | defer { | |
| 245 | for (cf_paths) |o| if (o) |p| rs.CFRelease(p); | |
| 246 | gpa.free(cf_paths); | |
| 247 | } | |
| 248 | for (fse.watch_roots, cf_paths) |raw_path, *cf_path| { | |
| 249 | cf_path.* = rs.CFStringCreateWithCString(cf_allocator, raw_path, .utf8); | |
| 250 | } | |
| 251 | const cf_paths_array = rs.CFArrayCreate(cf_allocator, @ptrCast(cf_paths), @intCast(cf_paths.len), null); | |
| 252 | defer rs.CFRelease(cf_paths_array); | |
| 253 | ||
| 254 | const callback_ctx: EventCallbackCtx = .{ | |
| 255 | .fse = fse, | |
| 256 | .gpa = gpa, | |
| 257 | }; | |
| 258 | const event_stream = rs.FSEventStreamCreate( | |
| 259 | null, | |
| 260 | &eventCallback, | |
| 261 | &.{ | |
| 262 | .version = 0, | |
| 263 | .info = @constCast(&callback_ctx), | |
| 264 | .retain = null, | |
| 265 | .release = null, | |
| 266 | .copy_description = null, | |
| 267 | }, | |
| 268 | cf_paths_array, | |
| 269 | fse.since_event, | |
| 270 | 0.05, // 0.05s latency; higher values increase efficiency by coalescing more events | |
| 271 | .{ .watch_root = true, .file_events = true }, | |
| 272 | ); | |
| 273 | defer rs.FSEventStreamRelease(event_stream); | |
| 274 | rs.FSEventStreamSetDispatchQueue(event_stream, fse.dispatch_queue); | |
| 275 | defer rs.FSEventStreamInvalidate(event_stream); | |
| 276 | if (!rs.FSEventStreamStart(event_stream)) return error.StartFailed; | |
| 277 | defer rs.FSEventStreamStop(event_stream); | |
| 278 | const result = fse.waiting_semaphore.wait(timeout: { | |
| 279 | const ns = timeout_ns orelse break :timeout .FOREVER; | |
| 280 | break :timeout .time(.NOW, @intCast(ns)); | |
| 281 | }); | |
| 282 | return switch (result) { | |
| 283 | 0 => .dirty, | |
| 284 | else => .timeout, | |
| 285 | }; | |
| 286 | } | |
| 287 | ||
| 288 | const cf_alloc_callbacks = struct { | |
| 289 | const log = std.log.scoped(.cf_alloc); | |
| 290 | fn allocate(size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque { | |
| 291 | if (enable_debug_logs) log.debug("allocate {d}", .{size}); | |
| 292 | _ = hint; | |
| 293 | const gpa: *const Allocator = @ptrCast(@alignCast(info)); | |
| 294 | const mem = gpa.alignedAlloc(u8, .of(usize), @intCast(size + @sizeOf(usize))) catch return null; | |
| 295 | const metadata: *usize = @ptrCast(mem); | |
| 296 | metadata.* = @intCast(size); | |
| 297 | return mem[@sizeOf(usize)..].ptr; | |
| 298 | } | |
| 299 | fn reallocate(ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque { | |
| 300 | if (enable_debug_logs) log.debug("reallocate @{*} {d}", .{ ptr, new_size }); | |
| 301 | _ = hint; | |
| 302 | if (ptr == null or new_size == 0) return null; // not a bug: documentation explicitly states that realloc on NULL should return NULL | |
| 303 | const gpa: *const Allocator = @ptrCast(@alignCast(info)); | |
| 304 | const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize)); | |
| 305 | const old_size = @as(*const usize, @ptrCast(old_base)).*; | |
| 306 | const old_mem = old_base[0 .. old_size + @sizeOf(usize)]; | |
| 307 | const new_mem = gpa.realloc(old_mem, @intCast(new_size + @sizeOf(usize))) catch return null; | |
| 308 | const metadata: *usize = @ptrCast(new_mem); | |
| 309 | metadata.* = @intCast(new_size); | |
| 310 | return new_mem[@sizeOf(usize)..].ptr; | |
| 311 | } | |
| 312 | fn deallocate(ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void { | |
| 313 | if (enable_debug_logs) log.debug("deallocate @{*}", .{ptr}); | |
| 314 | const gpa: *const Allocator = @ptrCast(@alignCast(info)); | |
| 315 | const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize)); | |
| 316 | const old_size = @as(*const usize, @ptrCast(old_base)).*; | |
| 317 | const old_mem = old_base[0 .. old_size + @sizeOf(usize)]; | |
| 318 | gpa.free(old_mem); | |
| 319 | } | |
| 320 | }; | |
| 321 | ||
| 322 | const EventCallbackCtx = struct { | |
| 323 | fse: *FsEvents, | |
| 324 | gpa: Allocator, | |
| 325 | }; | |
| 326 | ||
| 327 | fn eventCallback( | |
| 328 | stream: ConstFSEventStreamRef, | |
| 329 | client_callback_info: ?*anyopaque, | |
| 330 | num_events: usize, | |
| 331 | events_paths_ptr: *anyopaque, | |
| 332 | events_flags_ptr: [*]const FSEventStreamEventFlags, | |
| 333 | events_ids_ptr: [*]const FSEventStreamEventId, | |
| 334 | ) callconv(.c) void { | |
| 335 | const ctx: *const EventCallbackCtx = @ptrCast(@alignCast(client_callback_info)); | |
| 336 | const fse = ctx.fse; | |
| 337 | const gpa = ctx.gpa; | |
| 338 | const rs = fse.resolved_symbols; | |
| 339 | const events_paths_ptr_casted: [*]const [*:0]const u8 = @ptrCast(@alignCast(events_paths_ptr)); | |
| 340 | const events_paths = events_paths_ptr_casted[0..num_events]; | |
| 341 | const events_ids = events_ids_ptr[0..num_events]; | |
| 342 | const events_flags = events_flags_ptr[0..num_events]; | |
| 343 | var any_dirty = false; | |
| 344 | for (events_paths, events_ids, events_flags) |event_path_nts, event_id, event_flags| { | |
| 345 | _ = event_id; | |
| 346 | if (event_flags.history_done) continue; // sentinel | |
| 347 | const event_path = std.mem.span(event_path_nts); | |
| 348 | switch (event_flags.must_scan_sub_dirs) { | |
| 349 | false => { | |
| 350 | if (fse.watch_paths.get(event_path)) |steps| { | |
| 351 | assert(steps.len > 0); | |
| 352 | for (steps) |s| { | |
| 353 | if (s.invalidateResult(gpa)) any_dirty = true; | |
| 354 | } | |
| 355 | } | |
| 356 | if (std.fs.path.dirname(event_path)) |event_dirname| { | |
| 357 | // Modifying '/foo/bar' triggers the watch on '/foo'. | |
| 358 | if (fse.watch_paths.get(event_dirname)) |steps| { | |
| 359 | assert(steps.len > 0); | |
| 360 | for (steps) |s| { | |
| 361 | if (s.invalidateResult(gpa)) any_dirty = true; | |
| 362 | } | |
| 363 | } | |
| 364 | } | |
| 365 | }, | |
| 366 | true => { | |
| 367 | // This is unlikely, but can occasionally happen when bottlenecked: events have been | |
| 368 | // coalesced into one. We want to see if any of these events are actually relevant | |
| 369 | // to us. The only way we can reasonably do that in this rare edge case is iterate | |
| 370 | // the watch paths and see if any is under this directory. That's acceptable because | |
| 371 | // we would otherwise kick off a rebuild which would be clearing those paths anyway. | |
| 372 | const changed_path = std.fs.path.dirname(event_path) orelse event_path; | |
| 373 | for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| { | |
| 374 | if (dirStartsWith(watching_path, changed_path)) { | |
| 375 | for (steps) |s| { | |
| 376 | if (s.invalidateResult(gpa)) any_dirty = true; | |
| 377 | } | |
| 378 | } | |
| 379 | } | |
| 380 | }, | |
| 381 | } | |
| 382 | } | |
| 383 | if (any_dirty) { | |
| 384 | fse.since_event = rs.FSEventStreamGetLatestEventId(stream); | |
| 385 | _ = fse.waiting_semaphore.signal(); | |
| 386 | } | |
| 387 | } | |
| 388 | fn dirStartsWith(path: []const u8, prefix: []const u8) bool { | |
| 389 | if (std.mem.eql(u8, path, prefix)) return true; | |
| 390 | if (!std.mem.startsWith(u8, path, prefix)) return false; | |
| 391 | if (path[prefix.len] != '/') return false; // `path` is `/foo/barx`, `prefix` is `/foo/bar` | |
| 392 | return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar` | |
| 393 | } | |
| 394 | ||
| 395 | const CFAllocatorRef = ?*const opaque {}; | |
| 396 | const CFArrayRef = *const opaque {}; | |
| 397 | const CFStringRef = *const opaque {}; | |
| 398 | const CFTimeInterval = f64; | |
| 399 | const CFIndex = i32; | |
| 400 | const CFOptionFlags = enum(u32) { _ }; | |
| 401 | const CFAllocatorRetainCallBack = *const fn (info: ?*const anyopaque) callconv(.c) *const anyopaque; | |
| 402 | const CFAllocatorReleaseCallBack = *const fn (info: ?*const anyopaque) callconv(.c) void; | |
| 403 | const CFAllocatorCopyDescriptionCallBack = *const fn (info: ?*const anyopaque) callconv(.c) CFStringRef; | |
| 404 | const CFAllocatorAllocateCallBack = *const fn (alloc_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque; | |
| 405 | const CFAllocatorReallocateCallBack = *const fn (ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque; | |
| 406 | const CFAllocatorDeallocateCallBack = *const fn (ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void; | |
| 407 | const CFAllocatorPreferredSizeCallBack = *const fn (size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) CFIndex; | |
| 408 | const CFAllocatorContext = extern struct { | |
| 409 | version: CFIndex, | |
| 410 | info: ?*anyopaque, | |
| 411 | retain: ?CFAllocatorRetainCallBack, | |
| 412 | release: ?CFAllocatorReleaseCallBack, | |
| 413 | copy_description: ?CFAllocatorCopyDescriptionCallBack, | |
| 414 | allocate: CFAllocatorAllocateCallBack, | |
| 415 | reallocate: ?CFAllocatorReallocateCallBack, | |
| 416 | deallocate: ?CFAllocatorDeallocateCallBack, | |
| 417 | preferred_size: ?CFAllocatorPreferredSizeCallBack, | |
| 418 | }; | |
| 419 | const CFArrayCallBacks = opaque {}; | |
| 420 | const CFStringEncoding = enum(u32) { | |
| 421 | invalid_id = std.math.maxInt(u32), | |
| 422 | mac_roman = 0, | |
| 423 | windows_latin_1 = 0x500, | |
| 424 | iso_latin_1 = 0x201, | |
| 425 | next_step_latin = 0xB01, | |
| 426 | ascii = 0x600, | |
| 427 | unicode = 0x100, | |
| 428 | utf8 = 0x8000100, | |
| 429 | non_lossy_ascii = 0xBFF, | |
| 430 | }; | |
| 431 | ||
| 432 | const FSEventStreamRef = *opaque {}; | |
| 433 | const ConstFSEventStreamRef = *const @typeInfo(FSEventStreamRef).pointer.child; | |
| 434 | const FSEventStreamCallback = *const fn ( | |
| 435 | stream: ConstFSEventStreamRef, | |
| 436 | client_callback_info: ?*anyopaque, | |
| 437 | num_events: usize, | |
| 438 | event_paths: *anyopaque, | |
| 439 | event_flags: [*]const FSEventStreamEventFlags, | |
| 440 | event_ids: [*]const FSEventStreamEventId, | |
| 441 | ) callconv(.c) void; | |
| 442 | const FSEventStreamContext = extern struct { | |
| 443 | version: CFIndex, | |
| 444 | info: ?*anyopaque, | |
| 445 | retain: ?CFAllocatorRetainCallBack, | |
| 446 | release: ?CFAllocatorReleaseCallBack, | |
| 447 | copy_description: ?CFAllocatorCopyDescriptionCallBack, | |
| 448 | }; | |
| 449 | const FSEventStreamEventId = enum(u64) { | |
| 450 | since_now = std.math.maxInt(u64), | |
| 451 | _, | |
| 452 | }; | |
| 453 | const FSEventStreamCreateFlags = packed struct(u32) { | |
| 454 | use_cf_types: bool = false, | |
| 455 | no_defer: bool = false, | |
| 456 | watch_root: bool = false, | |
| 457 | ignore_self: bool = false, | |
| 458 | file_events: bool = false, | |
| 459 | _: u27 = 0, | |
| 460 | }; | |
| 461 | const FSEventStreamEventFlags = packed struct(u32) { | |
| 462 | must_scan_sub_dirs: bool, | |
| 463 | user_dropped: bool, | |
| 464 | kernel_dropped: bool, | |
| 465 | event_ids_wrapped: bool, | |
| 466 | history_done: bool, | |
| 467 | root_changed: bool, | |
| 468 | mount: bool, | |
| 469 | unmount: bool, | |
| 470 | _: u24 = 0, | |
| 471 | }; | |
| 472 | ||
| 473 | const dispatch = std.c.dispatch; | |
| 474 | const std = @import("std"); | |
| 475 | const Io = std.Io; | |
| 476 | const assert = std.debug.assert; | |
| 477 | const Allocator = std.mem.Allocator; | |
| 478 | const watch_log = std.log.scoped(.watch); | |
| 479 | const FsEvents = @This(); |
lib/compiler/maker/WebServer.zig created+926
| ... | ... | @@ -0,0 +1,926 @@ |
| 1 | gpa: Allocator, | |
| 2 | graph: *const Build.Graph, | |
| 3 | all_steps: []const *Build.Step, | |
| 4 | listen_address: net.IpAddress, | |
| 5 | root_prog_node: std.Progress.Node, | |
| 6 | watch: bool, | |
| 7 | ||
| 8 | tcp_server: ?net.Server, | |
| 9 | serve_task: ?Io.Future(Io.Cancelable!void), | |
| 10 | ||
| 11 | /// Uses `Io.Clock.awake`. | |
| 12 | base_timestamp: Io.Timestamp, | |
| 13 | /// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`. | |
| 14 | step_names_trailing: []u8, | |
| 15 | ||
| 16 | /// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps. | |
| 17 | /// Accessed atomically. | |
| 18 | step_status_bits: []u8, | |
| 19 | ||
| 20 | fuzz: ?Fuzz, | |
| 21 | time_report_mutex: Io.Mutex, | |
| 22 | time_report_msgs: [][]u8, | |
| 23 | time_report_update_times: []i64, | |
| 24 | ||
| 25 | build_status: std.atomic.Value(abi.BuildStatus), | |
| 26 | /// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate` | |
| 27 | /// to increment this value. Each client thread waits for this increment with `Io.futexWaitTimeout`, so | |
| 28 | /// `notifyUpdate` will wake those threads. Updates are sent on a short interval regardless, so it | |
| 29 | /// is recommended to only use `notifyUpdate` for changes which the user should see immediately. For | |
| 30 | /// instance, we do not call `notifyUpdate` when the number of "unique runs" in the fuzzer changes, | |
| 31 | /// because this value changes quickly so this would result in constantly spamming all clients with | |
| 32 | /// an unreasonable number of packets. | |
| 33 | update_id: std.atomic.Value(u32), | |
| 34 | ||
| 35 | runner_request_mutex: Io.Mutex, | |
| 36 | runner_request_ready_cond: Io.Condition, | |
| 37 | runner_request_empty_cond: Io.Condition, | |
| 38 | runner_request: ?RunnerRequest, | |
| 39 | ||
| 40 | /// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates | |
| 41 | /// on a fixed interval of this many milliseconds. | |
| 42 | const default_update_interval_ms = 500; | |
| 43 | ||
| 44 | pub const base_clock: Io.Clock = .awake; | |
| 45 | ||
| 46 | /// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`. | |
| 47 | pub fn notifyUpdate(ws: *WebServer) void { | |
| 48 | _ = ws.update_id.rmw(.Add, 1, .release); | |
| 49 | ws.graph.io.futexWake(u32, &ws.update_id.raw, 16); | |
| 50 | } | |
| 51 | ||
| 52 | pub const Options = struct { | |
| 53 | gpa: Allocator, | |
| 54 | graph: *const std.Build.Graph, | |
| 55 | all_steps: []const *Build.Step, | |
| 56 | root_prog_node: std.Progress.Node, | |
| 57 | watch: bool, | |
| 58 | listen_address: net.IpAddress, | |
| 59 | base_timestamp: Io.Clock.Timestamp, | |
| 60 | }; | |
| 61 | pub fn init(opts: Options) WebServer { | |
| 62 | // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` | |
| 63 | // instead of threads, so that the web server can function in single-threaded builds. | |
| 64 | comptime assert(!builtin.single_threaded); | |
| 65 | assert(opts.base_timestamp.clock == base_clock); | |
| 66 | ||
| 67 | const all_steps = opts.all_steps; | |
| 68 | ||
| 69 | const step_names_trailing = opts.gpa.alloc(u8, len: { | |
| 70 | var name_bytes: usize = 0; | |
| 71 | for (all_steps) |step| name_bytes += step.name.len; | |
| 72 | break :len name_bytes + all_steps.len * 4; | |
| 73 | }) catch @panic("out of memory"); | |
| 74 | { | |
| 75 | const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]); | |
| 76 | var idx: usize = all_steps.len * 4; | |
| 77 | for (all_steps, step_name_lens) |step, *name_len| { | |
| 78 | name_len.* = @intCast(step.name.len); | |
| 79 | @memcpy(step_names_trailing[idx..][0..step.name.len], step.name); | |
| 80 | idx += step.name.len; | |
| 81 | } | |
| 82 | assert(idx == step_names_trailing.len); | |
| 83 | } | |
| 84 | ||
| 85 | const step_status_bits = opts.gpa.alloc( | |
| 86 | u8, | |
| 87 | std.math.divCeil(usize, all_steps.len, 4) catch unreachable, | |
| 88 | ) catch @panic("out of memory"); | |
| 89 | @memset(step_status_bits, 0); | |
| 90 | ||
| 91 | const time_reports_len: usize = if (opts.graph.time_report) all_steps.len else 0; | |
| 92 | const time_report_msgs = opts.gpa.alloc([]u8, time_reports_len) catch @panic("out of memory"); | |
| 93 | const time_report_update_times = opts.gpa.alloc(i64, time_reports_len) catch @panic("out of memory"); | |
| 94 | @memset(time_report_msgs, &.{}); | |
| 95 | @memset(time_report_update_times, std.math.minInt(i64)); | |
| 96 | ||
| 97 | return .{ | |
| 98 | .gpa = opts.gpa, | |
| 99 | .graph = opts.graph, | |
| 100 | .all_steps = all_steps, | |
| 101 | .listen_address = opts.listen_address, | |
| 102 | .root_prog_node = opts.root_prog_node, | |
| 103 | .watch = opts.watch, | |
| 104 | ||
| 105 | .tcp_server = null, | |
| 106 | .serve_task = null, | |
| 107 | ||
| 108 | .base_timestamp = opts.base_timestamp.raw, | |
| 109 | .step_names_trailing = step_names_trailing, | |
| 110 | ||
| 111 | .step_status_bits = step_status_bits, | |
| 112 | ||
| 113 | .fuzz = null, | |
| 114 | .time_report_mutex = .init, | |
| 115 | .time_report_msgs = time_report_msgs, | |
| 116 | .time_report_update_times = time_report_update_times, | |
| 117 | ||
| 118 | .build_status = .init(.idle), | |
| 119 | .update_id = .init(0), | |
| 120 | ||
| 121 | .runner_request_mutex = .init, | |
| 122 | .runner_request_ready_cond = .init, | |
| 123 | .runner_request_empty_cond = .init, | |
| 124 | .runner_request = null, | |
| 125 | }; | |
| 126 | } | |
| 127 | pub fn deinit(ws: *WebServer) void { | |
| 128 | const gpa = ws.gpa; | |
| 129 | const io = ws.graph.io; | |
| 130 | ||
| 131 | gpa.free(ws.step_names_trailing); | |
| 132 | gpa.free(ws.step_status_bits); | |
| 133 | ||
| 134 | if (ws.fuzz) |*f| f.deinit(); | |
| 135 | for (ws.time_report_msgs) |msg| gpa.free(msg); | |
| 136 | gpa.free(ws.time_report_msgs); | |
| 137 | gpa.free(ws.time_report_update_times); | |
| 138 | ||
| 139 | if (ws.serve_task) |t| { | |
| 140 | if (ws.tcp_server) |*s| s.stream.close(io); | |
| 141 | t.await(); | |
| 142 | } | |
| 143 | if (ws.tcp_server) |*s| s.deinit(); | |
| 144 | ||
| 145 | gpa.free(ws.step_names_trailing); | |
| 146 | } | |
| 147 | pub fn start(ws: *WebServer) error{AlreadyReported}!void { | |
| 148 | assert(ws.tcp_server == null); | |
| 149 | assert(ws.serve_task == null); | |
| 150 | const io = ws.graph.io; | |
| 151 | ||
| 152 | ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| { | |
| 153 | log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err }); | |
| 154 | return error.AlreadyReported; | |
| 155 | }; | |
| 156 | ws.serve_task = io.concurrent(serve, .{ws}) catch |err| { | |
| 157 | log.err("unable to spawn web server thread: {t}", .{err}); | |
| 158 | ws.tcp_server.?.deinit(io); | |
| 159 | ws.tcp_server = null; | |
| 160 | return error.AlreadyReported; | |
| 161 | }; | |
| 162 | ||
| 163 | log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.socket.address}); | |
| 164 | if (ws.listen_address.getPort() == 0) { | |
| 165 | log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.socket.address}); | |
| 166 | } | |
| 167 | } | |
| 168 | fn serve(ws: *WebServer) Io.Cancelable!void { | |
| 169 | const io = ws.graph.io; | |
| 170 | var group: Io.Group = .init; | |
| 171 | defer group.cancel(io); | |
| 172 | while (true) { | |
| 173 | var stream = ws.tcp_server.?.accept(io) catch |err| switch (err) { | |
| 174 | error.Canceled => |e| return e, | |
| 175 | else => |e| { | |
| 176 | log.err("failed to accept connection: {t}", .{e}); | |
| 177 | return; | |
| 178 | }, | |
| 179 | }; | |
| 180 | group.concurrent(io, accept, .{ ws, stream }) catch |err| { | |
| 181 | log.err("unable to spawn connection thread: {t}", .{err}); | |
| 182 | stream.close(io); | |
| 183 | continue; | |
| 184 | }; | |
| 185 | } | |
| 186 | } | |
| 187 | ||
| 188 | pub fn startBuild(ws: *WebServer) void { | |
| 189 | if (ws.fuzz) |*fuzz| { | |
| 190 | fuzz.deinit(); | |
| 191 | ws.fuzz = null; | |
| 192 | } | |
| 193 | for (ws.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic); | |
| 194 | ws.build_status.store(.running, .monotonic); | |
| 195 | ws.notifyUpdate(); | |
| 196 | } | |
| 197 | ||
| 198 | pub fn updateStepStatus(ws: *WebServer, step: *Build.Step, new_status: abi.StepUpdate.Status) void { | |
| 199 | const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { | |
| 200 | if (s == step) break @intCast(i); | |
| 201 | } else unreachable; | |
| 202 | const ptr = &ws.step_status_bits[step_idx / 4]; | |
| 203 | const bit_offset: u3 = @intCast((step_idx % 4) * 2); | |
| 204 | const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset); | |
| 205 | const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset; | |
| 206 | _ = @atomicRmw(u8, ptr, .Xor, mask, .monotonic); | |
| 207 | ws.notifyUpdate(); | |
| 208 | } | |
| 209 | ||
| 210 | pub fn finishBuild(ws: *WebServer, opts: struct { | |
| 211 | fuzz: bool, | |
| 212 | }) void { | |
| 213 | if (opts.fuzz) { | |
| 214 | switch (builtin.os.tag) { | |
| 215 | // Current implementation depends on two things that need to be ported to Windows: | |
| 216 | // * Memory-mapping to share data between the fuzzer and build runner. | |
| 217 | // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving | |
| 218 | // many addresses to source locations). | |
| 219 | .windows => std.process.fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), | |
| 220 | else => {}, | |
| 221 | } | |
| 222 | if (@bitSizeOf(usize) != 64) { | |
| 223 | // Current implementation depends on posix.mmap()'s second | |
| 224 | // parameter, `length: usize`, being compatible with file system's | |
| 225 | // u64 return value. This is not the case on 32-bit platforms. | |
| 226 | // Affects or affected by issues #5185, #22523, and #22464. | |
| 227 | std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); | |
| 228 | } | |
| 229 | ||
| 230 | assert(ws.fuzz == null); | |
| 231 | ||
| 232 | ws.build_status.store(.fuzz_init, .monotonic); | |
| 233 | ws.notifyUpdate(); | |
| 234 | ||
| 235 | ws.fuzz = Fuzz.init( | |
| 236 | ws.gpa, | |
| 237 | ws.graph.io, | |
| 238 | ws.all_steps, | |
| 239 | ws.root_prog_node, | |
| 240 | .{ .forever = .{ .ws = ws } }, | |
| 241 | ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)}); | |
| 242 | ws.fuzz.?.start(); | |
| 243 | } | |
| 244 | ||
| 245 | ws.build_status.store(if (ws.watch) .watching else .idle, .monotonic); | |
| 246 | ws.notifyUpdate(); | |
| 247 | } | |
| 248 | ||
| 249 | pub fn now(s: *const WebServer) i64 { | |
| 250 | const io = s.graph.io; | |
| 251 | const ts = base_clock.now(io); | |
| 252 | return @intCast(s.base_timestamp.durationTo(ts).toNanoseconds()); | |
| 253 | } | |
| 254 | ||
| 255 | fn accept(ws: *WebServer, stream: net.Stream) void { | |
| 256 | const io = ws.graph.io; | |
| 257 | defer { | |
| 258 | // `net.Stream.close` wants to helpfully overwrite `stream` with | |
| 259 | // `undefined`, but it cannot do so since it is an immutable parameter. | |
| 260 | var copy = stream; | |
| 261 | copy.close(io); | |
| 262 | } | |
| 263 | var send_buffer: [4096]u8 = undefined; | |
| 264 | var recv_buffer: [4096]u8 = undefined; | |
| 265 | var connection_reader = stream.reader(io, &recv_buffer); | |
| 266 | var connection_writer = stream.writer(io, &send_buffer); | |
| 267 | var server: http.Server = .init(&connection_reader.interface, &connection_writer.interface); | |
| 268 | ||
| 269 | while (true) { | |
| 270 | var request = server.receiveHead() catch |err| switch (err) { | |
| 271 | error.HttpConnectionClosing => return, | |
| 272 | else => return log.err("failed to receive http request: {t}", .{err}), | |
| 273 | }; | |
| 274 | switch (request.upgradeRequested()) { | |
| 275 | .websocket => |opt_key| { | |
| 276 | const key = opt_key orelse return log.err("missing websocket key", .{}); | |
| 277 | var web_socket = request.respondWebSocket(.{ .key = key }) catch { | |
| 278 | return log.err("failed to respond web socket: {t}", .{connection_writer.err.?}); | |
| 279 | }; | |
| 280 | ws.serveWebSocket(&web_socket) catch |err| { | |
| 281 | log.err("failed to serve websocket: {t}", .{err}); | |
| 282 | return; | |
| 283 | }; | |
| 284 | comptime unreachable; | |
| 285 | }, | |
| 286 | .other => |name| return log.err("unknown upgrade request: {s}", .{name}), | |
| 287 | .none => { | |
| 288 | ws.serveRequest(&request) catch |err| switch (err) { | |
| 289 | error.AlreadyReported => return, | |
| 290 | else => { | |
| 291 | log.err("failed to serve '{s}': {t}", .{ request.head.target, err }); | |
| 292 | return; | |
| 293 | }, | |
| 294 | }; | |
| 295 | }, | |
| 296 | } | |
| 297 | } | |
| 298 | } | |
| 299 | ||
| 300 | fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { | |
| 301 | const io = ws.graph.io; | |
| 302 | ||
| 303 | var prev_build_status = ws.build_status.load(.monotonic); | |
| 304 | ||
| 305 | const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len); | |
| 306 | defer ws.gpa.free(prev_step_status_bits); | |
| 307 | for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| { | |
| 308 | copy.* = @atomicLoad(u8, shared, .monotonic); | |
| 309 | } | |
| 310 | ||
| 311 | var recv_thread = try io.concurrent(recvWebSocketMessages, .{ ws, sock }); | |
| 312 | defer recv_thread.cancel(io); | |
| 313 | ||
| 314 | { | |
| 315 | const hello_header: abi.Hello = .{ | |
| 316 | .status = prev_build_status, | |
| 317 | .flags = .{ | |
| 318 | .time_report = ws.graph.time_report, | |
| 319 | }, | |
| 320 | .timestamp = ws.now(), | |
| 321 | .steps_len = @intCast(ws.all_steps.len), | |
| 322 | }; | |
| 323 | var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits }; | |
| 324 | try sock.writeMessageVec(&bufs, .binary); | |
| 325 | } | |
| 326 | ||
| 327 | var prev_fuzz: Fuzz.Previous = .init; | |
| 328 | var prev_time: i64 = std.math.minInt(i64); | |
| 329 | while (true) { | |
| 330 | const start_time = ws.now(); | |
| 331 | const start_update_id = ws.update_id.load(.acquire); | |
| 332 | ||
| 333 | if (ws.fuzz) |*fuzz| { | |
| 334 | try fuzz.sendUpdate(sock, &prev_fuzz); | |
| 335 | } | |
| 336 | ||
| 337 | { | |
| 338 | try ws.time_report_mutex.lock(io); | |
| 339 | defer ws.time_report_mutex.unlock(io); | |
| 340 | for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| { | |
| 341 | if (update_time <= prev_time) continue; | |
| 342 | // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so | |
| 343 | // that we don't hold up the build system on the client accepting this packet. | |
| 344 | const owned_msg = try ws.gpa.dupe(u8, msg); | |
| 345 | defer ws.gpa.free(owned_msg); | |
| 346 | // Temporarily unlock, then re-lock after the message is sent. | |
| 347 | ws.time_report_mutex.unlock(io); | |
| 348 | defer ws.time_report_mutex.lockUncancelable(io); | |
| 349 | try sock.writeMessage(owned_msg, .binary); | |
| 350 | } | |
| 351 | } | |
| 352 | ||
| 353 | { | |
| 354 | const build_status = ws.build_status.load(.monotonic); | |
| 355 | if (build_status != prev_build_status) { | |
| 356 | prev_build_status = build_status; | |
| 357 | const msg: abi.StatusUpdate = .{ .new = build_status }; | |
| 358 | try sock.writeMessage(@ptrCast(&msg), .binary); | |
| 359 | } | |
| 360 | } | |
| 361 | ||
| 362 | for (prev_step_status_bits, ws.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| { | |
| 363 | const cur_byte = @atomicLoad(u8, shared, .monotonic); | |
| 364 | if (prev_byte.* == cur_byte) continue; | |
| 365 | const cur: [4]abi.StepUpdate.Status = .{ | |
| 366 | @enumFromInt(@as(u2, @truncate(cur_byte >> 0))), | |
| 367 | @enumFromInt(@as(u2, @truncate(cur_byte >> 2))), | |
| 368 | @enumFromInt(@as(u2, @truncate(cur_byte >> 4))), | |
| 369 | @enumFromInt(@as(u2, @truncate(cur_byte >> 6))), | |
| 370 | }; | |
| 371 | const prev: [4]abi.StepUpdate.Status = .{ | |
| 372 | @enumFromInt(@as(u2, @truncate(prev_byte.* >> 0))), | |
| 373 | @enumFromInt(@as(u2, @truncate(prev_byte.* >> 2))), | |
| 374 | @enumFromInt(@as(u2, @truncate(prev_byte.* >> 4))), | |
| 375 | @enumFromInt(@as(u2, @truncate(prev_byte.* >> 6))), | |
| 376 | }; | |
| 377 | for (cur, prev, byte_idx * 4..) |cur_status, prev_status, step_idx| { | |
| 378 | const msg: abi.StepUpdate = .{ .step_idx = @intCast(step_idx), .bits = .{ .status = cur_status } }; | |
| 379 | if (cur_status != prev_status) try sock.writeMessage(@ptrCast(&msg), .binary); | |
| 380 | } | |
| 381 | prev_byte.* = cur_byte; | |
| 382 | } | |
| 383 | ||
| 384 | prev_time = start_time; | |
| 385 | ||
| 386 | const old_cp = io.swapCancelProtection(.blocked); | |
| 387 | defer _ = io.swapCancelProtection(old_cp); | |
| 388 | io.futexWaitTimeout( | |
| 389 | u32, | |
| 390 | &ws.update_id.raw, | |
| 391 | start_update_id, | |
| 392 | .{ .duration = .{ | |
| 393 | .clock = .awake, | |
| 394 | .raw = .fromMilliseconds(default_update_interval_ms), | |
| 395 | } }, | |
| 396 | ) catch |err| switch (err) { | |
| 397 | error.Canceled => unreachable, | |
| 398 | }; | |
| 399 | } | |
| 400 | } | |
| 401 | fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void { | |
| 402 | const io = ws.graph.io; | |
| 403 | ||
| 404 | while (true) { | |
| 405 | const msg = sock.readSmallMessage() catch return; | |
| 406 | if (msg.opcode != .binary) continue; | |
| 407 | if (msg.data.len == 0) continue; | |
| 408 | const tag: abi.ToServerTag = @enumFromInt(msg.data[0]); | |
| 409 | switch (tag) { | |
| 410 | _ => continue, | |
| 411 | .rebuild => while (true) { | |
| 412 | ws.runner_request_mutex.lock(io) catch |err| switch (err) { | |
| 413 | error.Canceled => return, | |
| 414 | }; | |
| 415 | defer ws.runner_request_mutex.unlock(io); | |
| 416 | if (ws.runner_request == null) { | |
| 417 | ws.runner_request = .rebuild; | |
| 418 | ws.runner_request_ready_cond.signal(io); | |
| 419 | break; | |
| 420 | } | |
| 421 | ws.runner_request_empty_cond.wait(io, &ws.runner_request_mutex) catch return; | |
| 422 | }, | |
| 423 | } | |
| 424 | } | |
| 425 | } | |
| 426 | ||
| 427 | fn serveRequest(ws: *WebServer, req: *http.Server.Request) !void { | |
| 428 | // Strip an optional leading '/debug' component from the request. | |
| 429 | const target: []const u8, const debug: bool = target: { | |
| 430 | if (mem.eql(u8, req.head.target, "/debug")) break :target .{ "/", true }; | |
| 431 | if (mem.eql(u8, req.head.target, "/debug/")) break :target .{ "/", true }; | |
| 432 | if (mem.startsWith(u8, req.head.target, "/debug/")) break :target .{ req.head.target["/debug".len..], true }; | |
| 433 | break :target .{ req.head.target, false }; | |
| 434 | }; | |
| 435 | ||
| 436 | if (mem.eql(u8, target, "/")) return serveLibFile(ws, req, "build-web/index.html", "text/html"); | |
| 437 | if (mem.eql(u8, target, "/main.js")) return serveLibFile(ws, req, "build-web/main.js", "application/javascript"); | |
| 438 | if (mem.eql(u8, target, "/style.css")) return serveLibFile(ws, req, "build-web/style.css", "text/css"); | |
| 439 | if (mem.eql(u8, target, "/time_report.css")) return serveLibFile(ws, req, "build-web/time_report.css", "text/css"); | |
| 440 | if (mem.eql(u8, target, "/main.wasm")) return serveClientWasm(ws, req, if (debug) .Debug else .ReleaseFast); | |
| 441 | ||
| 442 | if (ws.fuzz) |*fuzz| { | |
| 443 | if (mem.eql(u8, target, "/sources.tar")) return fuzz.serveSourcesTar(req); | |
| 444 | } | |
| 445 | ||
| 446 | try req.respond("not found", .{ | |
| 447 | .status = .not_found, | |
| 448 | .extra_headers = &.{ | |
| 449 | .{ .name = "Content-Type", .value = "text/plain" }, | |
| 450 | }, | |
| 451 | }); | |
| 452 | } | |
| 453 | ||
| 454 | fn serveLibFile( | |
| 455 | ws: *WebServer, | |
| 456 | request: *http.Server.Request, | |
| 457 | sub_path: []const u8, | |
| 458 | content_type: []const u8, | |
| 459 | ) !void { | |
| 460 | return serveFile(ws, request, .{ | |
| 461 | .root_dir = ws.graph.zig_lib_directory, | |
| 462 | .sub_path = sub_path, | |
| 463 | }, content_type); | |
| 464 | } | |
| 465 | fn serveClientWasm( | |
| 466 | ws: *WebServer, | |
| 467 | req: *http.Server.Request, | |
| 468 | optimize_mode: std.builtin.OptimizeMode, | |
| 469 | ) !void { | |
| 470 | var arena_state: std.heap.ArenaAllocator = .init(ws.gpa); | |
| 471 | defer arena_state.deinit(); | |
| 472 | const arena = arena_state.allocator(); | |
| 473 | ||
| 474 | // We always rebuild the wasm on-the-fly, so that if it is edited the user can just refresh the page. | |
| 475 | const bin_path = try buildClientWasm(ws, arena, optimize_mode); | |
| 476 | return serveFile(ws, req, bin_path, "application/wasm"); | |
| 477 | } | |
| 478 | ||
| 479 | pub fn serveFile( | |
| 480 | ws: *WebServer, | |
| 481 | request: *http.Server.Request, | |
| 482 | path: Cache.Path, | |
| 483 | content_type: []const u8, | |
| 484 | ) !void { | |
| 485 | const gpa = ws.gpa; | |
| 486 | const io = ws.graph.io; | |
| 487 | // The desired API is actually sendfile, which will require enhancing http.Server. | |
| 488 | // We load the file with every request so that the user can make changes to the file | |
| 489 | // and refresh the HTML page without restarting this server. | |
| 490 | const file_contents = path.root_dir.handle.readFileAlloc(io, path.sub_path, gpa, .limited(10 * 1024 * 1024)) catch |err| { | |
| 491 | log.err("failed to read '{f}': {t}", .{ path, err }); | |
| 492 | return error.AlreadyReported; | |
| 493 | }; | |
| 494 | defer gpa.free(file_contents); | |
| 495 | try request.respond(file_contents, .{ | |
| 496 | .extra_headers = &.{ | |
| 497 | .{ .name = "Content-Type", .value = content_type }, | |
| 498 | cache_control_header, | |
| 499 | }, | |
| 500 | }); | |
| 501 | } | |
| 502 | pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void { | |
| 503 | const graph = ws.graph; | |
| 504 | const io = graph.io; | |
| 505 | ||
| 506 | var send_buffer: [0x4000]u8 = undefined; | |
| 507 | var response = try request.respondStreaming(&send_buffer, .{ | |
| 508 | .respond_options = .{ | |
| 509 | .extra_headers = &.{ | |
| 510 | .{ .name = "Content-Type", .value = "application/x-tar" }, | |
| 511 | cache_control_header, | |
| 512 | }, | |
| 513 | }, | |
| 514 | }); | |
| 515 | ||
| 516 | var archiver: std.tar.Writer = .{ .underlying_writer = &response.writer }; | |
| 517 | ||
| 518 | for (paths) |path| { | |
| 519 | var file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| { | |
| 520 | log.err("failed to open '{f}': {s}", .{ path, @errorName(err) }); | |
| 521 | continue; | |
| 522 | }; | |
| 523 | defer file.close(io); | |
| 524 | const stat = try file.stat(io); | |
| 525 | var read_buffer: [1024]u8 = undefined; | |
| 526 | var file_reader: Io.File.Reader = .initSize(file, io, &read_buffer, stat.size); | |
| 527 | ||
| 528 | // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can | |
| 529 | // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI: | |
| 530 | // it turns out the WASM treats the first path component as the module name, typically | |
| 531 | // resulting in modules named "" and "src". The compiler needs to tell the build system | |
| 532 | // about the module graph so that the build system can correctly encode this information in | |
| 533 | // the tar file. | |
| 534 | // | |
| 535 | // Additionally, this needs to ensure that all path separators for both prefix and | |
| 536 | // sub_path are using the POSIX-style `/` on platforms that don't use it as their native | |
| 537 | // path separator. | |
| 538 | archiver.prefix = path.root_dir.path orelse graph.cache.cwd; | |
| 539 | try archiver.writeFile(path.sub_path, &file_reader, @intCast(stat.mtime.toSeconds())); | |
| 540 | } | |
| 541 | ||
| 542 | // intentionally not calling `archiver.finishPedantically` | |
| 543 | try response.end(); | |
| 544 | } | |
| 545 | ||
| 546 | fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.OptimizeMode) !Cache.Path { | |
| 547 | const root_name = "build-web"; | |
| 548 | const arch_os_abi = "wasm32-freestanding"; | |
| 549 | const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; | |
| 550 | ||
| 551 | const gpa = ws.gpa; | |
| 552 | const graph = ws.graph; | |
| 553 | const io = graph.io; | |
| 554 | ||
| 555 | const main_src_path: Cache.Path = .{ | |
| 556 | .root_dir = graph.zig_lib_directory, | |
| 557 | .sub_path = "build-web/main.zig", | |
| 558 | }; | |
| 559 | const walk_src_path: Cache.Path = .{ | |
| 560 | .root_dir = graph.zig_lib_directory, | |
| 561 | .sub_path = "docs/wasm/Walk.zig", | |
| 562 | }; | |
| 563 | const html_render_src_path: Cache.Path = .{ | |
| 564 | .root_dir = graph.zig_lib_directory, | |
| 565 | .sub_path = "docs/wasm/html_render.zig", | |
| 566 | }; | |
| 567 | ||
| 568 | var argv: std.ArrayList([]const u8) = .empty; | |
| 569 | ||
| 570 | try argv.appendSlice(arena, &.{ | |
| 571 | graph.zig_exe, "build-exe", // | |
| 572 | "-fno-entry", // | |
| 573 | "-O", @tagName(optimize), // | |
| 574 | "-target", arch_os_abi, // | |
| 575 | "-mcpu", cpu_features, // | |
| 576 | "--cache-dir", graph.global_cache_root.path orelse ".", // | |
| 577 | "--global-cache-dir", graph.global_cache_root.path orelse ".", // | |
| 578 | "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // | |
| 579 | "--name", root_name, // | |
| 580 | "-rdynamic", // | |
| 581 | "-fsingle-threaded", // | |
| 582 | "--dep", "Walk", // | |
| 583 | "--dep", "html_render", // | |
| 584 | try std.fmt.allocPrint(arena, "-Mroot={f}", .{main_src_path}), // | |
| 585 | try std.fmt.allocPrint(arena, "-MWalk={f}", .{walk_src_path}), // | |
| 586 | "--dep", "Walk", // | |
| 587 | try std.fmt.allocPrint(arena, "-Mhtml_render={f}", .{html_render_src_path}), // | |
| 588 | "--listen=-", | |
| 589 | }); | |
| 590 | ||
| 591 | var child = try std.process.spawn(io, .{ | |
| 592 | .argv = argv.items, | |
| 593 | .environ_map = &graph.environ_map, | |
| 594 | .stdin = .pipe, | |
| 595 | .stdout = .pipe, | |
| 596 | .stderr = .pipe, | |
| 597 | }); | |
| 598 | defer child.kill(io); | |
| 599 | ||
| 600 | var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited }); | |
| 601 | defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {}; | |
| 602 | ||
| 603 | var stdout_buffer: [512]u8 = undefined; | |
| 604 | var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer); | |
| 605 | const stdout = &stdout_reader.interface; | |
| 606 | ||
| 607 | { | |
| 608 | var w = child.stdin.?.writer(io, &.{}); | |
| 609 | w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch |err| switch (err) { | |
| 610 | error.WriteFailed => return w.err.?, | |
| 611 | }; | |
| 612 | w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch |err| switch (err) { | |
| 613 | error.WriteFailed => return w.err.?, | |
| 614 | }; | |
| 615 | } | |
| 616 | ||
| 617 | const Header = std.zig.Server.Message.Header; | |
| 618 | ||
| 619 | var result: ?Cache.Path = null; | |
| 620 | var result_error_bundle = std.zig.ErrorBundle.empty; | |
| 621 | var body_buffer: std.ArrayList(u8) = .empty; | |
| 622 | defer body_buffer.deinit(gpa); | |
| 623 | ||
| 624 | while (true) { | |
| 625 | const header = stdout.takeStruct(Header, .little) catch |err| switch (err) { | |
| 626 | error.ReadFailed => |e| return e, | |
| 627 | error.EndOfStream => break, | |
| 628 | }; | |
| 629 | body_buffer.clearRetainingCapacity(); | |
| 630 | try stdout.appendExact(gpa, &body_buffer, header.bytes_len); | |
| 631 | const body = body_buffer.items; | |
| 632 | ||
| 633 | switch (header.tag) { | |
| 634 | .zig_version => { | |
| 635 | if (!std.mem.eql(u8, builtin.zig_version_string, body)) { | |
| 636 | return error.ZigProtocolVersionMismatch; | |
| 637 | } | |
| 638 | }, | |
| 639 | .error_bundle => { | |
| 640 | result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body); | |
| 641 | }, | |
| 642 | .emit_digest => { | |
| 643 | const EmitDigest = std.zig.Server.Message.EmitDigest; | |
| 644 | const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body); | |
| 645 | if (!ebp_hdr.flags.cache_hit) { | |
| 646 | log.info("source changes detected; rebuilt wasm component", .{}); | |
| 647 | } | |
| 648 | const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; | |
| 649 | result = .{ | |
| 650 | .root_dir = graph.global_cache_root, | |
| 651 | .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), | |
| 652 | }; | |
| 653 | }, | |
| 654 | else => {}, // ignore other messages | |
| 655 | } | |
| 656 | } | |
| 657 | ||
| 658 | const stderr_contents = try stderr_task.await(io); | |
| 659 | if (stderr_contents.len > 0) { | |
| 660 | std.debug.print("{s}", .{stderr_contents}); | |
| 661 | } | |
| 662 | ||
| 663 | // Send EOF to stdin. | |
| 664 | child.stdin.?.close(io); | |
| 665 | child.stdin = null; | |
| 666 | ||
| 667 | switch (try child.wait(io)) { | |
| 668 | .exited => |code| { | |
| 669 | if (code != 0) { | |
| 670 | log.err( | |
| 671 | "the following command exited with error code {d}:\n{s}", | |
| 672 | .{ code, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, | |
| 673 | ); | |
| 674 | return error.WasmCompilationFailed; | |
| 675 | } | |
| 676 | }, | |
| 677 | .signal => |sig| { | |
| 678 | log.err( | |
| 679 | "the following command terminated with signal {t}:\n{s}", | |
| 680 | .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, | |
| 681 | ); | |
| 682 | return error.WasmCompilationFailed; | |
| 683 | }, | |
| 684 | .stopped => |sig| { | |
| 685 | log.err( | |
| 686 | "the following command stopped unexpectedly with signal {t}:\n{s}", | |
| 687 | .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, | |
| 688 | ); | |
| 689 | return error.WasmCompilationFailed; | |
| 690 | }, | |
| 691 | .unknown => { | |
| 692 | log.err( | |
| 693 | "the following command terminated unexpectedly:\n{s}", | |
| 694 | .{try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items)}, | |
| 695 | ); | |
| 696 | return error.WasmCompilationFailed; | |
| 697 | }, | |
| 698 | } | |
| 699 | ||
| 700 | if (result_error_bundle.errorMessageCount() > 0) { | |
| 701 | try result_error_bundle.renderToStderr(io, .{}, .auto); | |
| 702 | log.err("the following command failed with {d} compilation errors:\n{s}", .{ | |
| 703 | result_error_bundle.errorMessageCount(), | |
| 704 | try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), | |
| 705 | }); | |
| 706 | return error.WasmCompilationFailed; | |
| 707 | } | |
| 708 | ||
| 709 | const base_path = result orelse { | |
| 710 | log.err("child process failed to report result\n{s}", .{ | |
| 711 | try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), | |
| 712 | }); | |
| 713 | return error.WasmCompilationFailed; | |
| 714 | }; | |
| 715 | const bin_name = try std.zig.binNameAlloc(arena, .{ | |
| 716 | .root_name = root_name, | |
| 717 | .target = &(std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ | |
| 718 | .arch_os_abi = arch_os_abi, | |
| 719 | .cpu_features = cpu_features, | |
| 720 | }) catch unreachable) catch unreachable), | |
| 721 | .output_mode = .Exe, | |
| 722 | }); | |
| 723 | return base_path.join(arena, bin_name); | |
| 724 | } | |
| 725 | ||
| 726 | fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 { | |
| 727 | var file_reader: Io.File.Reader = .initStreaming(file, io, &.{}); | |
| 728 | return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { | |
| 729 | error.ReadFailed => return file_reader.err.?, | |
| 730 | else => |e| return e, | |
| 731 | }; | |
| 732 | } | |
| 733 | ||
| 734 | pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { | |
| 735 | compile: *Build.Step.Compile, | |
| 736 | ||
| 737 | use_llvm: bool, | |
| 738 | stats: abi.time_report.CompileResult.Stats, | |
| 739 | ns_total: u64, | |
| 740 | ||
| 741 | llvm_pass_timings_len: u32, | |
| 742 | files_len: u32, | |
| 743 | decls_len: u32, | |
| 744 | ||
| 745 | /// The trailing data of `abi.time_report.CompileResult`, except the step name. | |
| 746 | trailing: []const u8, | |
| 747 | }) void { | |
| 748 | const gpa = ws.gpa; | |
| 749 | const io = ws.graph.io; | |
| 750 | ||
| 751 | const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { | |
| 752 | if (s == &opts.compile.step) break @intCast(i); | |
| 753 | } else unreachable; | |
| 754 | ||
| 755 | const old_buf = old: { | |
| 756 | ws.time_report_mutex.lock(io) catch return; | |
| 757 | defer ws.time_report_mutex.unlock(io); | |
| 758 | const old = ws.time_report_msgs[step_idx]; | |
| 759 | ws.time_report_msgs[step_idx] = &.{}; | |
| 760 | break :old old; | |
| 761 | }; | |
| 762 | const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory"); | |
| 763 | ||
| 764 | const out_header: *align(1) abi.time_report.CompileResult = @ptrCast(buf[0..@sizeOf(abi.time_report.CompileResult)]); | |
| 765 | out_header.* = .{ | |
| 766 | .step_idx = step_idx, | |
| 767 | .flags = .{ | |
| 768 | .use_llvm = opts.use_llvm, | |
| 769 | }, | |
| 770 | .stats = opts.stats, | |
| 771 | .ns_total = opts.ns_total, | |
| 772 | .llvm_pass_timings_len = opts.llvm_pass_timings_len, | |
| 773 | .files_len = opts.files_len, | |
| 774 | .decls_len = opts.decls_len, | |
| 775 | }; | |
| 776 | @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing); | |
| 777 | ||
| 778 | { | |
| 779 | ws.time_report_mutex.lock(io) catch return; | |
| 780 | defer ws.time_report_mutex.unlock(io); | |
| 781 | assert(ws.time_report_msgs[step_idx].len == 0); | |
| 782 | ws.time_report_msgs[step_idx] = buf; | |
| 783 | ws.time_report_update_times[step_idx] = ws.now(); | |
| 784 | } | |
| 785 | ws.notifyUpdate(); | |
| 786 | } | |
| 787 | ||
| 788 | pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.Duration) void { | |
| 789 | const gpa = ws.gpa; | |
| 790 | const io = ws.graph.io; | |
| 791 | ||
| 792 | const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { | |
| 793 | if (s == step) break @intCast(i); | |
| 794 | } else unreachable; | |
| 795 | ||
| 796 | const old_buf = old: { | |
| 797 | ws.time_report_mutex.lock(io) catch return; | |
| 798 | defer ws.time_report_mutex.unlock(io); | |
| 799 | const old = ws.time_report_msgs[step_idx]; | |
| 800 | ws.time_report_msgs[step_idx] = &.{}; | |
| 801 | break :old old; | |
| 802 | }; | |
| 803 | const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory"); | |
| 804 | const out: *align(1) abi.time_report.GenericResult = @ptrCast(buf); | |
| 805 | out.* = .{ | |
| 806 | .step_idx = step_idx, | |
| 807 | .ns_total = @intCast(duration.toNanoseconds()), | |
| 808 | }; | |
| 809 | { | |
| 810 | ws.time_report_mutex.lock(io) catch return; | |
| 811 | defer ws.time_report_mutex.unlock(io); | |
| 812 | assert(ws.time_report_msgs[step_idx].len == 0); | |
| 813 | ws.time_report_msgs[step_idx] = buf; | |
| 814 | ws.time_report_update_times[step_idx] = ws.now(); | |
| 815 | } | |
| 816 | ws.notifyUpdate(); | |
| 817 | } | |
| 818 | ||
| 819 | pub fn updateTimeReportRunTest( | |
| 820 | ws: *WebServer, | |
| 821 | run: *Build.Step.Run, | |
| 822 | tests: *const Build.Step.Run.CachedTestMetadata, | |
| 823 | ns_per_test: []const u64, | |
| 824 | ) void { | |
| 825 | const gpa = ws.gpa; | |
| 826 | const io = ws.graph.io; | |
| 827 | ||
| 828 | const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { | |
| 829 | if (s == &run.step) break @intCast(i); | |
| 830 | } else unreachable; | |
| 831 | ||
| 832 | assert(tests.names.len == ns_per_test.len); | |
| 833 | const tests_len: u32 = @intCast(tests.names.len); | |
| 834 | ||
| 835 | const new_len: u64 = len: { | |
| 836 | var names_len: u64 = 0; | |
| 837 | for (0..tests_len) |i| { | |
| 838 | names_len += tests.testName(@intCast(i)).len + 1; | |
| 839 | } | |
| 840 | break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len; | |
| 841 | }; | |
| 842 | const old_buf = old: { | |
| 843 | ws.time_report_mutex.lock(io) catch return; | |
| 844 | defer ws.time_report_mutex.unlock(io); | |
| 845 | const old = ws.time_report_msgs[step_idx]; | |
| 846 | ws.time_report_msgs[step_idx] = &.{}; | |
| 847 | break :old old; | |
| 848 | }; | |
| 849 | const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory"); | |
| 850 | ||
| 851 | const out_header: *align(1) abi.time_report.RunTestResult = @ptrCast(buf[0..@sizeOf(abi.time_report.RunTestResult)]); | |
| 852 | out_header.* = .{ | |
| 853 | .step_idx = step_idx, | |
| 854 | .tests_len = tests_len, | |
| 855 | }; | |
| 856 | var offset: usize = @sizeOf(abi.time_report.RunTestResult); | |
| 857 | const ns_per_test_out: []align(1) u64 = @ptrCast(buf[offset..][0 .. tests_len * 8]); | |
| 858 | @memcpy(ns_per_test_out, ns_per_test); | |
| 859 | offset += tests_len * 8; | |
| 860 | for (0..tests_len) |i| { | |
| 861 | const name = tests.testName(@intCast(i)); | |
| 862 | @memcpy(buf[offset..][0..name.len], name); | |
| 863 | buf[offset..][name.len] = 0; | |
| 864 | offset += name.len + 1; | |
| 865 | } | |
| 866 | assert(offset == buf.len); | |
| 867 | ||
| 868 | { | |
| 869 | ws.time_report_mutex.lock(io) catch return; | |
| 870 | defer ws.time_report_mutex.unlock(io); | |
| 871 | assert(ws.time_report_msgs[step_idx].len == 0); | |
| 872 | ws.time_report_msgs[step_idx] = buf; | |
| 873 | ws.time_report_update_times[step_idx] = ws.now(); | |
| 874 | } | |
| 875 | ws.notifyUpdate(); | |
| 876 | } | |
| 877 | ||
| 878 | const RunnerRequest = union(enum) { | |
| 879 | rebuild, | |
| 880 | }; | |
| 881 | pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { | |
| 882 | const io = ws.graph.io; | |
| 883 | ws.runner_request_mutex.lock(io) catch return; | |
| 884 | defer ws.runner_request_mutex.unlock(io); | |
| 885 | if (ws.runner_request) |req| { | |
| 886 | ws.runner_request = null; | |
| 887 | ws.runner_request_empty_cond.signal(); | |
| 888 | return req; | |
| 889 | } | |
| 890 | return null; | |
| 891 | } | |
| 892 | pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest { | |
| 893 | const io = ws.graph.io; | |
| 894 | try ws.runner_request_mutex.lock(io); | |
| 895 | defer ws.runner_request_mutex.unlock(io); | |
| 896 | while (true) { | |
| 897 | if (ws.runner_request) |req| { | |
| 898 | ws.runner_request = null; | |
| 899 | ws.runner_request_empty_cond.signal(io); | |
| 900 | return req; | |
| 901 | } | |
| 902 | try ws.runner_request_ready_cond.wait(io, &ws.runner_request_mutex); | |
| 903 | } | |
| 904 | } | |
| 905 | ||
| 906 | const cache_control_header: http.Header = .{ | |
| 907 | .name = "Cache-Control", | |
| 908 | .value = "max-age=0, must-revalidate", | |
| 909 | }; | |
| 910 | ||
| 911 | const builtin = @import("builtin"); | |
| 912 | ||
| 913 | const std = @import("std"); | |
| 914 | const Io = std.Io; | |
| 915 | const net = std.Io.net; | |
| 916 | const assert = std.debug.assert; | |
| 917 | const mem = std.mem; | |
| 918 | const log = std.log.scoped(.web_server); | |
| 919 | const Allocator = std.mem.Allocator; | |
| 920 | const Build = std.Build; | |
| 921 | const Cache = Build.Cache; | |
| 922 | const Fuzz = Build.Fuzz; | |
| 923 | const abi = Build.abi; | |
| 924 | const http = std.http; | |
| 925 | ||
| 926 | const WebServer = @This(); |
lib/std/Build.zig+23-175
| ... | ... | @@ -19,15 +19,14 @@ const ArrayList = std.ArrayList; |
| 19 | 19 | pub const Cache = @import("Build/Cache.zig"); |
| 20 | 20 | pub const Step = @import("Build/Step.zig"); |
| 21 | 21 | pub const Module = @import("Build/Module.zig"); |
| 22 | pub const Watch = @import("Build/Watch.zig"); | |
| 23 | pub const Fuzz = @import("Build/Fuzz.zig"); | |
| 24 | pub const WebServer = @import("Build/WebServer.zig"); | |
| 25 | 22 | pub const abi = @import("Build/abi.zig"); |
| 23 | /// The serialized output of configure phase ingested by make phase. | |
| 24 | pub const Configuration = @import("zig/Configuration.zig"); | |
| 26 | 25 | |
| 27 | 26 | /// Shared state among all Build instances. |
| 28 | 27 | graph: *Graph, |
| 29 | install_tls: TopLevelStep, | |
| 30 | uninstall_tls: TopLevelStep, | |
| 28 | install_tls: Step.TopLevel, | |
| 29 | uninstall_tls: Step.TopLevel, | |
| 31 | 30 | allocator: Allocator, |
| 32 | 31 | user_input_options: UserInputOptionsMap, |
| 33 | 32 | available_options_map: AvailableOptionsMap, |
| ... | ... | @@ -39,28 +38,17 @@ verbose_air: bool, |
| 39 | 38 | verbose_llvm_ir: ?[]const u8, |
| 40 | 39 | verbose_llvm_bc: ?[]const u8, |
| 41 | 40 | verbose_llvm_cpu_features: bool, |
| 42 | reference_trace: ?u32 = null, | |
| 43 | 41 | invalid_user_input: bool, |
| 44 | 42 | default_step: *Step, |
| 45 | top_level_steps: std.StringArrayHashMapUnmanaged(*TopLevelStep), | |
| 43 | top_level_steps: std.StringArrayHashMapUnmanaged(*Step.TopLevel), | |
| 46 | 44 | install_prefix: []const u8, |
| 47 | dest_dir: ?[]const u8, | |
| 48 | lib_dir: []const u8, | |
| 49 | exe_dir: []const u8, | |
| 50 | h_dir: []const u8, | |
| 51 | install_path: []const u8, | |
| 52 | sysroot: ?[]const u8 = null, | |
| 53 | search_prefixes: ArrayList([]const u8), | |
| 54 | libc_file: ?[]const u8 = null, | |
| 55 | 45 | /// Path to the directory containing build.zig. |
| 56 | 46 | build_root: Cache.Directory, |
| 57 | 47 | cache_root: Cache.Directory, |
| 58 | 48 | pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, |
| 59 | args: ?[]const []const u8 = null, | |
| 60 | 49 | debug_log_scopes: []const []const u8 = &.{}, |
| 61 | 50 | debug_compile_errors: bool = false, |
| 62 | 51 | debug_incremental: bool = false, |
| 63 | debug_pkg_config: bool = false, | |
| 64 | 52 | /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, |
| 65 | 53 | /// in particular at `Step` creation. |
| 66 | 54 | /// Set to 0 to disable stack collection. |
| ... | ... | @@ -76,12 +64,6 @@ enable_rosetta: bool = false, |
| 76 | 64 | enable_wasmtime: bool = false, |
| 77 | 65 | /// Use system Wine installation to run cross compiled Windows build artifacts. |
| 78 | 66 | enable_wine: bool = false, |
| 79 | /// After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, | |
| 80 | /// this will be the directory $glibc-build-dir/install/glibcs | |
| 81 | /// Given the example of the aarch64 target, this is the directory | |
| 82 | /// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. | |
| 83 | /// Also works for dynamic musl. | |
| 84 | libc_runtimes_dir: ?[]const u8 = null, | |
| 85 | 67 | |
| 86 | 68 | dep_prefix: []const u8 = "", |
| 87 | 69 | |
| ... | ... | @@ -94,8 +76,6 @@ pkg_hash: []const u8, |
| 94 | 76 | /// A mapping from dependency names to package hashes. |
| 95 | 77 | available_deps: AvailableDeps, |
| 96 | 78 | |
| 97 | release_mode: ReleaseMode, | |
| 98 | ||
| 99 | 79 | build_id: ?std.zig.BuildId = null, |
| 100 | 80 | |
| 101 | 81 | pub const ReleaseMode = enum { |
| ... | ... | @@ -116,14 +96,13 @@ pub const Graph = struct { |
| 116 | 96 | system_package_mode: bool = false, |
| 117 | 97 | debug_compiler_runtime_libs: ?std.builtin.OptimizeMode = null, |
| 118 | 98 | cache: Cache, |
| 119 | zig_exe: [:0]const u8, | |
| 99 | zig_exe: []const u8, | |
| 120 | 100 | environ_map: process.Environ.Map, |
| 121 | 101 | global_cache_root: Cache.Directory, |
| 122 | 102 | zig_lib_directory: Cache.Directory, |
| 123 | 103 | needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .empty, |
| 124 | 104 | /// Information about the native target. Computed before build() is invoked. |
| 125 | 105 | host: ResolvedTarget, |
| 126 | incremental: ?bool = null, | |
| 127 | 106 | random_seed: u32 = 0, |
| 128 | 107 | dependency_cache: InitializedDepMap = .empty, |
| 129 | 108 | allow_so_scripts: ?bool = null, |
| ... | ... | @@ -134,11 +113,12 @@ pub const Graph = struct { |
| 134 | 113 | /// Similar to the `Io.Terminal.Mode` returned by `Io.lockStderr`, but also |
| 135 | 114 | /// respects the '--color' flag. |
| 136 | 115 | stderr_mode: ?Io.Terminal.Mode = null, |
| 116 | release_mode: ReleaseMode = .off, | |
| 137 | 117 | }; |
| 138 | 118 | |
| 139 | 119 | const AvailableDeps = []const struct { []const u8, []const u8 }; |
| 140 | 120 | |
| 141 | const SystemLibraryMode = enum { | |
| 121 | pub const SystemLibraryMode = enum { | |
| 142 | 122 | /// User asked for the library to be disabled. |
| 143 | 123 | /// The build runner has not confirmed whether the setting is recognized yet. |
| 144 | 124 | user_disabled, |
| ... | ... | @@ -245,19 +225,6 @@ const TypeId = enum { |
| 245 | 225 | lazy_path_list, |
| 246 | 226 | }; |
| 247 | 227 | |
| 248 | const TopLevelStep = struct { | |
| 249 | pub const base_id: Step.Id = .top_level; | |
| 250 | ||
| 251 | step: Step, | |
| 252 | description: []const u8, | |
| 253 | }; | |
| 254 | ||
| 255 | pub const DirList = struct { | |
| 256 | lib_dir: ?[]const u8 = null, | |
| 257 | exe_dir: ?[]const u8 = null, | |
| 258 | include_dir: ?[]const u8 = null, | |
| 259 | }; | |
| 260 | ||
| 261 | 228 | pub fn create( |
| 262 | 229 | graph: *Graph, |
| 263 | 230 | build_root: Cache.Directory, |
| ... | ... | @@ -285,15 +252,10 @@ pub fn create( |
| 285 | 252 | .available_options_list = std.array_list.Managed(AvailableOption).init(arena), |
| 286 | 253 | .top_level_steps = .{}, |
| 287 | 254 | .default_step = undefined, |
| 288 | .search_prefixes = .empty, | |
| 289 | 255 | .install_prefix = undefined, |
| 290 | .lib_dir = undefined, | |
| 291 | .exe_dir = undefined, | |
| 292 | .h_dir = undefined, | |
| 293 | .dest_dir = graph.environ_map.get("DESTDIR"), | |
| 294 | 256 | .install_tls = .{ |
| 295 | 257 | .step = .init(.{ |
| 296 | .id = TopLevelStep.base_id, | |
| 258 | .tag = .top_level, | |
| 297 | 259 | .name = "install", |
| 298 | 260 | .owner = b, |
| 299 | 261 | }), |
| ... | ... | @@ -301,21 +263,17 @@ pub fn create( |
| 301 | 263 | }, |
| 302 | 264 | .uninstall_tls = .{ |
| 303 | 265 | .step = .init(.{ |
| 304 | .id = TopLevelStep.base_id, | |
| 266 | .tag = .top_level, | |
| 305 | 267 | .name = "uninstall", |
| 306 | 268 | .owner = b, |
| 307 | .makeFn = makeUninstall, | |
| 308 | 269 | }), |
| 309 | 270 | .description = "Remove build artifacts from prefix path", |
| 310 | 271 | }, |
| 311 | .install_path = undefined, | |
| 312 | .args = null, | |
| 313 | 272 | .modules = .empty, |
| 314 | 273 | .named_writefiles = .empty, |
| 315 | 274 | .named_lazy_paths = .empty, |
| 316 | 275 | .pkg_hash = "", |
| 317 | 276 | .available_deps = available_deps, |
| 318 | .release_mode = .off, | |
| 319 | 277 | }; |
| 320 | 278 | try b.top_level_steps.put(arena, b.install_tls.step.name, &b.install_tls); |
| 321 | 279 | try b.top_level_steps.put(arena, b.uninstall_tls.step.name, &b.uninstall_tls); |
| ... | ... | @@ -330,19 +288,6 @@ fn createChild( |
| 330 | 288 | pkg_hash: []const u8, |
| 331 | 289 | pkg_deps: AvailableDeps, |
| 332 | 290 | user_input_options: UserInputOptionsMap, |
| 333 | ) error{OutOfMemory}!*Build { | |
| 334 | const child = try createChildOnly(parent, dep_name, build_root, pkg_hash, pkg_deps, user_input_options); | |
| 335 | try determineAndApplyInstallPrefix(child); | |
| 336 | return child; | |
| 337 | } | |
| 338 | ||
| 339 | fn createChildOnly( | |
| 340 | parent: *Build, | |
| 341 | dep_name: []const u8, | |
| 342 | build_root: Cache.Directory, | |
| 343 | pkg_hash: []const u8, | |
| 344 | pkg_deps: AvailableDeps, | |
| 345 | user_input_options: UserInputOptionsMap, | |
| 346 | 291 | ) error{OutOfMemory}!*Build { |
| 347 | 292 | const allocator = parent.allocator; |
| 348 | 293 | const child = try allocator.create(Build); |
| ... | ... | @@ -351,7 +296,7 @@ fn createChildOnly( |
| 351 | 296 | .allocator = allocator, |
| 352 | 297 | .install_tls = .{ |
| 353 | 298 | .step = .init(.{ |
| 354 | .id = TopLevelStep.base_id, | |
| 299 | .tag = .top_level, | |
| 355 | 300 | .name = "install", |
| 356 | 301 | .owner = child, |
| 357 | 302 | }), |
| ... | ... | @@ -359,10 +304,9 @@ fn createChildOnly( |
| 359 | 304 | }, |
| 360 | 305 | .uninstall_tls = .{ |
| 361 | 306 | .step = .init(.{ |
| 362 | .id = TopLevelStep.base_id, | |
| 307 | .tag = .top_level, | |
| 363 | 308 | .name = "uninstall", |
| 364 | 309 | .owner = child, |
| 365 | .makeFn = makeUninstall, | |
| 366 | 310 | }), |
| 367 | 311 | .description = "Remove build artifacts from prefix path", |
| 368 | 312 | }, |
| ... | ... | @@ -376,38 +320,31 @@ fn createChildOnly( |
| 376 | 320 | .verbose_llvm_ir = parent.verbose_llvm_ir, |
| 377 | 321 | .verbose_llvm_bc = parent.verbose_llvm_bc, |
| 378 | 322 | .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, |
| 379 | .reference_trace = parent.reference_trace, | |
| 380 | 323 | .invalid_user_input = false, |
| 381 | 324 | .default_step = undefined, |
| 382 | 325 | .top_level_steps = .{}, |
| 383 | 326 | .install_prefix = undefined, |
| 384 | .dest_dir = parent.dest_dir, | |
| 385 | 327 | .lib_dir = parent.lib_dir, |
| 386 | 328 | .exe_dir = parent.exe_dir, |
| 387 | 329 | .h_dir = parent.h_dir, |
| 388 | 330 | .install_path = parent.install_path, |
| 389 | 331 | .sysroot = parent.sysroot, |
| 390 | .search_prefixes = parent.search_prefixes, | |
| 391 | .libc_file = parent.libc_file, | |
| 392 | 332 | .build_root = build_root, |
| 393 | 333 | .cache_root = parent.cache_root, |
| 394 | 334 | .debug_log_scopes = parent.debug_log_scopes, |
| 395 | 335 | .debug_compile_errors = parent.debug_compile_errors, |
| 396 | 336 | .debug_incremental = parent.debug_incremental, |
| 397 | .debug_pkg_config = parent.debug_pkg_config, | |
| 398 | 337 | .enable_darling = parent.enable_darling, |
| 399 | 338 | .enable_qemu = parent.enable_qemu, |
| 400 | 339 | .enable_rosetta = parent.enable_rosetta, |
| 401 | 340 | .enable_wasmtime = parent.enable_wasmtime, |
| 402 | 341 | .enable_wine = parent.enable_wine, |
| 403 | .libc_runtimes_dir = parent.libc_runtimes_dir, | |
| 404 | 342 | .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }), |
| 405 | 343 | .modules = .empty, |
| 406 | 344 | .named_writefiles = .empty, |
| 407 | 345 | .named_lazy_paths = .empty, |
| 408 | 346 | .pkg_hash = pkg_hash, |
| 409 | 347 | .available_deps = pkg_deps, |
| 410 | .release_mode = parent.release_mode, | |
| 411 | 348 | }; |
| 412 | 349 | try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls); |
| 413 | 350 | try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls); |
| ... | ... | @@ -702,59 +639,6 @@ fn hashUserInputOptionsMap(allocator: Allocator, user_input_options: UserInputOp |
| 702 | 639 | user_option.hash(hasher); |
| 703 | 640 | } |
| 704 | 641 | |
| 705 | fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { | |
| 706 | // Create an installation directory local to this package. This will be used when | |
| 707 | // dependant packages require a standard prefix, such as include directories for C headers. | |
| 708 | var hash = b.graph.cache.hash; | |
| 709 | // Random bytes to make unique. Refresh this with new random bytes when | |
| 710 | // implementation is modified in a non-backwards-compatible way. | |
| 711 | hash.add(@as(u32, 0xd8cb0055)); | |
| 712 | hash.addBytes(b.dep_prefix); | |
| 713 | ||
| 714 | var wyhash = std.hash.Wyhash.init(0); | |
| 715 | hashUserInputOptionsMap(b.allocator, b.user_input_options, &wyhash); | |
| 716 | hash.add(wyhash.final()); | |
| 717 | ||
| 718 | const digest = hash.final(); | |
| 719 | const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); | |
| 720 | b.resolveInstallPrefix(install_prefix, .{}); | |
| 721 | } | |
| 722 | ||
| 723 | /// This function is intended to be called by lib/build_runner.zig, not a build.zig file. | |
| 724 | pub fn resolveInstallPrefix(b: *Build, install_prefix: ?[]const u8, dir_list: DirList) void { | |
| 725 | if (b.dest_dir) |dest_dir| { | |
| 726 | b.install_prefix = install_prefix orelse "/usr"; | |
| 727 | b.install_path = b.pathJoin(&.{ dest_dir, b.install_prefix }); | |
| 728 | } else { | |
| 729 | b.install_prefix = install_prefix orelse | |
| 730 | (b.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); | |
| 731 | b.install_path = b.install_prefix; | |
| 732 | } | |
| 733 | ||
| 734 | var lib_list = [_][]const u8{ b.install_path, "lib" }; | |
| 735 | var exe_list = [_][]const u8{ b.install_path, "bin" }; | |
| 736 | var h_list = [_][]const u8{ b.install_path, "include" }; | |
| 737 | ||
| 738 | if (dir_list.lib_dir) |dir| { | |
| 739 | if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; | |
| 740 | lib_list[1] = dir; | |
| 741 | } | |
| 742 | ||
| 743 | if (dir_list.exe_dir) |dir| { | |
| 744 | if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; | |
| 745 | exe_list[1] = dir; | |
| 746 | } | |
| 747 | ||
| 748 | if (dir_list.include_dir) |dir| { | |
| 749 | if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; | |
| 750 | h_list[1] = dir; | |
| 751 | } | |
| 752 | ||
| 753 | b.lib_dir = b.pathJoin(&lib_list); | |
| 754 | b.exe_dir = b.pathJoin(&exe_list); | |
| 755 | b.h_dir = b.pathJoin(&h_list); | |
| 756 | } | |
| 757 | ||
| 758 | 642 | /// Create a set of key-value pairs that can be converted into a Zig source |
| 759 | 643 | /// file and then inserted into a Zig compilation's module table for importing. |
| 760 | 644 | /// In other words, this provides a way to expose build.zig values to Zig |
| ... | ... | @@ -1121,15 +1005,6 @@ pub fn getUninstallStep(b: *Build) *Step { |
| 1121 | 1005 | return &b.uninstall_tls.step; |
| 1122 | 1006 | } |
| 1123 | 1007 | |
| 1124 | fn makeUninstall(uninstall_step: *Step, options: Step.MakeOptions) anyerror!void { | |
| 1125 | _ = options; | |
| 1126 | const uninstall_tls: *TopLevelStep = @fieldParentPtr("step", uninstall_step); | |
| 1127 | const b: *Build = @fieldParentPtr("uninstall_tls", uninstall_tls); | |
| 1128 | ||
| 1129 | _ = b; | |
| 1130 | @panic("TODO implement https://github.com/ziglang/zig/issues/14943"); | |
| 1131 | } | |
| 1132 | ||
| 1133 | 1008 | /// Creates a configuration option to be passed to the build.zig script. |
| 1134 | 1009 | /// When a user directly runs `zig build`, they can set these options with `-D` arguments. |
| 1135 | 1010 | /// When a project depends on a Zig package as a dependency, it programmatically sets |
| ... | ... | @@ -1350,10 +1225,10 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw |
| 1350 | 1225 | } |
| 1351 | 1226 | |
| 1352 | 1227 | pub fn step(b: *Build, name: []const u8, description: []const u8) *Step { |
| 1353 | const step_info = b.allocator.create(TopLevelStep) catch @panic("OOM"); | |
| 1228 | const step_info = b.allocator.create(Step.TopLevel) catch @panic("OOM"); | |
| 1354 | 1229 | step_info.* = .{ |
| 1355 | 1230 | .step = .init(.{ |
| 1356 | .id = TopLevelStep.base_id, | |
| 1231 | .tag = .top_level, | |
| 1357 | 1232 | .name = name, |
| 1358 | 1233 | .owner = b, |
| 1359 | 1234 | }), |
| ... | ... | @@ -1373,8 +1248,10 @@ pub const StandardOptimizeOptionOptions = struct { |
| 1373 | 1248 | }; |
| 1374 | 1249 | |
| 1375 | 1250 | pub fn standardOptimizeOption(b: *Build, options: StandardOptimizeOptionOptions) std.builtin.OptimizeMode { |
| 1251 | const graph = b.graph; | |
| 1252 | ||
| 1376 | 1253 | if (options.preferred_optimize_mode) |mode| { |
| 1377 | if (b.option(bool, "release", "optimize for end users") orelse (b.release_mode != .off)) { | |
| 1254 | if (b.option(bool, "release", "optimize for end users") orelse (graph.release_mode != .off)) { | |
| 1378 | 1255 | return mode; |
| 1379 | 1256 | } else { |
| 1380 | 1257 | return .Debug; |
| ... | ... | @@ -1389,7 +1266,7 @@ pub fn standardOptimizeOption(b: *Build, options: StandardOptimizeOptionOptions) |
| 1389 | 1266 | return mode; |
| 1390 | 1267 | } |
| 1391 | 1268 | |
| 1392 | return switch (b.release_mode) { | |
| 1269 | return switch (graph.release_mode) { | |
| 1393 | 1270 | .off => .Debug, |
| 1394 | 1271 | .any => { |
| 1395 | 1272 | std.debug.print("the project does not declare a preferred optimization mode. choose: --release=fast, --release=safe, or --release=small\n", .{}); |
| ... | ... | @@ -1824,36 +1701,11 @@ fn tryFindProgram(b: *Build, full_path: []const u8) ?[]const u8 { |
| 1824 | 1701 | return null; |
| 1825 | 1702 | } |
| 1826 | 1703 | |
| 1827 | pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) error{FileNotFound}![]const u8 { | |
| 1828 | // TODO report error for ambiguous situations | |
| 1829 | for (b.search_prefixes.items) |search_prefix| { | |
| 1830 | for (names) |name| { | |
| 1831 | if (fs.path.isAbsolute(name)) { | |
| 1832 | return name; | |
| 1833 | } | |
| 1834 | return tryFindProgram(b, b.pathJoin(&.{ search_prefix, "bin", name })) orelse continue; | |
| 1835 | } | |
| 1836 | } | |
| 1837 | if (b.graph.environ_map.get("PATH")) |PATH| { | |
| 1838 | for (names) |name| { | |
| 1839 | if (fs.path.isAbsolute(name)) { | |
| 1840 | return name; | |
| 1841 | } | |
| 1842 | var it = mem.tokenizeScalar(u8, PATH, fs.path.delimiter); | |
| 1843 | while (it.next()) |p| { | |
| 1844 | return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; | |
| 1845 | } | |
| 1846 | } | |
| 1847 | } | |
| 1848 | for (names) |name| { | |
| 1849 | if (fs.path.isAbsolute(name)) { | |
| 1850 | return name; | |
| 1851 | } | |
| 1852 | for (paths) |p| { | |
| 1853 | return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; | |
| 1854 | } | |
| 1855 | } | |
| 1856 | return error.FileNotFound; | |
| 1704 | pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) LazyPath { | |
| 1705 | _ = b; | |
| 1706 | _ = names; | |
| 1707 | _ = paths; | |
| 1708 | @panic("TODO rework findProgram to be based on LazyPath"); | |
| 1857 | 1709 | } |
| 1858 | 1710 | |
| 1859 | 1711 | pub fn runAllowFail( |
| ... | ... | @@ -1918,10 +1770,6 @@ pub fn run(b: *Build, argv: []const []const u8) []u8 { |
| 1918 | 1770 | ); |
| 1919 | 1771 | } |
| 1920 | 1772 | |
| 1921 | pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { | |
| 1922 | b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); | |
| 1923 | } | |
| 1924 | ||
| 1925 | 1773 | pub fn getInstallPath(b: *Build, dir: InstallDir, dest_rel_path: []const u8) []const u8 { |
| 1926 | 1774 | assert(!fs.path.isAbsolute(dest_rel_path)); // Install paths must be relative to the prefix |
| 1927 | 1775 | const base_dir = switch (dir) { |
lib/std/Build/Fuzz.zig deleted-597
| ... | ... | @@ -1,597 +0,0 @@ |
| 1 | const std = @import("../std.zig"); | |
| 2 | const Io = std.Io; | |
| 3 | const Build = std.Build; | |
| 4 | const Cache = Build.Cache; | |
| 5 | const Step = std.Build.Step; | |
| 6 | const assert = std.debug.assert; | |
| 7 | const fatal = std.process.fatal; | |
| 8 | const Allocator = std.mem.Allocator; | |
| 9 | const log = std.log; | |
| 10 | const Coverage = std.debug.Coverage; | |
| 11 | const abi = Build.abi.fuzz; | |
| 12 | ||
| 13 | const Fuzz = @This(); | |
| 14 | const build_runner = @import("root"); | |
| 15 | ||
| 16 | gpa: Allocator, | |
| 17 | io: Io, | |
| 18 | mode: Mode, | |
| 19 | ||
| 20 | /// Allocated into `gpa`. | |
| 21 | run_steps: []const *Step.Run, | |
| 22 | ||
| 23 | group: Io.Group, | |
| 24 | root_prog_node: std.Progress.Node, | |
| 25 | prog_node: std.Progress.Node, | |
| 26 | ||
| 27 | /// Protects `coverage_files`. | |
| 28 | coverage_mutex: Io.Mutex, | |
| 29 | coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap), | |
| 30 | ||
| 31 | queue_mutex: Io.Mutex, | |
| 32 | queue_cond: Io.Condition, | |
| 33 | msg_queue: std.ArrayList(Msg), | |
| 34 | ||
| 35 | pub const Mode = union(enum) { | |
| 36 | forever: struct { ws: *Build.WebServer }, | |
| 37 | limit: Limited, | |
| 38 | ||
| 39 | pub const Limited = struct { | |
| 40 | amount: u64, | |
| 41 | }; | |
| 42 | }; | |
| 43 | ||
| 44 | const Msg = union(enum) { | |
| 45 | coverage: struct { | |
| 46 | id: u64, | |
| 47 | cumulative: struct { | |
| 48 | runs: u64, | |
| 49 | unique: u64, | |
| 50 | coverage: u64, | |
| 51 | }, | |
| 52 | run: *Step.Run, | |
| 53 | }, | |
| 54 | entry_point: struct { | |
| 55 | coverage_id: u64, | |
| 56 | addr: u64, | |
| 57 | }, | |
| 58 | }; | |
| 59 | ||
| 60 | const CoverageMap = struct { | |
| 61 | mapped_memory: []align(std.heap.page_size_min) const u8, | |
| 62 | coverage: Coverage, | |
| 63 | source_locations: []Coverage.SourceLocation, | |
| 64 | /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. | |
| 65 | entry_points: std.ArrayList(u32), | |
| 66 | start_timestamp: i64, | |
| 67 | start_n_runs: u64, | |
| 68 | ||
| 69 | fn deinit(cm: *CoverageMap, gpa: Allocator) void { | |
| 70 | std.posix.munmap(cm.mapped_memory); | |
| 71 | cm.coverage.deinit(gpa); | |
| 72 | cm.* = undefined; | |
| 73 | } | |
| 74 | }; | |
| 75 | ||
| 76 | pub fn init( | |
| 77 | gpa: Allocator, | |
| 78 | io: Io, | |
| 79 | all_steps: []const *Build.Step, | |
| 80 | root_prog_node: std.Progress.Node, | |
| 81 | mode: Mode, | |
| 82 | ) error{ OutOfMemory, Canceled }!Fuzz { | |
| 83 | const run_steps: []const *Step.Run = steps: { | |
| 84 | var steps: std.ArrayList(*Step.Run) = .empty; | |
| 85 | defer steps.deinit(gpa); | |
| 86 | const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0); | |
| 87 | defer rebuild_node.end(); | |
| 88 | var rebuild_group: Io.Group = .init; | |
| 89 | defer rebuild_group.cancel(io); | |
| 90 | ||
| 91 | for (all_steps) |step| { | |
| 92 | const run = step.cast(Step.Run) orelse continue; | |
| 93 | if (run.producer == null) continue; | |
| 94 | if (run.fuzz_tests.items.len == 0) continue; | |
| 95 | try steps.append(gpa, run); | |
| 96 | rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node }); | |
| 97 | } | |
| 98 | ||
| 99 | if (steps.items.len == 0) fatal("no fuzz tests found", .{}); | |
| 100 | rebuild_node.setEstimatedTotalItems(steps.items.len); | |
| 101 | const run_steps = try gpa.dupe(*Step.Run, steps.items); | |
| 102 | try rebuild_group.await(io); | |
| 103 | break :steps run_steps; | |
| 104 | }; | |
| 105 | errdefer gpa.free(run_steps); | |
| 106 | ||
| 107 | for (run_steps) |run| { | |
| 108 | assert(run.fuzz_tests.items.len > 0); | |
| 109 | if (run.rebuilt_executable == null) | |
| 110 | fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); | |
| 111 | } | |
| 112 | ||
| 113 | return .{ | |
| 114 | .gpa = gpa, | |
| 115 | .io = io, | |
| 116 | .mode = mode, | |
| 117 | .run_steps = run_steps, | |
| 118 | .group = .init, | |
| 119 | .root_prog_node = root_prog_node, | |
| 120 | .prog_node = .none, | |
| 121 | .coverage_files = .empty, | |
| 122 | .coverage_mutex = .init, | |
| 123 | .queue_mutex = .init, | |
| 124 | .queue_cond = .init, | |
| 125 | .msg_queue = .empty, | |
| 126 | }; | |
| 127 | } | |
| 128 | ||
| 129 | pub fn start(fuzz: *Fuzz) void { | |
| 130 | const io = fuzz.io; | |
| 131 | fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0); | |
| 132 | ||
| 133 | if (fuzz.mode == .forever) { | |
| 134 | // For polling messages and sending updates to subscribers. | |
| 135 | fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err| | |
| 136 | fatal("unable to spawn coverage task: {t}", .{err}); | |
| 137 | } | |
| 138 | ||
| 139 | for (fuzz.run_steps) |run| { | |
| 140 | assert(run.rebuilt_executable != null); | |
| 141 | fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run }); | |
| 142 | } | |
| 143 | } | |
| 144 | ||
| 145 | pub fn deinit(fuzz: *Fuzz) void { | |
| 146 | const io = fuzz.io; | |
| 147 | fuzz.group.cancel(io); | |
| 148 | fuzz.prog_node.end(); | |
| 149 | fuzz.gpa.free(fuzz.run_steps); | |
| 150 | } | |
| 151 | ||
| 152 | fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) void { | |
| 153 | rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { | |
| 154 | const compile = run.producer.?; | |
| 155 | log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); | |
| 156 | }; | |
| 157 | } | |
| 158 | ||
| 159 | fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { | |
| 160 | const graph = run.step.owner.graph; | |
| 161 | const io = graph.io; | |
| 162 | const compile = run.producer.?; | |
| 163 | const prog_node = parent_prog_node.start(compile.step.name, 0); | |
| 164 | defer prog_node.end(); | |
| 165 | ||
| 166 | const result = compile.rebuildInFuzzMode(gpa, prog_node); | |
| 167 | ||
| 168 | const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; | |
| 169 | const show_error_msgs = compile.step.result_error_msgs.items.len > 0; | |
| 170 | const show_stderr = compile.step.result_stderr.len > 0; | |
| 171 | ||
| 172 | if (show_error_msgs or show_compile_errors or show_stderr) { | |
| 173 | var buf: [256]u8 = undefined; | |
| 174 | const stderr = try io.lockStderr(&buf, graph.stderr_mode); | |
| 175 | defer io.unlockStderr(); | |
| 176 | build_runner.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; | |
| 177 | } | |
| 178 | ||
| 179 | const rebuilt_bin_path = result catch |err| switch (err) { | |
| 180 | error.MakeFailed => return, | |
| 181 | else => |other| return other, | |
| 182 | }; | |
| 183 | run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); | |
| 184 | } | |
| 185 | ||
| 186 | fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run) void { | |
| 187 | const owner = run.step.owner; | |
| 188 | const gpa = owner.allocator; | |
| 189 | const graph = owner.graph; | |
| 190 | const io = graph.io; | |
| 191 | ||
| 192 | run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) { | |
| 193 | error.MakeFailed => { | |
| 194 | var buf: [256]u8 = undefined; | |
| 195 | const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) { | |
| 196 | error.Canceled => return, | |
| 197 | }; | |
| 198 | defer io.unlockStderr(); | |
| 199 | build_runner.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; | |
| 200 | return; | |
| 201 | }, | |
| 202 | else => { | |
| 203 | log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err }); | |
| 204 | return; | |
| 205 | }, | |
| 206 | }; | |
| 207 | } | |
| 208 | ||
| 209 | pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { | |
| 210 | assert(fuzz.mode == .forever); | |
| 211 | ||
| 212 | var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); | |
| 213 | defer arena_state.deinit(); | |
| 214 | const arena = arena_state.allocator(); | |
| 215 | ||
| 216 | const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); | |
| 217 | var dedup_table: DedupTable = .empty; | |
| 218 | defer dedup_table.deinit(fuzz.gpa); | |
| 219 | ||
| 220 | for (fuzz.run_steps) |run_step| { | |
| 221 | const compile_inputs = run_step.producer.?.step.inputs.table; | |
| 222 | for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { | |
| 223 | try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len); | |
| 224 | for (file_list.items) |sub_path| { | |
| 225 | if (!std.mem.endsWith(u8, sub_path, ".zig")) continue; | |
| 226 | const joined_path = try dir_path.join(arena, sub_path); | |
| 227 | dedup_table.putAssumeCapacity(joined_path, {}); | |
| 228 | } | |
| 229 | } | |
| 230 | } | |
| 231 | ||
| 232 | const deduped_paths = dedup_table.keys(); | |
| 233 | const SortContext = struct { | |
| 234 | pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { | |
| 235 | _ = this; | |
| 236 | return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { | |
| 237 | .lt => true, | |
| 238 | .gt => false, | |
| 239 | .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), | |
| 240 | }; | |
| 241 | } | |
| 242 | }; | |
| 243 | std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); | |
| 244 | return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths); | |
| 245 | } | |
| 246 | ||
| 247 | pub const Previous = struct { | |
| 248 | unique_runs: usize, | |
| 249 | entry_points: usize, | |
| 250 | sent_source_index: bool, | |
| 251 | pub const init: Previous = .{ | |
| 252 | .unique_runs = 0, | |
| 253 | .entry_points = 0, | |
| 254 | .sent_source_index = false, | |
| 255 | }; | |
| 256 | }; | |
| 257 | pub fn sendUpdate( | |
| 258 | fuzz: *Fuzz, | |
| 259 | socket: *std.http.Server.WebSocket, | |
| 260 | prev: *Previous, | |
| 261 | ) !void { | |
| 262 | const io = fuzz.io; | |
| 263 | ||
| 264 | try fuzz.coverage_mutex.lock(io); | |
| 265 | defer fuzz.coverage_mutex.unlock(io); | |
| 266 | ||
| 267 | const coverage_maps = fuzz.coverage_files.values(); | |
| 268 | if (coverage_maps.len == 0) return; | |
| 269 | // TODO: handle multiple fuzz steps in the WebSocket packets | |
| 270 | const coverage_map = &coverage_maps[0]; | |
| 271 | const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); | |
| 272 | // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the | |
| 273 | // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the | |
| 274 | // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass | |
| 275 | // this data straight to the socket with sendfile... | |
| 276 | const seen_pcs = cov_header.seenBits(); | |
| 277 | const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); | |
| 278 | const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); | |
| 279 | { | |
| 280 | if (!prev.sent_source_index) { | |
| 281 | prev.sent_source_index = true; | |
| 282 | // We need to send initial context. | |
| 283 | const header: abi.SourceIndexHeader = .{ | |
| 284 | .directories_len = @intCast(coverage_map.coverage.directories.entries.len), | |
| 285 | .files_len = @intCast(coverage_map.coverage.files.entries.len), | |
| 286 | .source_locations_len = @intCast(coverage_map.source_locations.len), | |
| 287 | .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), | |
| 288 | .start_timestamp = coverage_map.start_timestamp, | |
| 289 | .start_n_runs = coverage_map.start_n_runs, | |
| 290 | }; | |
| 291 | var iovecs: [5][]const u8 = .{ | |
| 292 | @ptrCast(&header), | |
| 293 | @ptrCast(coverage_map.coverage.directories.keys()), | |
| 294 | @ptrCast(coverage_map.coverage.files.keys()), | |
| 295 | @ptrCast(coverage_map.source_locations), | |
| 296 | coverage_map.coverage.string_bytes.items, | |
| 297 | }; | |
| 298 | try socket.writeMessageVec(&iovecs, .binary); | |
| 299 | } | |
| 300 | ||
| 301 | const header: abi.CoverageUpdateHeader = .{ | |
| 302 | .n_runs = n_runs, | |
| 303 | .unique_runs = unique_runs, | |
| 304 | }; | |
| 305 | var iovecs: [2][]const u8 = .{ | |
| 306 | @ptrCast(&header), | |
| 307 | @ptrCast(seen_pcs), | |
| 308 | }; | |
| 309 | try socket.writeMessageVec(&iovecs, .binary); | |
| 310 | ||
| 311 | prev.unique_runs = unique_runs; | |
| 312 | } | |
| 313 | ||
| 314 | if (prev.entry_points != coverage_map.entry_points.items.len) { | |
| 315 | const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len)); | |
| 316 | var iovecs: [2][]const u8 = .{ | |
| 317 | @ptrCast(&header), | |
| 318 | @ptrCast(coverage_map.entry_points.items), | |
| 319 | }; | |
| 320 | try socket.writeMessageVec(&iovecs, .binary); | |
| 321 | ||
| 322 | prev.entry_points = coverage_map.entry_points.items.len; | |
| 323 | } | |
| 324 | } | |
| 325 | ||
| 326 | fn coverageRun(fuzz: *Fuzz) void { | |
| 327 | coverageRunCancelable(fuzz) catch |err| switch (err) { | |
| 328 | error.Canceled => return, | |
| 329 | }; | |
| 330 | } | |
| 331 | ||
| 332 | fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { | |
| 333 | const io = fuzz.io; | |
| 334 | ||
| 335 | try fuzz.queue_mutex.lock(io); | |
| 336 | defer fuzz.queue_mutex.unlock(io); | |
| 337 | ||
| 338 | while (true) { | |
| 339 | try fuzz.queue_cond.wait(io, &fuzz.queue_mutex); | |
| 340 | for (fuzz.msg_queue.items) |msg| switch (msg) { | |
| 341 | .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) { | |
| 342 | error.AlreadyReported => continue, | |
| 343 | error.Canceled => return, | |
| 344 | else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), | |
| 345 | }, | |
| 346 | .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) { | |
| 347 | error.AlreadyReported => continue, | |
| 348 | error.Canceled => return, | |
| 349 | else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), | |
| 350 | }, | |
| 351 | }; | |
| 352 | fuzz.msg_queue.clearRetainingCapacity(); | |
| 353 | } | |
| 354 | } | |
| 355 | fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { | |
| 356 | assert(fuzz.mode == .forever); | |
| 357 | const ws = fuzz.mode.forever.ws; | |
| 358 | const gpa = fuzz.gpa; | |
| 359 | const io = fuzz.io; | |
| 360 | ||
| 361 | try fuzz.coverage_mutex.lock(io); | |
| 362 | defer fuzz.coverage_mutex.unlock(io); | |
| 363 | ||
| 364 | const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id); | |
| 365 | if (gop.found_existing) { | |
| 366 | // We are fuzzing the same executable with multiple threads. | |
| 367 | // Perhaps the same unit test; perhaps a different one. In any | |
| 368 | // case, since the coverage file is the same, we only have to | |
| 369 | // notice changes to that one file in order to learn coverage for | |
| 370 | // this particular executable. | |
| 371 | return; | |
| 372 | } | |
| 373 | errdefer _ = fuzz.coverage_files.pop(); | |
| 374 | ||
| 375 | gop.value_ptr.* = .{ | |
| 376 | .coverage = std.debug.Coverage.init, | |
| 377 | .mapped_memory = undefined, // populated below | |
| 378 | .source_locations = undefined, // populated below | |
| 379 | .entry_points = .empty, | |
| 380 | .start_timestamp = ws.now(), | |
| 381 | .start_n_runs = undefined, // populated below | |
| 382 | }; | |
| 383 | errdefer gop.value_ptr.coverage.deinit(gpa); | |
| 384 | ||
| 385 | const rebuilt_exe_path = run_step.rebuilt_executable.?; | |
| 386 | const target = run_step.producer.?.rootModuleTarget(); | |
| 387 | var debug_info = std.debug.Info.load( | |
| 388 | gpa, | |
| 389 | io, | |
| 390 | rebuilt_exe_path, | |
| 391 | &gop.value_ptr.coverage, | |
| 392 | target.ofmt, | |
| 393 | target.cpu.arch, | |
| 394 | ) catch |err| { | |
| 395 | log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ | |
| 396 | run_step.step.name, rebuilt_exe_path, err, | |
| 397 | }); | |
| 398 | return error.AlreadyReported; | |
| 399 | }; | |
| 400 | defer debug_info.deinit(gpa); | |
| 401 | ||
| 402 | const coverage_file_path: Build.Cache.Path = .{ | |
| 403 | .root_dir = run_step.step.owner.cache_root, | |
| 404 | .sub_path = "v/" ++ std.fmt.hex(coverage_id), | |
| 405 | }; | |
| 406 | var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { | |
| 407 | log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ | |
| 408 | run_step.step.name, coverage_file_path, err, | |
| 409 | }); | |
| 410 | return error.AlreadyReported; | |
| 411 | }; | |
| 412 | defer coverage_file.close(io); | |
| 413 | ||
| 414 | const file_size = coverage_file.length(io) catch |err| { | |
| 415 | log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err }); | |
| 416 | return error.AlreadyReported; | |
| 417 | }; | |
| 418 | ||
| 419 | const mapped_memory = std.posix.mmap( | |
| 420 | null, | |
| 421 | file_size, | |
| 422 | .{ .READ = true }, | |
| 423 | .{ .TYPE = .SHARED }, | |
| 424 | coverage_file.handle, | |
| 425 | 0, | |
| 426 | ) catch |err| { | |
| 427 | log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err }); | |
| 428 | return error.AlreadyReported; | |
| 429 | }; | |
| 430 | gop.value_ptr.mapped_memory = mapped_memory; | |
| 431 | ||
| 432 | const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); | |
| 433 | const pcs = header.pcAddrs(); | |
| 434 | const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len); | |
| 435 | errdefer gpa.free(source_locations); | |
| 436 | ||
| 437 | // Unfortunately the PCs array that LLVM gives us from the 8-bit PC | |
| 438 | // counters feature is not sorted. | |
| 439 | var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .empty; | |
| 440 | defer sorted_pcs.deinit(gpa); | |
| 441 | try sorted_pcs.resize(gpa, pcs.len); | |
| 442 | @memcpy(sorted_pcs.items(.pc), pcs); | |
| 443 | for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i); | |
| 444 | sorted_pcs.sortUnstable(struct { | |
| 445 | addrs: []const u64, | |
| 446 | ||
| 447 | pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { | |
| 448 | return ctx.addrs[a_index] < ctx.addrs[b_index]; | |
| 449 | } | |
| 450 | }{ .addrs = sorted_pcs.items(.pc) }); | |
| 451 | ||
| 452 | debug_info.resolveAddresses(gpa, io, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| { | |
| 453 | log.err("failed to resolve addresses to source locations: {t}", .{err}); | |
| 454 | return error.AlreadyReported; | |
| 455 | }; | |
| 456 | ||
| 457 | for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl; | |
| 458 | gop.value_ptr.source_locations = source_locations; | |
| 459 | gop.value_ptr.start_n_runs = header.n_runs; | |
| 460 | ||
| 461 | ws.notifyUpdate(); | |
| 462 | } | |
| 463 | ||
| 464 | fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void { | |
| 465 | const io = fuzz.io; | |
| 466 | ||
| 467 | try fuzz.coverage_mutex.lock(io); | |
| 468 | defer fuzz.coverage_mutex.unlock(io); | |
| 469 | ||
| 470 | const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?; | |
| 471 | const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); | |
| 472 | const pcs = header.pcAddrs(); | |
| 473 | ||
| 474 | // Since this pcs list is unsorted, we must linear scan for the best index. | |
| 475 | const index = i: { | |
| 476 | var best: usize = 0; | |
| 477 | for (pcs[1..], 1..) |elem_addr, i| { | |
| 478 | if (elem_addr == addr) break :i i; | |
| 479 | if (elem_addr > addr) continue; | |
| 480 | if (elem_addr > pcs[best]) best = i; | |
| 481 | } | |
| 482 | break :i best; | |
| 483 | }; | |
| 484 | if (index >= pcs.len) { | |
| 485 | log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{ | |
| 486 | addr, pcs[0], pcs[pcs.len - 1], | |
| 487 | }); | |
| 488 | return error.AlreadyReported; | |
| 489 | } | |
| 490 | if (false) { | |
| 491 | const sl = coverage_map.source_locations[index]; | |
| 492 | const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename); | |
| 493 | if (pcs.len == 1) { | |
| 494 | log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{ | |
| 495 | addr, file_name, sl.line, sl.column, | |
| 496 | }); | |
| 497 | } else if (index == 0) { | |
| 498 | log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{ | |
| 499 | addr, file_name, sl.line, sl.column, pcs[index + 1], | |
| 500 | }); | |
| 501 | } else if (index == pcs.len - 1) { | |
| 502 | log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{ | |
| 503 | addr, file_name, sl.line, sl.column, index, pcs[index - 1], | |
| 504 | }); | |
| 505 | } else { | |
| 506 | log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{ | |
| 507 | addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1], | |
| 508 | }); | |
| 509 | } | |
| 510 | } | |
| 511 | try coverage_map.entry_points.append(fuzz.gpa, @intCast(index)); | |
| 512 | } | |
| 513 | ||
| 514 | pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { | |
| 515 | assert(fuzz.mode == .limit); | |
| 516 | const io = fuzz.io; | |
| 517 | ||
| 518 | try fuzz.group.await(io); | |
| 519 | fuzz.group = .init; | |
| 520 | ||
| 521 | std.debug.print("======= FUZZING REPORT =======\n", .{}); | |
| 522 | for (fuzz.msg_queue.items) |msg| { | |
| 523 | if (msg != .coverage) continue; | |
| 524 | ||
| 525 | const cov = msg.coverage; | |
| 526 | const coverage_file_path: std.Build.Cache.Path = .{ | |
| 527 | .root_dir = cov.run.step.owner.cache_root, | |
| 528 | .sub_path = "v/" ++ std.fmt.hex(cov.id), | |
| 529 | }; | |
| 530 | var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { | |
| 531 | fatal("step '{s}': failed to load coverage file '{f}': {t}", .{ | |
| 532 | cov.run.step.name, coverage_file_path, err, | |
| 533 | }); | |
| 534 | }; | |
| 535 | defer coverage_file.close(io); | |
| 536 | ||
| 537 | const fuzz_abi = std.Build.abi.fuzz; | |
| 538 | var rbuf: [0x1000]u8 = undefined; | |
| 539 | var r = coverage_file.reader(io, &rbuf); | |
| 540 | ||
| 541 | var header: fuzz_abi.SeenPcsHeader = undefined; | |
| 542 | r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { | |
| 543 | fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ | |
| 544 | cov.run.step.name, coverage_file_path, err, | |
| 545 | }); | |
| 546 | }; | |
| 547 | ||
| 548 | if (header.pcs_len == 0) { | |
| 549 | fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{ | |
| 550 | cov.run.step.name, coverage_file_path, | |
| 551 | }); | |
| 552 | } | |
| 553 | ||
| 554 | var seen_count: usize = 0; | |
| 555 | const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len); | |
| 556 | for (0..chunk_count) |_| { | |
| 557 | const seen = r.interface.takeInt(usize, .little) catch |err| { | |
| 558 | fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ | |
| 559 | cov.run.step.name, coverage_file_path, err, | |
| 560 | }); | |
| 561 | }; | |
| 562 | seen_count += @popCount(seen); | |
| 563 | } | |
| 564 | ||
| 565 | const seen_f: f64 = @floatFromInt(seen_count); | |
| 566 | const total_f: f64 = @floatFromInt(header.pcs_len); | |
| 567 | const ratio = seen_f / total_f; | |
| 568 | std.debug.print( | |
| 569 | \\Step: {s} | |
| 570 | \\Fuzz test: "{s}" ({x}) | |
| 571 | \\Runs: {} -> {} | |
| 572 | \\Unique runs: {} -> {} | |
| 573 | \\Coverage: {}/{} -> {}/{} ({:.02}%) | |
| 574 | \\ | |
| 575 | , .{ | |
| 576 | cov.run.step.name, | |
| 577 | cov.run.fuzz_tests.items[0], | |
| 578 | cov.id, | |
| 579 | cov.cumulative.runs, | |
| 580 | header.n_runs, | |
| 581 | cov.cumulative.unique, | |
| 582 | header.unique_runs, | |
| 583 | cov.cumulative.coverage, | |
| 584 | header.pcs_len, | |
| 585 | seen_count, | |
| 586 | header.pcs_len, | |
| 587 | ratio * 100, | |
| 588 | }); | |
| 589 | ||
| 590 | std.debug.print("------------------------------\n", .{}); | |
| 591 | } | |
| 592 | std.debug.print( | |
| 593 | \\Values are accumulated across multiple runs when preserving the cache. | |
| 594 | \\============================== | |
| 595 | \\ | |
| 596 | , .{}); | |
| 597 | } |
lib/std/Build/Step.zig+33-882
| ... | ... | @@ -10,24 +10,11 @@ const Cache = Build.Cache; |
| 10 | 10 | const Path = Cache.Path; |
| 11 | 11 | const ArrayList = std.ArrayList; |
| 12 | 12 | |
| 13 | id: Id, | |
| 13 | tag: std.Build.Configuration.Step.Tag, | |
| 14 | 14 | name: []const u8, |
| 15 | 15 | owner: *Build, |
| 16 | makeFn: MakeFn, | |
| 17 | 16 | |
| 18 | dependencies: std.array_list.Managed(*Step), | |
| 19 | /// This field is empty during execution of the user's build script, and | |
| 20 | /// then populated during dependency loop checking in the build runner. | |
| 21 | dependants: ArrayList(*Step), | |
| 22 | /// Collects the set of files that retrigger this step to run. | |
| 23 | /// | |
| 24 | /// This is used by the build system's implementation of `--watch` but it can | |
| 25 | /// also be potentially useful for IDEs to know what effects editing a | |
| 26 | /// particular file has. | |
| 27 | /// | |
| 28 | /// Populated within `make`. Implementation may choose to clear and repopulate, | |
| 29 | /// retain previous value, or update. | |
| 30 | inputs: Inputs, | |
| 17 | dependencies: ArrayList(*Step), | |
| 31 | 18 | |
| 32 | 19 | /// Set this field to declare an upper bound on the amount of bytes of memory it will |
| 33 | 20 | /// take to run the step. Zero means no limit. |
| ... | ... | @@ -51,77 +38,11 @@ inputs: Inputs, |
| 51 | 38 | max_rss: usize, |
| 52 | 39 | |
| 53 | 40 | state: State, |
| 54 | pending_deps: u32, | |
| 55 | ||
| 56 | result_error_msgs: ArrayList([]const u8), | |
| 57 | result_error_bundle: std.zig.ErrorBundle, | |
| 58 | result_stderr: []const u8, | |
| 59 | result_cached: bool, | |
| 60 | result_duration_ns: ?u64, | |
| 61 | /// 0 means unavailable or not reported. | |
| 62 | result_peak_rss: usize, | |
| 63 | /// If the step is failed and this field is populated, this is the command which failed. | |
| 64 | /// This field may be populated even if the step succeeded. | |
| 65 | result_failed_command: ?[]const u8, | |
| 66 | test_results: TestResults, | |
| 67 | 41 | |
| 68 | 42 | /// The return address associated with creation of this step that can be useful |
| 69 | 43 | /// to print along with debugging messages. |
| 70 | 44 | debug_stack_trace: std.debug.StackTrace, |
| 71 | 45 | |
| 72 | pub const TestResults = struct { | |
| 73 | /// The total number of tests in the step. Every test has a "status" from the following: | |
| 74 | /// * passed | |
| 75 | /// * skipped | |
| 76 | /// * failed cleanly | |
| 77 | /// * crashed | |
| 78 | /// * timed out | |
| 79 | test_count: u32 = 0, | |
| 80 | ||
| 81 | /// The number of tests which were skipped (`error.SkipZigTest`). | |
| 82 | skip_count: u32 = 0, | |
| 83 | /// The number of tests which failed cleanly. | |
| 84 | fail_count: u32 = 0, | |
| 85 | /// The number of tests which terminated unexpectedly, i.e. crashed. | |
| 86 | crash_count: u32 = 0, | |
| 87 | /// The number of tests which timed out. | |
| 88 | timeout_count: u32 = 0, | |
| 89 | ||
| 90 | /// The number of detected memory leaks. The associated test may still have passed; indeed, *all* | |
| 91 | /// individual tests may have passed. However, the step as a whole fails if any test has leaks. | |
| 92 | leak_count: u32 = 0, | |
| 93 | /// The number of detected error logs. The associated test may still have passed; indeed, *all* | |
| 94 | /// individual tests may have passed. However, the step as a whole fails if any test logs errors. | |
| 95 | log_err_count: u32 = 0, | |
| 96 | ||
| 97 | pub fn isSuccess(tr: TestResults) bool { | |
| 98 | // all steps are success or skip | |
| 99 | return tr.fail_count == 0 and | |
| 100 | tr.crash_count == 0 and | |
| 101 | tr.timeout_count == 0 and | |
| 102 | // no (otherwise successful) step leaked memory or logged errors | |
| 103 | tr.leak_count == 0 and | |
| 104 | tr.log_err_count == 0; | |
| 105 | } | |
| 106 | ||
| 107 | /// Computes the number of tests which passed from the other values. | |
| 108 | pub fn passCount(tr: TestResults) u32 { | |
| 109 | return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count; | |
| 110 | } | |
| 111 | }; | |
| 112 | ||
| 113 | pub const MakeOptions = struct { | |
| 114 | progress_node: std.Progress.Node, | |
| 115 | watch: bool, | |
| 116 | web_server: ?*Build.WebServer, | |
| 117 | /// If set, this is a timeout to enforce on all individual unit tests, in nanoseconds. | |
| 118 | unit_test_timeout_ns: ?u64, | |
| 119 | /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. | |
| 120 | gpa: Allocator, | |
| 121 | }; | |
| 122 | ||
| 123 | pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; | |
| 124 | ||
| 125 | 46 | pub const State = enum { |
| 126 | 47 | precheck_unstarted, |
| 127 | 48 | precheck_started, |
| ... | ... | @@ -132,57 +53,29 @@ pub const State = enum { |
| 132 | 53 | /// be re-evaluated. |
| 133 | 54 | precheck_done, |
| 134 | 55 | dependency_failure, |
| 135 | success, | |
| 136 | failure, | |
| 137 | /// This state indicates that the step did not complete, however, it also did not fail, | |
| 138 | /// and it is safe to continue executing its dependencies. | |
| 139 | skipped, | |
| 140 | /// This step was skipped because it specified a max_rss that exceeded the runner's maximum. | |
| 141 | /// It is not safe to run its dependencies. | |
| 142 | skipped_oom, | |
| 143 | 56 | }; |
| 144 | 57 | |
| 145 | pub const Id = enum { | |
| 146 | top_level, | |
| 147 | compile, | |
| 148 | install_artifact, | |
| 149 | install_file, | |
| 150 | install_dir, | |
| 151 | remove_dir, | |
| 152 | fail, | |
| 153 | fmt, | |
| 154 | translate_c, | |
| 155 | write_file, | |
| 156 | update_source_files, | |
| 157 | run, | |
| 158 | check_file, | |
| 159 | check_object, | |
| 160 | config_header, | |
| 161 | objcopy, | |
| 162 | options, | |
| 163 | custom, | |
| 164 | ||
| 165 | pub fn Type(comptime id: Id) type { | |
| 166 | return switch (id) { | |
| 167 | .top_level => Build.TopLevelStep, | |
| 168 | .compile => Compile, | |
| 169 | .install_artifact => InstallArtifact, | |
| 170 | .install_file => InstallFile, | |
| 171 | .install_dir => InstallDir, | |
| 172 | .fail => Fail, | |
| 173 | .fmt => Fmt, | |
| 174 | .translate_c => TranslateC, | |
| 175 | .write_file => WriteFile, | |
| 176 | .update_source_files => UpdateSourceFiles, | |
| 177 | .run => Run, | |
| 178 | .check_file => CheckFile, | |
| 179 | .config_header => ConfigHeader, | |
| 180 | .objcopy => ObjCopy, | |
| 181 | .options => Options, | |
| 182 | .custom => @compileError("no type available for custom step"), | |
| 183 | }; | |
| 184 | } | |
| 185 | }; | |
| 58 | pub const Tag = std.Build.Configuration.Step.Tag; | |
| 59 | ||
| 60 | pub fn Type(comptime tag: Tag) type { | |
| 61 | return switch (tag) { | |
| 62 | .top_level => Build.TopLevelStep, | |
| 63 | .compile => Compile, | |
| 64 | .install_artifact => InstallArtifact, | |
| 65 | .install_file => InstallFile, | |
| 66 | .install_dir => InstallDir, | |
| 67 | .fail => Fail, | |
| 68 | .fmt => Fmt, | |
| 69 | .translate_c => TranslateC, | |
| 70 | .write_file => WriteFile, | |
| 71 | .update_source_files => UpdateSourceFiles, | |
| 72 | .run => Run, | |
| 73 | .check_file => CheckFile, | |
| 74 | .config_header => ConfigHeader, | |
| 75 | .objcopy => ObjCopy, | |
| 76 | .options => Options, | |
| 77 | }; | |
| 78 | } | |
| 186 | 79 | |
| 187 | 80 | pub const CheckFile = @import("Step/CheckFile.zig"); |
| 188 | 81 | pub const ConfigHeader = @import("Step/ConfigHeader.zig"); |
| ... | ... | @@ -199,32 +92,17 @@ pub const TranslateC = @import("Step/TranslateC.zig"); |
| 199 | 92 | pub const WriteFile = @import("Step/WriteFile.zig"); |
| 200 | 93 | pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); |
| 201 | 94 | |
| 202 | pub const Inputs = struct { | |
| 203 | table: Table, | |
| 204 | ||
| 205 | pub const init: Inputs = .{ | |
| 206 | .table = .{}, | |
| 207 | }; | |
| 95 | pub const TopLevel = struct { | |
| 96 | pub const base_tag: Step.Tag = .top_level; | |
| 208 | 97 | |
| 209 | pub const Table = std.ArrayHashMapUnmanaged(Build.Cache.Path, Files, Build.Cache.Path.TableAdapter, false); | |
| 210 | /// The special file name "." means any changes inside the directory. | |
| 211 | pub const Files = ArrayList([]const u8); | |
| 212 | ||
| 213 | pub fn populated(inputs: *Inputs) bool { | |
| 214 | return inputs.table.count() != 0; | |
| 215 | } | |
| 216 | ||
| 217 | pub fn clear(inputs: *Inputs, gpa: Allocator) void { | |
| 218 | for (inputs.table.values()) |*files| files.deinit(gpa); | |
| 219 | inputs.table.clearRetainingCapacity(); | |
| 220 | } | |
| 98 | step: Step, | |
| 99 | description: []const u8, | |
| 221 | 100 | }; |
| 222 | 101 | |
| 223 | 102 | pub const StepOptions = struct { |
| 224 | id: Id, | |
| 103 | tag: Tag, | |
| 225 | 104 | name: []const u8, |
| 226 | 105 | owner: *Build, |
| 227 | makeFn: MakeFn = makeNoOp, | |
| 228 | 106 | first_ret_addr: ?usize = null, |
| 229 | 107 | max_rss: usize = 0, |
| 230 | 108 | }; |
| ... | ... | @@ -233,90 +111,27 @@ pub fn init(options: StepOptions) Step { |
| 233 | 111 | const arena = options.owner.allocator; |
| 234 | 112 | |
| 235 | 113 | return .{ |
| 236 | .id = options.id, | |
| 114 | .tag = options.tag, | |
| 237 | 115 | .name = arena.dupe(u8, options.name) catch @panic("OOM"), |
| 238 | 116 | .owner = options.owner, |
| 239 | .makeFn = options.makeFn, | |
| 240 | .dependencies = std.array_list.Managed(*Step).init(arena), | |
| 241 | .dependants = .empty, | |
| 242 | .inputs = Inputs.init, | |
| 117 | .dependencies = .empty, | |
| 243 | 118 | .state = .precheck_unstarted, |
| 244 | .pending_deps = undefined, // initialized by build runner | |
| 245 | 119 | .max_rss = options.max_rss, |
| 246 | 120 | .debug_stack_trace = blk: { |
| 247 | 121 | const addr_buf = arena.alloc(usize, options.owner.debug_stack_frames_count) catch @panic("OOM"); |
| 248 | 122 | const first_ret_addr = options.first_ret_addr orelse @returnAddress(); |
| 249 | 123 | break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf); |
| 250 | 124 | }, |
| 251 | .result_error_msgs = .empty, | |
| 252 | .result_error_bundle = std.zig.ErrorBundle.empty, | |
| 253 | .result_stderr = "", | |
| 254 | .result_cached = false, | |
| 255 | .result_duration_ns = null, | |
| 256 | .result_peak_rss = 0, | |
| 257 | .result_failed_command = null, | |
| 258 | .test_results = .{}, | |
| 259 | }; | |
| 260 | } | |
| 261 | ||
| 262 | /// If the Step's `make` function reports `error.MakeFailed`, it indicates they | |
| 263 | /// have already reported the error. Otherwise, we add a simple error report | |
| 264 | /// here. | |
| 265 | pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { | |
| 266 | const arena = s.owner.allocator; | |
| 267 | const graph = s.owner.graph; | |
| 268 | const io = graph.io; | |
| 269 | ||
| 270 | var start_ts: ?Io.Timestamp = t: { | |
| 271 | if (!graph.time_report) break :t null; | |
| 272 | if (s.id == .compile) break :t null; | |
| 273 | if (s.id == .run and s.cast(Run).?.stdio == .zig_test) break :t null; | |
| 274 | break :t Io.Clock.awake.now(io); | |
| 275 | 125 | }; |
| 276 | const make_result = s.makeFn(s, options); | |
| 277 | if (start_ts) |*ts| { | |
| 278 | const duration = ts.untilNow(io, .awake); | |
| 279 | options.web_server.?.updateTimeReportGeneric(s, duration); | |
| 280 | } | |
| 281 | ||
| 282 | make_result catch |err| switch (err) { | |
| 283 | error.MakeFailed, error.MakeSkipped => |e| return e, | |
| 284 | else => { | |
| 285 | s.result_error_msgs.append(arena, @errorName(err)) catch @panic("OOM"); | |
| 286 | return error.MakeFailed; | |
| 287 | }, | |
| 288 | }; | |
| 289 | ||
| 290 | if (!s.test_results.isSuccess()) { | |
| 291 | return error.MakeFailed; | |
| 292 | } | |
| 293 | ||
| 294 | if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { | |
| 295 | const msg = std.fmt.allocPrint(arena, "memory usage peaked at {0B:.2} ({0d} bytes), exceeding the declared upper bound of {1B:.2} ({1d} bytes)", .{ | |
| 296 | s.result_peak_rss, s.max_rss, | |
| 297 | }) catch @panic("OOM"); | |
| 298 | s.result_error_msgs.append(arena, msg) catch @panic("OOM"); | |
| 299 | } | |
| 300 | 126 | } |
| 301 | 127 | |
| 302 | 128 | pub fn dependOn(step: *Step, other: *Step) void { |
| 303 | step.dependencies.append(other) catch @panic("OOM"); | |
| 304 | } | |
| 305 | ||
| 306 | fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { | |
| 307 | _ = options; | |
| 308 | ||
| 309 | var all_cached = true; | |
| 310 | ||
| 311 | for (step.dependencies.items) |dep| { | |
| 312 | all_cached = all_cached and dep.result_cached; | |
| 313 | } | |
| 314 | ||
| 315 | step.result_cached = all_cached; | |
| 129 | const arena = step.owner.allocator; | |
| 130 | step.dependencies.append(arena, other) catch @panic("OOM"); | |
| 316 | 131 | } |
| 317 | 132 | |
| 318 | 133 | pub fn cast(step: *Step, comptime T: type) ?*T { |
| 319 | if (step.id == T.base_id) { | |
| 134 | if (step.tag == T.base_tag) { | |
| 320 | 135 | return @fieldParentPtr("step", step); |
| 321 | 136 | } |
| 322 | 137 | return null; |
| ... | ... | @@ -337,670 +152,6 @@ pub fn dump(step: *Step, t: Io.Terminal) void { |
| 337 | 152 | } |
| 338 | 153 | } |
| 339 | 154 | |
| 340 | /// Populates `s.result_failed_command`. | |
| 341 | pub fn captureChildProcess( | |
| 342 | s: *Step, | |
| 343 | gpa: Allocator, | |
| 344 | progress_node: std.Progress.Node, | |
| 345 | argv: []const []const u8, | |
| 346 | ) !std.process.RunResult { | |
| 347 | const graph = s.owner.graph; | |
| 348 | const arena = graph.arena; | |
| 349 | const io = graph.io; | |
| 350 | ||
| 351 | // If an error occurs, it's happened in this command: | |
| 352 | assert(s.result_failed_command == null); | |
| 353 | s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); | |
| 354 | ||
| 355 | try handleChildProcUnsupported(s); | |
| 356 | try handleVerbose(s.owner, .inherit, argv); | |
| 357 | ||
| 358 | const result = std.process.run(arena, io, .{ | |
| 359 | .argv = argv, | |
| 360 | .environ_map = &graph.environ_map, | |
| 361 | .progress_node = progress_node, | |
| 362 | }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err }); | |
| 363 | ||
| 364 | if (result.stderr.len > 0) { | |
| 365 | try s.result_error_msgs.append(arena, result.stderr); | |
| 366 | } | |
| 367 | ||
| 368 | return result; | |
| 369 | } | |
| 370 | ||
| 371 | pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { | |
| 372 | try step.addError(fmt, args); | |
| 373 | return error.MakeFailed; | |
| 374 | } | |
| 375 | ||
| 376 | pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { | |
| 377 | const arena = step.owner.allocator; | |
| 378 | const msg = try std.fmt.allocPrint(arena, fmt, args); | |
| 379 | try step.result_error_msgs.append(arena, msg); | |
| 380 | } | |
| 381 | ||
| 382 | pub const ZigProcess = struct { | |
| 383 | child: std.process.Child, | |
| 384 | multi_reader_buffer: Io.File.MultiReader.Buffer(2), | |
| 385 | multi_reader: Io.File.MultiReader, | |
| 386 | progress_ipc_index: ?if (std.Progress.have_ipc) std.Progress.Ipc.Index else noreturn, | |
| 387 | ||
| 388 | pub const StreamEnum = enum { stdout, stderr }; | |
| 389 | ||
| 390 | pub fn saveState(zp: *ZigProcess, prog_node: std.Progress.Node) void { | |
| 391 | zp.progress_ipc_index = if (std.Progress.have_ipc) prog_node.takeIpcIndex() else null; | |
| 392 | } | |
| 393 | ||
| 394 | pub fn deinit(zp: *ZigProcess, io: Io) void { | |
| 395 | zp.child.kill(io); | |
| 396 | zp.multi_reader.deinit(); | |
| 397 | zp.* = undefined; | |
| 398 | } | |
| 399 | }; | |
| 400 | ||
| 401 | /// Assumes that argv contains `--listen=-` and that the process being spawned | |
| 402 | /// is the zig compiler - the same version that compiled the build runner. | |
| 403 | /// Populates `s.result_failed_command`. | |
| 404 | pub fn evalZigProcess( | |
| 405 | s: *Step, | |
| 406 | argv: []const []const u8, | |
| 407 | prog_node: std.Progress.Node, | |
| 408 | watch: bool, | |
| 409 | web_server: ?*Build.WebServer, | |
| 410 | gpa: Allocator, | |
| 411 | ) !?Path { | |
| 412 | const b = s.owner; | |
| 413 | const io = b.graph.io; | |
| 414 | ||
| 415 | // If an error occurs, it's happened in this command: | |
| 416 | assert(s.result_failed_command == null); | |
| 417 | s.result_failed_command = try allocPrintCmd(gpa, .inherit, null, argv); | |
| 418 | ||
| 419 | if (s.getZigProcess()) |zp| update: { | |
| 420 | assert(watch); | |
| 421 | if (zp.progress_ipc_index) |ipc_index| prog_node.setIpcIndex(ipc_index); | |
| 422 | zp.progress_ipc_index = null; | |
| 423 | var exited = false; | |
| 424 | defer if (exited) { | |
| 425 | s.cast(Compile).?.zig_process = null; | |
| 426 | zp.deinit(io); | |
| 427 | gpa.destroy(zp); | |
| 428 | } else zp.saveState(prog_node); | |
| 429 | const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { | |
| 430 | error.BrokenPipe, error.EndOfStream => |reason| { | |
| 431 | std.log.info("{s} restart required: {t}", .{ argv[0], reason }); | |
| 432 | // Process restart required. | |
| 433 | const term = zp.child.wait(io) catch |e| { | |
| 434 | return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); | |
| 435 | }; | |
| 436 | _ = term; | |
| 437 | exited = true; | |
| 438 | break :update; | |
| 439 | }, | |
| 440 | else => |e| return e, | |
| 441 | }; | |
| 442 | ||
| 443 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 444 | return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); | |
| 445 | } | |
| 446 | ||
| 447 | if (s.result_error_msgs.items.len > 0 and result == null) { | |
| 448 | // Crash detected. | |
| 449 | const term = zp.child.wait(io) catch |e| { | |
| 450 | return s.fail("unable to wait for {s}: {t}", .{ argv[0], e }); | |
| 451 | }; | |
| 452 | s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; | |
| 453 | exited = true; | |
| 454 | try handleChildProcessTerm(s, term); | |
| 455 | return error.MakeFailed; | |
| 456 | } | |
| 457 | ||
| 458 | return result; | |
| 459 | } | |
| 460 | assert(argv.len != 0); | |
| 461 | ||
| 462 | try handleChildProcUnsupported(s); | |
| 463 | try handleVerbose(s.owner, .inherit, argv); | |
| 464 | ||
| 465 | const zp = try gpa.create(ZigProcess); | |
| 466 | defer if (!watch) gpa.destroy(zp); | |
| 467 | ||
| 468 | zp.child = std.process.spawn(io, .{ | |
| 469 | .argv = argv, | |
| 470 | .environ_map = &b.graph.environ_map, | |
| 471 | .stdin = .pipe, | |
| 472 | .stdout = .pipe, | |
| 473 | .stderr = .pipe, | |
| 474 | .request_resource_usage_statistics = true, | |
| 475 | .progress_node = prog_node, | |
| 476 | }) catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err }); | |
| 477 | ||
| 478 | zp.multi_reader.init(gpa, io, zp.multi_reader_buffer.toStreams(), &.{ | |
| 479 | zp.child.stdout.?, zp.child.stderr.?, | |
| 480 | }); | |
| 481 | if (watch) s.cast(Compile).?.zig_process = zp; | |
| 482 | defer if (!watch) zp.deinit(io); | |
| 483 | ||
| 484 | const result = result: { | |
| 485 | defer if (watch) zp.saveState(prog_node); | |
| 486 | break :result try zigProcessUpdate(s, zp, watch, web_server, gpa); | |
| 487 | }; | |
| 488 | ||
| 489 | if (!watch) { | |
| 490 | // Send EOF to stdin. | |
| 491 | zp.child.stdin.?.close(io); | |
| 492 | zp.child.stdin = null; | |
| 493 | ||
| 494 | const term = zp.child.wait(io) catch |err| { | |
| 495 | return s.fail("unable to wait for {s}: {t}", .{ argv[0], err }); | |
| 496 | }; | |
| 497 | s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; | |
| 498 | ||
| 499 | // Special handling for Compile step that is expecting compile errors. | |
| 500 | if (s.cast(Compile)) |compile| switch (term) { | |
| 501 | .exited => { | |
| 502 | // Note that the exit code may be 0 in this case due to the | |
| 503 | // compiler server protocol. | |
| 504 | if (compile.expect_errors != null) { | |
| 505 | return error.NeedCompileErrorCheck; | |
| 506 | } | |
| 507 | }, | |
| 508 | else => {}, | |
| 509 | }; | |
| 510 | ||
| 511 | try handleChildProcessTerm(s, term); | |
| 512 | } | |
| 513 | ||
| 514 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 515 | return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); | |
| 516 | } | |
| 517 | ||
| 518 | return result; | |
| 519 | } | |
| 520 | ||
| 521 | /// Wrapper around `Io.Dir.updateFile` that handles verbose and error output. | |
| 522 | pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { | |
| 523 | const b = s.owner; | |
| 524 | const io = b.graph.io; | |
| 525 | const src_path = src_lazy_path.getPath3(b, s); | |
| 526 | try handleVerbose(b, .inherit, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); | |
| 527 | return Io.Dir.updateFile(src_path.root_dir.handle, io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| | |
| 528 | return s.fail("unable to update file from '{f}' to '{s}': {t}", .{ src_path, dest_path, err }); | |
| 529 | } | |
| 530 | ||
| 531 | /// Wrapper around `Io.Dir.createDirPathStatus` that handles verbose and error output. | |
| 532 | pub fn installDir(s: *Step, dest_path: []const u8) !Io.Dir.CreatePathStatus { | |
| 533 | const b = s.owner; | |
| 534 | const io = b.graph.io; | |
| 535 | try handleVerbose(b, .inherit, &.{ "install", "-d", dest_path }); | |
| 536 | return Io.Dir.cwd().createDirPathStatus(io, dest_path, .default_dir) catch |err| | |
| 537 | return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err }); | |
| 538 | } | |
| 539 | ||
| 540 | fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build.WebServer, gpa: Allocator) !?Path { | |
| 541 | const b = s.owner; | |
| 542 | const arena = b.allocator; | |
| 543 | const io = b.graph.io; | |
| 544 | ||
| 545 | const start_ts = Io.Clock.awake.now(io); | |
| 546 | ||
| 547 | try sendMessage(io, zp.child.stdin.?, .update); | |
| 548 | if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); | |
| 549 | ||
| 550 | var result: ?Path = null; | |
| 551 | var eos_err: error{EndOfStream}!void = {}; | |
| 552 | ||
| 553 | const stdout = zp.multi_reader.fileReader(0); | |
| 554 | ||
| 555 | while (true) { | |
| 556 | const Header = std.zig.Server.Message.Header; | |
| 557 | const header = stdout.interface.takeStruct(Header, .little) catch |err| switch (err) { | |
| 558 | error.EndOfStream => break, | |
| 559 | error.ReadFailed => return stdout.err.?, | |
| 560 | }; | |
| 561 | const body = stdout.interface.take(header.bytes_len) catch |err| switch (err) { | |
| 562 | error.EndOfStream => |e| { | |
| 563 | // Better to report the crash with stderr below, but we set | |
| 564 | // this in case the child exits successfully while violating | |
| 565 | // this protocol. | |
| 566 | eos_err = e; | |
| 567 | break; | |
| 568 | }, | |
| 569 | error.ReadFailed => return stdout.err.?, | |
| 570 | }; | |
| 571 | switch (header.tag) { | |
| 572 | .zig_version => { | |
| 573 | if (!std.mem.eql(u8, builtin.zig_version_string, body)) { | |
| 574 | return s.fail( | |
| 575 | "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", | |
| 576 | .{ builtin.zig_version_string, body }, | |
| 577 | ); | |
| 578 | } | |
| 579 | }, | |
| 580 | .error_bundle => { | |
| 581 | s.result_error_bundle = try std.zig.Server.allocErrorBundle(gpa, body); | |
| 582 | // This message indicates the end of the update. | |
| 583 | if (watch) break; | |
| 584 | }, | |
| 585 | .emit_digest => { | |
| 586 | const EmitDigest = std.zig.Server.Message.EmitDigest; | |
| 587 | const emit_digest: *align(1) const EmitDigest = @ptrCast(body); | |
| 588 | s.result_cached = emit_digest.flags.cache_hit; | |
| 589 | const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; | |
| 590 | result = .{ | |
| 591 | .root_dir = b.cache_root, | |
| 592 | .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), | |
| 593 | }; | |
| 594 | }, | |
| 595 | .file_system_inputs => { | |
| 596 | s.clearWatchInputs(); | |
| 597 | var it = std.mem.splitScalar(u8, body, 0); | |
| 598 | while (it.next()) |prefixed_path| { | |
| 599 | const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); | |
| 600 | const sub_path = try arena.dupe(u8, prefixed_path[1..]); | |
| 601 | const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; | |
| 602 | switch (prefix_index) { | |
| 603 | .cwd => { | |
| 604 | const path: Build.Cache.Path = .{ | |
| 605 | .root_dir = Build.Cache.Directory.cwd(), | |
| 606 | .sub_path = sub_path_dirname, | |
| 607 | }; | |
| 608 | try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); | |
| 609 | }, | |
| 610 | .zig_lib => zl: { | |
| 611 | if (s.cast(Step.Compile)) |compile| { | |
| 612 | if (compile.zig_lib_dir) |zig_lib_dir| { | |
| 613 | const lp = try zig_lib_dir.join(arena, sub_path); | |
| 614 | try addWatchInput(s, lp); | |
| 615 | break :zl; | |
| 616 | } | |
| 617 | } | |
| 618 | const path: Build.Cache.Path = .{ | |
| 619 | .root_dir = s.owner.graph.zig_lib_directory, | |
| 620 | .sub_path = sub_path_dirname, | |
| 621 | }; | |
| 622 | try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); | |
| 623 | }, | |
| 624 | .local_cache => { | |
| 625 | const path: Build.Cache.Path = .{ | |
| 626 | .root_dir = b.cache_root, | |
| 627 | .sub_path = sub_path_dirname, | |
| 628 | }; | |
| 629 | try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); | |
| 630 | }, | |
| 631 | .global_cache => { | |
| 632 | const path: Build.Cache.Path = .{ | |
| 633 | .root_dir = s.owner.graph.global_cache_root, | |
| 634 | .sub_path = sub_path_dirname, | |
| 635 | }; | |
| 636 | try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); | |
| 637 | }, | |
| 638 | } | |
| 639 | } | |
| 640 | }, | |
| 641 | .time_report => if (web_server) |ws| { | |
| 642 | const TimeReport = std.zig.Server.Message.TimeReport; | |
| 643 | const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); | |
| 644 | ws.updateTimeReportCompile(.{ | |
| 645 | .compile = s.cast(Step.Compile).?, | |
| 646 | .use_llvm = tr.flags.use_llvm, | |
| 647 | .stats = tr.stats, | |
| 648 | .ns_total = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()), | |
| 649 | .llvm_pass_timings_len = tr.llvm_pass_timings_len, | |
| 650 | .files_len = tr.files_len, | |
| 651 | .decls_len = tr.decls_len, | |
| 652 | .trailing = body[@sizeOf(TimeReport)..], | |
| 653 | }); | |
| 654 | }, | |
| 655 | else => {}, // ignore other messages | |
| 656 | } | |
| 657 | } | |
| 658 | ||
| 659 | s.result_duration_ns = @intCast(start_ts.untilNow(io, .awake).toNanoseconds()); | |
| 660 | ||
| 661 | const stderr_contents = zp.multi_reader.reader(1).buffered(); | |
| 662 | if (stderr_contents.len > 0) { | |
| 663 | try s.result_error_msgs.append(arena, try arena.dupe(u8, stderr_contents)); | |
| 664 | } | |
| 665 | ||
| 666 | try eos_err; | |
| 667 | ||
| 668 | return result; | |
| 669 | } | |
| 670 | ||
| 671 | pub fn getZigProcess(s: *Step) ?*ZigProcess { | |
| 672 | return switch (s.id) { | |
| 673 | .compile => s.cast(Compile).?.zig_process, | |
| 674 | else => null, | |
| 675 | }; | |
| 676 | } | |
| 677 | ||
| 678 | fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { | |
| 679 | const header: std.zig.Client.Message.Header = .{ | |
| 680 | .tag = tag, | |
| 681 | .bytes_len = 0, | |
| 682 | }; | |
| 683 | var w = file.writer(io, &.{}); | |
| 684 | w.interface.writeStruct(header, .little) catch |err| switch (err) { | |
| 685 | error.WriteFailed => return w.err.?, | |
| 686 | }; | |
| 687 | } | |
| 688 | ||
| 689 | pub fn handleVerbose( | |
| 690 | b: *Build, | |
| 691 | cwd: std.process.Child.Cwd, | |
| 692 | argv: []const []const u8, | |
| 693 | ) error{OutOfMemory}!void { | |
| 694 | return handleVerbose2(b, cwd, null, argv); | |
| 695 | } | |
| 696 | ||
| 697 | pub fn handleVerbose2( | |
| 698 | b: *Build, | |
| 699 | cwd: std.process.Child.Cwd, | |
| 700 | opt_env: ?*const std.process.Environ.Map, | |
| 701 | argv: []const []const u8, | |
| 702 | ) error{OutOfMemory}!void { | |
| 703 | if (b.verbose) { | |
| 704 | const graph = b.graph; | |
| 705 | // Intention of verbose is to print all sub-process command lines to | |
| 706 | // stderr before spawning them. | |
| 707 | const text = try allocPrintCmd(b.allocator, cwd, if (opt_env) |env| .{ | |
| 708 | .child = env, | |
| 709 | .parent = &graph.environ_map, | |
| 710 | } else null, argv); | |
| 711 | std.debug.print("{s}\n", .{text}); | |
| 712 | } | |
| 713 | } | |
| 714 | ||
| 715 | /// Asserts that the caller has already populated `s.result_failed_command`. | |
| 716 | pub inline fn handleChildProcUnsupported(s: *Step) error{ OutOfMemory, MakeFailed }!void { | |
| 717 | if (!std.process.can_spawn) { | |
| 718 | return s.fail("unable to spawn process: host cannot spawn child processes", .{}); | |
| 719 | } | |
| 720 | } | |
| 721 | ||
| 722 | /// Asserts that the caller has already populated `s.result_failed_command`. | |
| 723 | pub fn handleChildProcessTerm(s: *Step, term: std.process.Child.Term) error{ MakeFailed, OutOfMemory }!void { | |
| 724 | assert(s.result_failed_command != null); | |
| 725 | return switch (term) { | |
| 726 | .exited => |code| if (code != 0) s.fail("process exited with error code {d}", .{code}), | |
| 727 | .signal => |sig| s.fail("process terminated with signal {t}", .{sig}), | |
| 728 | .stopped => |sig| s.fail("process stopped with signal {t}", .{sig}), | |
| 729 | .unknown => s.fail("process terminated unexpectedly", .{}), | |
| 730 | }; | |
| 731 | } | |
| 732 | ||
| 733 | pub fn allocPrintCmd( | |
| 734 | gpa: Allocator, | |
| 735 | cwd: std.process.Child.Cwd, | |
| 736 | opt_env: ?struct { | |
| 737 | child: *const std.process.Environ.Map, | |
| 738 | parent: *const std.process.Environ.Map, | |
| 739 | }, | |
| 740 | argv: []const []const u8, | |
| 741 | ) Allocator.Error![]u8 { | |
| 742 | const shell = struct { | |
| 743 | fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { | |
| 744 | for (string) |c| { | |
| 745 | if (switch (c) { | |
| 746 | else => true, | |
| 747 | '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, | |
| 748 | '=' => is_argv0, | |
| 749 | }) break; | |
| 750 | } else return writer.writeAll(string); | |
| 751 | ||
| 752 | try writer.writeByte('"'); | |
| 753 | for (string) |c| { | |
| 754 | if (switch (c) { | |
| 755 | std.ascii.control_code.nul => break, | |
| 756 | '!', '"', '$', '\\', '`' => true, | |
| 757 | else => !std.ascii.isPrint(c), | |
| 758 | }) try writer.writeByte('\\'); | |
| 759 | switch (c) { | |
| 760 | std.ascii.control_code.nul => unreachable, | |
| 761 | std.ascii.control_code.bel => try writer.writeByte('a'), | |
| 762 | std.ascii.control_code.bs => try writer.writeByte('b'), | |
| 763 | std.ascii.control_code.ht => try writer.writeByte('t'), | |
| 764 | std.ascii.control_code.lf => try writer.writeByte('n'), | |
| 765 | std.ascii.control_code.vt => try writer.writeByte('v'), | |
| 766 | std.ascii.control_code.ff => try writer.writeByte('f'), | |
| 767 | std.ascii.control_code.cr => try writer.writeByte('r'), | |
| 768 | std.ascii.control_code.esc => try writer.writeByte('E'), | |
| 769 | ' '...'~' => try writer.writeByte(c), | |
| 770 | else => try writer.print("{o:0>3}", .{c}), | |
| 771 | } | |
| 772 | } | |
| 773 | try writer.writeByte('"'); | |
| 774 | } | |
| 775 | }; | |
| 776 | ||
| 777 | var aw: Io.Writer.Allocating = .init(gpa); | |
| 778 | defer aw.deinit(); | |
| 779 | const writer = &aw.writer; | |
| 780 | switch (cwd) { | |
| 781 | .inherit => {}, | |
| 782 | .path => |path| writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory, | |
| 783 | .dir => @panic("TODO"), | |
| 784 | } | |
| 785 | if (opt_env) |env| { | |
| 786 | var it = env.child.iterator(); | |
| 787 | while (it.next()) |entry| { | |
| 788 | const key = entry.key_ptr.*; | |
| 789 | const value = entry.value_ptr.*; | |
| 790 | if (env.parent.get(key)) |process_value| { | |
| 791 | if (std.mem.eql(u8, value, process_value)) continue; | |
| 792 | } | |
| 793 | writer.print("{s}=", .{key}) catch return error.OutOfMemory; | |
| 794 | shell.escape(writer, value, false) catch return error.OutOfMemory; | |
| 795 | writer.writeByte(' ') catch return error.OutOfMemory; | |
| 796 | } | |
| 797 | } | |
| 798 | shell.escape(writer, argv[0], true) catch return error.OutOfMemory; | |
| 799 | for (argv[1..]) |arg| { | |
| 800 | writer.writeByte(' ') catch return error.OutOfMemory; | |
| 801 | shell.escape(writer, arg, false) catch return error.OutOfMemory; | |
| 802 | } | |
| 803 | return aw.toOwnedSlice(); | |
| 804 | } | |
| 805 | ||
| 806 | /// Prefer `cacheHitAndWatch` unless you already added watch inputs | |
| 807 | /// separately from using the cache system. | |
| 808 | pub fn cacheHit(s: *Step, man: *Build.Cache.Manifest) !bool { | |
| 809 | s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); | |
| 810 | return s.result_cached; | |
| 811 | } | |
| 812 | ||
| 813 | /// Clears previous watch inputs, if any, and then populates watch inputs from | |
| 814 | /// the full set of files picked up by the cache manifest. | |
| 815 | /// | |
| 816 | /// Must be accompanied with `writeManifestAndWatch`. | |
| 817 | pub fn cacheHitAndWatch(s: *Step, man: *Build.Cache.Manifest) !bool { | |
| 818 | const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); | |
| 819 | s.result_cached = is_hit; | |
| 820 | // The above call to hit() populates the manifest with files, so in case of | |
| 821 | // a hit, we need to populate watch inputs. | |
| 822 | if (is_hit) try setWatchInputsFromManifest(s, man); | |
| 823 | return is_hit; | |
| 824 | } | |
| 825 | ||
| 826 | fn failWithCacheError( | |
| 827 | s: *Step, | |
| 828 | man: *const Build.Cache.Manifest, | |
| 829 | err: Build.Cache.Manifest.HitError, | |
| 830 | ) error{ OutOfMemory, Canceled, MakeFailed } { | |
| 831 | switch (err) { | |
| 832 | error.CacheCheckFailed => switch (man.diagnostic) { | |
| 833 | .none => unreachable, | |
| 834 | .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{ | |
| 835 | man.diagnostic, e, | |
| 836 | }), | |
| 837 | .file_open, .file_stat, .file_read, .file_hash => |op| { | |
| 838 | const pp = man.files.keys()[op.file_index].prefixed_path; | |
| 839 | const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; | |
| 840 | return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{ | |
| 841 | prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err, | |
| 842 | }); | |
| 843 | }, | |
| 844 | }, | |
| 845 | error.OutOfMemory, error.Canceled => |e| return e, | |
| 846 | error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}), | |
| 847 | } | |
| 848 | } | |
| 849 | ||
| 850 | /// Prefer `writeManifestAndWatch` unless you already added watch inputs | |
| 851 | /// separately from using the cache system. | |
| 852 | pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void { | |
| 853 | if (s.test_results.isSuccess()) { | |
| 854 | man.writeManifest() catch |err| { | |
| 855 | try s.addError("unable to write cache manifest: {t}", .{err}); | |
| 856 | }; | |
| 857 | } | |
| 858 | } | |
| 859 | ||
| 860 | /// Clears previous watch inputs, if any, and then populates watch inputs from | |
| 861 | /// the full set of files picked up by the cache manifest. | |
| 862 | /// | |
| 863 | /// Must be accompanied with `cacheHitAndWatch`. | |
| 864 | pub fn writeManifestAndWatch(s: *Step, man: *Build.Cache.Manifest) !void { | |
| 865 | try writeManifest(s, man); | |
| 866 | try setWatchInputsFromManifest(s, man); | |
| 867 | } | |
| 868 | ||
| 869 | fn setWatchInputsFromManifest(s: *Step, man: *Build.Cache.Manifest) !void { | |
| 870 | const arena = s.owner.allocator; | |
| 871 | const prefixes = man.cache.prefixes(); | |
| 872 | clearWatchInputs(s); | |
| 873 | for (man.files.keys()) |file| { | |
| 874 | // The file path data is freed when the cache manifest is cleaned up at the end of `make`. | |
| 875 | const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); | |
| 876 | try addWatchInputFromPath(s, .{ | |
| 877 | .root_dir = prefixes[file.prefixed_path.prefix], | |
| 878 | .sub_path = std.fs.path.dirname(sub_path) orelse "", | |
| 879 | }, std.fs.path.basename(sub_path)); | |
| 880 | } | |
| 881 | } | |
| 882 | ||
| 883 | /// For steps that have a single input that never changes when re-running `make`. | |
| 884 | pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { | |
| 885 | if (!step.inputs.populated()) try step.addWatchInput(lazy_path); | |
| 886 | } | |
| 887 | ||
| 888 | pub fn clearWatchInputs(step: *Step) void { | |
| 889 | const gpa = step.owner.allocator; | |
| 890 | step.inputs.clear(gpa); | |
| 891 | } | |
| 892 | ||
| 893 | /// Places a *file* dependency on the path. | |
| 894 | pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!void { | |
| 895 | switch (lazy_file) { | |
| 896 | .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), | |
| 897 | .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), | |
| 898 | .cwd_relative => |path_string| { | |
| 899 | try addWatchInputFromPath(step, .{ | |
| 900 | .root_dir = .{ | |
| 901 | .path = null, | |
| 902 | .handle = Io.Dir.cwd(), | |
| 903 | }, | |
| 904 | .sub_path = std.fs.path.dirname(path_string) orelse "", | |
| 905 | }, std.fs.path.basename(path_string)); | |
| 906 | }, | |
| 907 | // Nothing to watch because this dependency edge is modeled instead via `dependants`. | |
| 908 | .generated => {}, | |
| 909 | } | |
| 910 | } | |
| 911 | ||
| 912 | /// Any changes inside the directory will trigger invalidation. | |
| 913 | /// | |
| 914 | /// See also `addDirectoryWatchInputFromPath` which takes a `Build.Cache.Path` instead. | |
| 915 | /// | |
| 916 | /// Paths derived from this directory should also be manually added via | |
| 917 | /// `addDirectoryWatchInputFromPath` if and only if this function returns | |
| 918 | /// `true`. | |
| 919 | pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool { | |
| 920 | switch (lazy_directory) { | |
| 921 | .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), | |
| 922 | .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), | |
| 923 | .cwd_relative => |path_string| { | |
| 924 | try addDirectoryWatchInputFromPath(step, .{ | |
| 925 | .root_dir = .{ | |
| 926 | .path = null, | |
| 927 | .handle = Io.Dir.cwd(), | |
| 928 | }, | |
| 929 | .sub_path = path_string, | |
| 930 | }); | |
| 931 | }, | |
| 932 | // Nothing to watch because this dependency edge is modeled instead via `dependants`. | |
| 933 | .generated => return false, | |
| 934 | } | |
| 935 | return true; | |
| 936 | } | |
| 937 | ||
| 938 | /// Any changes inside the directory will trigger invalidation. | |
| 939 | /// | |
| 940 | /// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. | |
| 941 | /// | |
| 942 | /// This function should only be called when it has been verified that the | |
| 943 | /// dependency on `path` is not already accounted for by a `Step` dependency. | |
| 944 | /// In other words, before calling this function, first check that the | |
| 945 | /// `Build.LazyPath` which this `path` is derived from is not `generated`. | |
| 946 | pub fn addDirectoryWatchInputFromPath(step: *Step, path: Build.Cache.Path) !void { | |
| 947 | return addWatchInputFromPath(step, path, "."); | |
| 948 | } | |
| 949 | ||
| 950 | fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { | |
| 951 | return addWatchInputFromPath(step, .{ | |
| 952 | .root_dir = builder.build_root, | |
| 953 | .sub_path = std.fs.path.dirname(sub_path) orelse "", | |
| 954 | }, std.fs.path.basename(sub_path)); | |
| 955 | } | |
| 956 | ||
| 957 | fn addDirectoryWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { | |
| 958 | return addDirectoryWatchInputFromPath(step, .{ | |
| 959 | .root_dir = builder.build_root, | |
| 960 | .sub_path = sub_path, | |
| 961 | }); | |
| 962 | } | |
| 963 | ||
| 964 | fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const u8) !void { | |
| 965 | const gpa = step.owner.allocator; | |
| 966 | const gop = try step.inputs.table.getOrPut(gpa, path); | |
| 967 | if (!gop.found_existing) gop.value_ptr.* = .empty; | |
| 968 | try gop.value_ptr.append(gpa, basename); | |
| 969 | } | |
| 970 | ||
| 971 | /// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. | |
| 972 | pub fn reset(step: *Step, gpa: Allocator) void { | |
| 973 | assert(step.state == .precheck_done); | |
| 974 | ||
| 975 | if (step.result_failed_command) |cmd| gpa.free(cmd); | |
| 976 | ||
| 977 | step.result_error_msgs.clearRetainingCapacity(); | |
| 978 | step.result_stderr = ""; | |
| 979 | step.result_cached = false; | |
| 980 | step.result_duration_ns = null; | |
| 981 | step.result_peak_rss = 0; | |
| 982 | step.result_failed_command = null; | |
| 983 | step.test_results = .{}; | |
| 984 | step.clearWatchInputs(); | |
| 985 | ||
| 986 | step.result_error_bundle.deinit(gpa); | |
| 987 | step.result_error_bundle = std.zig.ErrorBundle.empty; | |
| 988 | } | |
| 989 | ||
| 990 | /// Implementation detail of file watching. Prepares the step for being re-evaluated. | |
| 991 | /// Returns `true` if the step was newly invalidated, `false` if it was already invalidated. | |
| 992 | pub fn invalidateResult(step: *Step, gpa: Allocator) bool { | |
| 993 | if (step.state == .precheck_done) return false; | |
| 994 | assert(step.pending_deps == 0); | |
| 995 | step.state = .precheck_done; | |
| 996 | step.reset(gpa); | |
| 997 | for (step.dependants.items) |dependant| { | |
| 998 | _ = dependant.invalidateResult(gpa); | |
| 999 | dependant.pending_deps += 1; | |
| 1000 | } | |
| 1001 | return true; | |
| 1002 | } | |
| 1003 | ||
| 1004 | 155 | test { |
| 1005 | 156 | _ = CheckFile; |
| 1006 | 157 | _ = Fail; |
lib/std/Build/Step/CheckFile.zig+2-2
| ... | ... | @@ -16,7 +16,7 @@ expected_exact: ?[]const u8, |
| 16 | 16 | source: std.Build.LazyPath, |
| 17 | 17 | max_bytes: usize = 20 * 1024 * 1024, |
| 18 | 18 | |
| 19 | pub const base_id: Step.Id = .check_file; | |
| 19 | pub const base_tag: Step.Tag = .check_file; | |
| 20 | 20 | |
| 21 | 21 | pub const Options = struct { |
| 22 | 22 | expected_matches: []const []const u8 = &.{}, |
| ... | ... | @@ -31,7 +31,7 @@ pub fn create( |
| 31 | 31 | const check_file = owner.allocator.create(CheckFile) catch @panic("OOM"); |
| 32 | 32 | check_file.* = .{ |
| 33 | 33 | .step = Step.init(.{ |
| 34 | .id = base_id, | |
| 34 | .tag = base_tag, | |
| 35 | 35 | .name = "CheckFile", |
| 36 | 36 | .owner = owner, |
| 37 | 37 | .makeFn = make, |
lib/std/Build/Step/Compile.zig+2-1076
| ... | ... | @@ -20,7 +20,7 @@ const InstallDir = std.Build.InstallDir; |
| 20 | 20 | const GeneratedFile = std.Build.GeneratedFile; |
| 21 | 21 | const Path = std.Build.Cache.Path; |
| 22 | 22 | |
| 23 | pub const base_id: Step.Id = .compile; | |
| 23 | pub const base_tag: Step.Tag = .compile; | |
| 24 | 24 | |
| 25 | 25 | step: Step, |
| 26 | 26 | root_module: *Module, |
| ... | ... | @@ -235,10 +235,6 @@ is_linking_libc: bool = false, |
| 235 | 235 | /// Computed during make(). |
| 236 | 236 | is_linking_libcpp: bool = false, |
| 237 | 237 | |
| 238 | /// Populated during the make phase when there is a long-lived compiler process. | |
| 239 | /// Managed by the build runner, not user build script. | |
| 240 | zig_process: ?*Step.ZigProcess, | |
| 241 | ||
| 242 | 238 | /// Enables coverage instrumentation that is only useful if you are using third |
| 243 | 239 | /// party fuzzers that depend on it. Otherwise, slows down the instrumented |
| 244 | 240 | /// binary with unnecessary function calls. |
| ... | ... | @@ -418,10 +414,9 @@ pub fn create(owner: *std.Build, options: Options) *Compile { |
| 418 | 414 | .kind = options.kind, |
| 419 | 415 | .name = name, |
| 420 | 416 | .step = .init(.{ |
| 421 | .id = base_id, | |
| 417 | .tag = base_tag, | |
| 422 | 418 | .name = step_name, |
| 423 | 419 | .owner = owner, |
| 424 | .makeFn = make, | |
| 425 | 420 | .max_rss = options.max_rss, |
| 426 | 421 | }), |
| 427 | 422 | .version = options.version, |
| ... | ... | @@ -452,8 +447,6 @@ pub fn create(owner: *std.Build, options: Options) *Compile { |
| 452 | 447 | .use_llvm = options.use_llvm, |
| 453 | 448 | .use_lld = options.use_lld, |
| 454 | 449 | .use_new_linker = null, |
| 455 | ||
| 456 | .zig_process = null, | |
| 457 | 450 | }; |
| 458 | 451 | |
| 459 | 452 | if (options.zig_lib_dir) |lp| { |
| ... | ... | @@ -701,122 +694,6 @@ pub fn producesImplib(compile: *Compile) bool { |
| 701 | 694 | return compile.isDll(); |
| 702 | 695 | } |
| 703 | 696 | |
| 704 | const PkgConfigResult = struct { | |
| 705 | cflags: []const []const u8, | |
| 706 | libs: []const []const u8, | |
| 707 | }; | |
| 708 | ||
| 709 | /// Run pkg-config for the given library name and parse the output, returning the arguments | |
| 710 | /// that should be passed to zig to link the given library. | |
| 711 | pub fn runPkgConfig(step: *Step, lib_name: []const u8) !PkgConfigResult { | |
| 712 | const wl_rpath_prefix = "-Wl,-rpath,"; | |
| 713 | ||
| 714 | const b = step.owner; | |
| 715 | const pkg_name = match: { | |
| 716 | // First we have to map the library name to pkg config name. Unfortunately, | |
| 717 | // there are several examples where this is not straightforward: | |
| 718 | // -lSDL2 -> pkg-config sdl2 | |
| 719 | // -lgdk-3 -> pkg-config gdk-3.0 | |
| 720 | // -latk-1.0 -> pkg-config atk | |
| 721 | // -lpulse -> pkg-config libpulse | |
| 722 | const pkgs = try getPkgConfigList(b); | |
| 723 | ||
| 724 | // Exact match means instant winner. | |
| 725 | for (pkgs) |pkg| { | |
| 726 | if (mem.eql(u8, pkg.name, lib_name)) { | |
| 727 | break :match pkg.name; | |
| 728 | } | |
| 729 | } | |
| 730 | ||
| 731 | // Next we'll try ignoring case. | |
| 732 | for (pkgs) |pkg| { | |
| 733 | if (std.ascii.eqlIgnoreCase(pkg.name, lib_name)) { | |
| 734 | break :match pkg.name; | |
| 735 | } | |
| 736 | } | |
| 737 | ||
| 738 | // Prefixed "lib" or suffixed ".0". | |
| 739 | for (pkgs) |pkg| { | |
| 740 | if (std.ascii.findIgnoreCase(pkg.name, lib_name)) |pos| { | |
| 741 | const prefix = pkg.name[0..pos]; | |
| 742 | const suffix = pkg.name[pos + lib_name.len ..]; | |
| 743 | if (prefix.len > 0 and !mem.eql(u8, prefix, "lib")) continue; | |
| 744 | if (suffix.len > 0 and !mem.eql(u8, suffix, ".0")) continue; | |
| 745 | break :match pkg.name; | |
| 746 | } | |
| 747 | } | |
| 748 | ||
| 749 | // Trimming "-1.0". | |
| 750 | if (mem.endsWith(u8, lib_name, "-1.0")) { | |
| 751 | const trimmed_lib_name = lib_name[0 .. lib_name.len - "-1.0".len]; | |
| 752 | for (pkgs) |pkg| { | |
| 753 | if (std.ascii.eqlIgnoreCase(pkg.name, trimmed_lib_name)) { | |
| 754 | break :match pkg.name; | |
| 755 | } | |
| 756 | } | |
| 757 | } | |
| 758 | ||
| 759 | return error.PackageNotFound; | |
| 760 | }; | |
| 761 | ||
| 762 | var code: u8 = undefined; | |
| 763 | const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; | |
| 764 | const stdout = if (b.runAllowFail(&[_][]const u8{ | |
| 765 | pkg_config_exe, | |
| 766 | pkg_name, | |
| 767 | "--cflags", | |
| 768 | "--libs", | |
| 769 | }, &code, .ignore)) |stdout| stdout else |err| switch (err) { | |
| 770 | error.ProcessTerminated => return error.PkgConfigCrashed, | |
| 771 | error.ExecNotSupported => return error.PkgConfigFailed, | |
| 772 | error.ExitCodeFailure => return error.PkgConfigFailed, | |
| 773 | error.FileNotFound => return error.PkgConfigNotInstalled, | |
| 774 | else => return err, | |
| 775 | }; | |
| 776 | ||
| 777 | var zig_cflags: std.ArrayList([]const u8) = .empty; | |
| 778 | defer zig_cflags.deinit(b.allocator); | |
| 779 | var zig_libs: std.ArrayList([]const u8) = .empty; | |
| 780 | defer zig_libs.deinit(b.allocator); | |
| 781 | ||
| 782 | var arg_it = mem.tokenizeAny(u8, stdout, " \r\n\t"); | |
| 783 | while (arg_it.next()) |arg| { | |
| 784 | if (mem.eql(u8, arg, "-I")) { | |
| 785 | const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; | |
| 786 | try zig_cflags.appendSlice(b.allocator, &.{ "-I", dir }); | |
| 787 | } else if (mem.startsWith(u8, arg, "-I")) { | |
| 788 | try zig_cflags.append(b.allocator, arg); | |
| 789 | } else if (mem.eql(u8, arg, "-L")) { | |
| 790 | const dir = arg_it.next() orelse return error.PkgConfigInvalidOutput; | |
| 791 | try zig_libs.appendSlice(b.allocator, &.{ "-L", dir }); | |
| 792 | } else if (mem.startsWith(u8, arg, "-L")) { | |
| 793 | try zig_libs.append(b.allocator, arg); | |
| 794 | } else if (mem.eql(u8, arg, "-l")) { | |
| 795 | const lib = arg_it.next() orelse return error.PkgConfigInvalidOutput; | |
| 796 | try zig_libs.appendSlice(b.allocator, &.{ "-l", lib }); | |
| 797 | } else if (mem.startsWith(u8, arg, "-l")) { | |
| 798 | try zig_libs.append(b.allocator, arg); | |
| 799 | } else if (mem.eql(u8, arg, "-D")) { | |
| 800 | const macro = arg_it.next() orelse return error.PkgConfigInvalidOutput; | |
| 801 | try zig_cflags.appendSlice(b.allocator, &.{ "-D", macro }); | |
| 802 | } else if (mem.startsWith(u8, arg, "-D")) { | |
| 803 | try zig_cflags.append(b.allocator, arg); | |
| 804 | } else if (mem.startsWith(u8, arg, wl_rpath_prefix)) { | |
| 805 | try zig_cflags.appendSlice(b.allocator, &.{ "-rpath", arg[wl_rpath_prefix.len..] }); | |
| 806 | } else if (b.debug_pkg_config) { | |
| 807 | return step.fail("unknown pkg-config flag '{s}'", .{arg}); | |
| 808 | } | |
| 809 | } | |
| 810 | ||
| 811 | try zig_cflags.shrinkToLen(b.allocator); | |
| 812 | try zig_libs.shrinkToLen(b.allocator); | |
| 813 | ||
| 814 | return .{ | |
| 815 | .cflags = zig_cflags.toOwnedSliceAssert(), | |
| 816 | .libs = zig_libs.toOwnedSliceAssert(), | |
| 817 | }; | |
| 818 | } | |
| 819 | ||
| 820 | 697 | pub fn setVerboseLink(compile: *Compile, value: bool) void { |
| 821 | 698 | compile.verbose_link = value; |
| 822 | 699 | } |
| ... | ... | @@ -974,863 +851,6 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking |
| 974 | 851 | return path; |
| 975 | 852 | } |
| 976 | 853 | |
| 977 | fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { | |
| 978 | const step = &compile.step; | |
| 979 | const b = step.owner; | |
| 980 | const arena = b.allocator; | |
| 981 | ||
| 982 | var zig_args = std.array_list.Managed([]const u8).init(arena); | |
| 983 | defer zig_args.deinit(); | |
| 984 | ||
| 985 | try zig_args.append(b.graph.zig_exe); | |
| 986 | ||
| 987 | const cmd = switch (compile.kind) { | |
| 988 | .lib => "build-lib", | |
| 989 | .exe => "build-exe", | |
| 990 | .obj => "build-obj", | |
| 991 | .@"test" => "test", | |
| 992 | .test_obj => "test-obj", | |
| 993 | }; | |
| 994 | try zig_args.append(cmd); | |
| 995 | ||
| 996 | if (b.reference_trace) |some| { | |
| 997 | try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some})); | |
| 998 | } | |
| 999 | try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts); | |
| 1000 | ||
| 1001 | try addFlag(&zig_args, "llvm", compile.use_llvm); | |
| 1002 | try addFlag(&zig_args, "lld", compile.use_lld); | |
| 1003 | try addFlag(&zig_args, "new-linker", compile.use_new_linker); | |
| 1004 | ||
| 1005 | if (compile.root_module.resolved_target.?.query.ofmt) |ofmt| { | |
| 1006 | try zig_args.append(try std.fmt.allocPrint(arena, "-ofmt={s}", .{@tagName(ofmt)})); | |
| 1007 | } | |
| 1008 | ||
| 1009 | switch (compile.entry) { | |
| 1010 | .default => {}, | |
| 1011 | .disabled => try zig_args.append("-fno-entry"), | |
| 1012 | .enabled => try zig_args.append("-fentry"), | |
| 1013 | .symbol_name => |entry_name| { | |
| 1014 | try zig_args.append(try std.fmt.allocPrint(arena, "-fentry={s}", .{entry_name})); | |
| 1015 | }, | |
| 1016 | } | |
| 1017 | ||
| 1018 | { | |
| 1019 | var symbol_it = compile.force_undefined_symbols.keyIterator(); | |
| 1020 | while (symbol_it.next()) |symbol_name| { | |
| 1021 | try zig_args.append("--force_undefined"); | |
| 1022 | try zig_args.append(symbol_name.*); | |
| 1023 | } | |
| 1024 | } | |
| 1025 | ||
| 1026 | if (compile.stack_size) |stack_size| { | |
| 1027 | try zig_args.append("--stack"); | |
| 1028 | try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); | |
| 1029 | } | |
| 1030 | ||
| 1031 | if (fuzz) { | |
| 1032 | try zig_args.append("-ffuzz"); | |
| 1033 | } | |
| 1034 | ||
| 1035 | { | |
| 1036 | // Stores system libraries that have already been seen for at least one | |
| 1037 | // module, along with any arguments that need to be passed to the | |
| 1038 | // compiler for each module individually. | |
| 1039 | var seen_system_libs: std.StringHashMapUnmanaged([]const []const u8) = .empty; | |
| 1040 | var frameworks: std.StringArrayHashMapUnmanaged(Module.LinkFrameworkOptions) = .empty; | |
| 1041 | ||
| 1042 | var prev_has_cflags = false; | |
| 1043 | var prev_has_rcflags = false; | |
| 1044 | var prev_search_strategy: Module.SystemLib.SearchStrategy = .paths_first; | |
| 1045 | var prev_preferred_link_mode: std.builtin.LinkMode = .dynamic; | |
| 1046 | // Track the number of positional arguments so that a nice error can be | |
| 1047 | // emitted if there is nothing to link. | |
| 1048 | var total_linker_objects: usize = @intFromBool(compile.root_module.root_source_file != null); | |
| 1049 | ||
| 1050 | // Fully recursive iteration including dynamic libraries to detect | |
| 1051 | // libc and libc++ linkage. | |
| 1052 | for (compile.getCompileDependencies(true)) |some_compile| { | |
| 1053 | for (some_compile.root_module.getGraph().modules) |mod| { | |
| 1054 | if (mod.link_libc == true) compile.is_linking_libc = true; | |
| 1055 | if (mod.link_libcpp == true) compile.is_linking_libcpp = true; | |
| 1056 | } | |
| 1057 | } | |
| 1058 | ||
| 1059 | var cli_named_modules = try CliNamedModules.init(arena, compile.root_module); | |
| 1060 | ||
| 1061 | // For this loop, don't chase dynamic libraries because their link | |
| 1062 | // objects are already linked. | |
| 1063 | for (compile.getCompileDependencies(false)) |dep_compile| { | |
| 1064 | for (dep_compile.root_module.getGraph().modules) |mod| { | |
| 1065 | // While walking transitive dependencies, if a given link object is | |
| 1066 | // already included in a library, it should not redundantly be | |
| 1067 | // placed on the linker line of the dependee. | |
| 1068 | const my_responsibility = dep_compile == compile; | |
| 1069 | const already_linked = !my_responsibility and dep_compile.isDynamicLibrary(); | |
| 1070 | ||
| 1071 | // Inherit dependencies on darwin frameworks. | |
| 1072 | if (!already_linked) { | |
| 1073 | for (mod.frameworks.keys(), mod.frameworks.values()) |name, info| { | |
| 1074 | try frameworks.put(arena, name, info); | |
| 1075 | } | |
| 1076 | } | |
| 1077 | ||
| 1078 | // Inherit dependencies on system libraries and static libraries. | |
| 1079 | for (mod.link_objects.items) |link_object| { | |
| 1080 | switch (link_object) { | |
| 1081 | .static_path => |static_path| { | |
| 1082 | if (my_responsibility) { | |
| 1083 | try zig_args.append(static_path.getPath2(mod.owner, step)); | |
| 1084 | total_linker_objects += 1; | |
| 1085 | } | |
| 1086 | }, | |
| 1087 | .system_lib => |system_lib| { | |
| 1088 | const system_lib_gop = try seen_system_libs.getOrPut(arena, system_lib.name); | |
| 1089 | if (system_lib_gop.found_existing) { | |
| 1090 | try zig_args.appendSlice(system_lib_gop.value_ptr.*); | |
| 1091 | continue; | |
| 1092 | } else { | |
| 1093 | system_lib_gop.value_ptr.* = &.{}; | |
| 1094 | } | |
| 1095 | ||
| 1096 | if (already_linked) | |
| 1097 | continue; | |
| 1098 | ||
| 1099 | if ((system_lib.search_strategy != prev_search_strategy or | |
| 1100 | system_lib.preferred_link_mode != prev_preferred_link_mode) and | |
| 1101 | compile.linkage != .static) | |
| 1102 | { | |
| 1103 | switch (system_lib.search_strategy) { | |
| 1104 | .no_fallback => switch (system_lib.preferred_link_mode) { | |
| 1105 | .dynamic => try zig_args.append("-search_dylibs_only"), | |
| 1106 | .static => try zig_args.append("-search_static_only"), | |
| 1107 | }, | |
| 1108 | .paths_first => switch (system_lib.preferred_link_mode) { | |
| 1109 | .dynamic => try zig_args.append("-search_paths_first"), | |
| 1110 | .static => try zig_args.append("-search_paths_first_static"), | |
| 1111 | }, | |
| 1112 | .mode_first => switch (system_lib.preferred_link_mode) { | |
| 1113 | .dynamic => try zig_args.append("-search_dylibs_first"), | |
| 1114 | .static => try zig_args.append("-search_static_first"), | |
| 1115 | }, | |
| 1116 | } | |
| 1117 | prev_search_strategy = system_lib.search_strategy; | |
| 1118 | prev_preferred_link_mode = system_lib.preferred_link_mode; | |
| 1119 | } | |
| 1120 | ||
| 1121 | const prefix: []const u8 = prefix: { | |
| 1122 | if (system_lib.needed) break :prefix "-needed-l"; | |
| 1123 | if (system_lib.weak) break :prefix "-weak-l"; | |
| 1124 | break :prefix "-l"; | |
| 1125 | }; | |
| 1126 | switch (system_lib.use_pkg_config) { | |
| 1127 | .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), | |
| 1128 | .yes, .force => { | |
| 1129 | if (runPkgConfig(&compile.step, system_lib.name)) |result| { | |
| 1130 | try zig_args.appendSlice(result.cflags); | |
| 1131 | try zig_args.appendSlice(result.libs); | |
| 1132 | try seen_system_libs.put(arena, system_lib.name, result.cflags); | |
| 1133 | } else |err| switch (err) { | |
| 1134 | error.PkgConfigInvalidOutput, | |
| 1135 | error.PkgConfigCrashed, | |
| 1136 | error.PkgConfigFailed, | |
| 1137 | error.PkgConfigNotInstalled, | |
| 1138 | error.PackageNotFound, | |
| 1139 | => switch (system_lib.use_pkg_config) { | |
| 1140 | .yes => { | |
| 1141 | // pkg-config failed, so fall back to linking the library | |
| 1142 | // by name directly. | |
| 1143 | try zig_args.append(b.fmt("{s}{s}", .{ | |
| 1144 | prefix, | |
| 1145 | system_lib.name, | |
| 1146 | })); | |
| 1147 | }, | |
| 1148 | .force => { | |
| 1149 | panic("pkg-config failed for library {s}", .{system_lib.name}); | |
| 1150 | }, | |
| 1151 | .no => unreachable, | |
| 1152 | }, | |
| 1153 | ||
| 1154 | else => |e| return e, | |
| 1155 | } | |
| 1156 | }, | |
| 1157 | } | |
| 1158 | }, | |
| 1159 | .other_step => |other| { | |
| 1160 | switch (other.kind) { | |
| 1161 | .exe => return step.fail("cannot link with an executable build artifact", .{}), | |
| 1162 | .@"test" => return step.fail("cannot link with a test", .{}), | |
| 1163 | .obj, .test_obj => { | |
| 1164 | const included_in_lib_or_obj = !my_responsibility and | |
| 1165 | (dep_compile.kind == .lib or dep_compile.kind == .obj or dep_compile.kind == .test_obj); | |
| 1166 | if (!already_linked and !included_in_lib_or_obj) { | |
| 1167 | try zig_args.append(other.getEmittedBin().getPath2(b, step)); | |
| 1168 | total_linker_objects += 1; | |
| 1169 | } | |
| 1170 | }, | |
| 1171 | .lib => l: { | |
| 1172 | const other_produces_implib = other.producesImplib(); | |
| 1173 | const other_is_static = other_produces_implib or other.isStaticLibrary(); | |
| 1174 | ||
| 1175 | if (compile.isStaticLibrary() and other_is_static) { | |
| 1176 | // Avoid putting a static library inside a static library. | |
| 1177 | break :l; | |
| 1178 | } | |
| 1179 | ||
| 1180 | // For DLLs, we must link against the implib. | |
| 1181 | // For everything else, we directly link | |
| 1182 | // against the library file. | |
| 1183 | const full_path_lib = if (other_produces_implib) | |
| 1184 | try other.getGeneratedFilePath("generated_implib", &compile.step) | |
| 1185 | else | |
| 1186 | try other.getGeneratedFilePath("generated_bin", &compile.step); | |
| 1187 | ||
| 1188 | try zig_args.append(full_path_lib); | |
| 1189 | total_linker_objects += 1; | |
| 1190 | ||
| 1191 | if (other.linkage == .dynamic and | |
| 1192 | compile.rootModuleTarget().os.tag != .windows) | |
| 1193 | { | |
| 1194 | if (fs.path.dirname(full_path_lib)) |dirname| { | |
| 1195 | try zig_args.append("-rpath"); | |
| 1196 | try zig_args.append(dirname); | |
| 1197 | } | |
| 1198 | } | |
| 1199 | }, | |
| 1200 | } | |
| 1201 | }, | |
| 1202 | .assembly_file => |asm_file| l: { | |
| 1203 | if (!my_responsibility) break :l; | |
| 1204 | ||
| 1205 | if (prev_has_cflags) { | |
| 1206 | try zig_args.append("-cflags"); | |
| 1207 | try zig_args.append("--"); | |
| 1208 | prev_has_cflags = false; | |
| 1209 | } | |
| 1210 | try zig_args.append(asm_file.getPath2(mod.owner, step)); | |
| 1211 | total_linker_objects += 1; | |
| 1212 | }, | |
| 1213 | ||
| 1214 | .c_source_file => |c_source_file| l: { | |
| 1215 | if (!my_responsibility) break :l; | |
| 1216 | ||
| 1217 | if (prev_has_cflags or c_source_file.flags.len != 0) { | |
| 1218 | try zig_args.append("-cflags"); | |
| 1219 | for (c_source_file.flags) |arg| { | |
| 1220 | try zig_args.append(arg); | |
| 1221 | } | |
| 1222 | try zig_args.append("--"); | |
| 1223 | } | |
| 1224 | prev_has_cflags = (c_source_file.flags.len != 0); | |
| 1225 | ||
| 1226 | if (c_source_file.language) |lang| { | |
| 1227 | try zig_args.append("-x"); | |
| 1228 | try zig_args.append(lang.internalIdentifier()); | |
| 1229 | } | |
| 1230 | ||
| 1231 | try zig_args.append(c_source_file.file.getPath2(mod.owner, step)); | |
| 1232 | ||
| 1233 | if (c_source_file.language != null) { | |
| 1234 | try zig_args.append("-x"); | |
| 1235 | try zig_args.append("none"); | |
| 1236 | } | |
| 1237 | total_linker_objects += 1; | |
| 1238 | }, | |
| 1239 | ||
| 1240 | .c_source_files => |c_source_files| l: { | |
| 1241 | if (!my_responsibility) break :l; | |
| 1242 | ||
| 1243 | if (prev_has_cflags or c_source_files.flags.len != 0) { | |
| 1244 | try zig_args.append("-cflags"); | |
| 1245 | for (c_source_files.flags) |arg| { | |
| 1246 | try zig_args.append(arg); | |
| 1247 | } | |
| 1248 | try zig_args.append("--"); | |
| 1249 | } | |
| 1250 | prev_has_cflags = (c_source_files.flags.len != 0); | |
| 1251 | ||
| 1252 | if (c_source_files.language) |lang| { | |
| 1253 | try zig_args.append("-x"); | |
| 1254 | try zig_args.append(lang.internalIdentifier()); | |
| 1255 | } | |
| 1256 | ||
| 1257 | const root_path = c_source_files.root.getPath2(mod.owner, step); | |
| 1258 | for (c_source_files.files) |file| { | |
| 1259 | try zig_args.append(b.pathJoin(&.{ root_path, file })); | |
| 1260 | } | |
| 1261 | ||
| 1262 | if (c_source_files.language != null) { | |
| 1263 | try zig_args.append("-x"); | |
| 1264 | try zig_args.append("none"); | |
| 1265 | } | |
| 1266 | ||
| 1267 | total_linker_objects += c_source_files.files.len; | |
| 1268 | }, | |
| 1269 | ||
| 1270 | .win32_resource_file => |rc_source_file| l: { | |
| 1271 | if (!my_responsibility) break :l; | |
| 1272 | ||
| 1273 | if (rc_source_file.flags.len == 0 and rc_source_file.include_paths.len == 0) { | |
| 1274 | if (prev_has_rcflags) { | |
| 1275 | try zig_args.append("-rcflags"); | |
| 1276 | try zig_args.append("--"); | |
| 1277 | prev_has_rcflags = false; | |
| 1278 | } | |
| 1279 | } else { | |
| 1280 | try zig_args.append("-rcflags"); | |
| 1281 | for (rc_source_file.flags) |arg| { | |
| 1282 | try zig_args.append(arg); | |
| 1283 | } | |
| 1284 | for (rc_source_file.include_paths) |include_path| { | |
| 1285 | try zig_args.append("/I"); | |
| 1286 | try zig_args.append(include_path.getPath2(mod.owner, step)); | |
| 1287 | } | |
| 1288 | try zig_args.append("--"); | |
| 1289 | prev_has_rcflags = true; | |
| 1290 | } | |
| 1291 | try zig_args.append(rc_source_file.file.getPath2(mod.owner, step)); | |
| 1292 | total_linker_objects += 1; | |
| 1293 | }, | |
| 1294 | } | |
| 1295 | } | |
| 1296 | ||
| 1297 | // We need to emit the --mod argument here so that the above link objects | |
| 1298 | // have the correct parent module, but only if the module is part of | |
| 1299 | // this compilation. | |
| 1300 | if (!my_responsibility) continue; | |
| 1301 | if (cli_named_modules.modules.getIndex(mod)) |module_cli_index| { | |
| 1302 | const module_cli_name = cli_named_modules.names.keys()[module_cli_index]; | |
| 1303 | try mod.appendZigProcessFlags(&zig_args, step); | |
| 1304 | ||
| 1305 | // --dep arguments | |
| 1306 | try zig_args.ensureUnusedCapacity(mod.import_table.count() * 2); | |
| 1307 | for (mod.import_table.keys(), mod.import_table.values()) |name, import| { | |
| 1308 | const import_index = cli_named_modules.modules.getIndex(import).?; | |
| 1309 | const import_cli_name = cli_named_modules.names.keys()[import_index]; | |
| 1310 | zig_args.appendAssumeCapacity("--dep"); | |
| 1311 | if (std.mem.eql(u8, import_cli_name, name)) { | |
| 1312 | zig_args.appendAssumeCapacity(import_cli_name); | |
| 1313 | } else { | |
| 1314 | zig_args.appendAssumeCapacity(b.fmt("{s}={s}", .{ name, import_cli_name })); | |
| 1315 | } | |
| 1316 | } | |
| 1317 | ||
| 1318 | // When the CLI sees a -M argument, it determines whether it | |
| 1319 | // implies the existence of a Zig compilation unit based on | |
| 1320 | // whether there is a root source file. If there is no root | |
| 1321 | // source file, then this is not a zig compilation unit - it is | |
| 1322 | // perhaps a set of linker objects, or C source files instead. | |
| 1323 | // Linker objects are added to the CLI globally, while C source | |
| 1324 | // files must have a module parent. | |
| 1325 | if (mod.root_source_file) |lp| { | |
| 1326 | const src = lp.getPath2(mod.owner, step); | |
| 1327 | try zig_args.append(b.fmt("-M{s}={s}", .{ module_cli_name, src })); | |
| 1328 | } else if (moduleNeedsCliArg(mod)) { | |
| 1329 | try zig_args.append(b.fmt("-M{s}", .{module_cli_name})); | |
| 1330 | } | |
| 1331 | } | |
| 1332 | } | |
| 1333 | } | |
| 1334 | ||
| 1335 | if (total_linker_objects == 0) { | |
| 1336 | return step.fail("the linker needs one or more objects to link", .{}); | |
| 1337 | } | |
| 1338 | ||
| 1339 | for (frameworks.keys(), frameworks.values()) |name, info| { | |
| 1340 | if (info.needed) { | |
| 1341 | try zig_args.append("-needed_framework"); | |
| 1342 | } else if (info.weak) { | |
| 1343 | try zig_args.append("-weak_framework"); | |
| 1344 | } else { | |
| 1345 | try zig_args.append("-framework"); | |
| 1346 | } | |
| 1347 | try zig_args.append(name); | |
| 1348 | } | |
| 1349 | ||
| 1350 | if (compile.is_linking_libcpp) { | |
| 1351 | try zig_args.append("-lc++"); | |
| 1352 | } | |
| 1353 | ||
| 1354 | if (compile.is_linking_libc) { | |
| 1355 | try zig_args.append("-lc"); | |
| 1356 | } | |
| 1357 | } | |
| 1358 | ||
| 1359 | if (compile.win32_manifest) |manifest_file| { | |
| 1360 | try zig_args.append(manifest_file.getPath2(b, step)); | |
| 1361 | } | |
| 1362 | ||
| 1363 | if (compile.win32_module_definition) |module_file| { | |
| 1364 | try zig_args.append(module_file.getPath2(b, step)); | |
| 1365 | } | |
| 1366 | ||
| 1367 | if (compile.image_base) |image_base| { | |
| 1368 | try zig_args.append("--image-base"); | |
| 1369 | try zig_args.append(b.fmt("0x{x}", .{image_base})); | |
| 1370 | } | |
| 1371 | ||
| 1372 | for (compile.filters) |filter| { | |
| 1373 | try zig_args.append("--test-filter"); | |
| 1374 | try zig_args.append(filter); | |
| 1375 | } | |
| 1376 | ||
| 1377 | if (compile.test_runner) |test_runner| { | |
| 1378 | try zig_args.append("--test-runner"); | |
| 1379 | try zig_args.append(test_runner.path.getPath2(b, step)); | |
| 1380 | } | |
| 1381 | ||
| 1382 | for (b.debug_log_scopes) |log_scope| { | |
| 1383 | try zig_args.append("--debug-log"); | |
| 1384 | try zig_args.append(log_scope); | |
| 1385 | } | |
| 1386 | ||
| 1387 | if (b.debug_compile_errors) { | |
| 1388 | try zig_args.append("--debug-compile-errors"); | |
| 1389 | } | |
| 1390 | ||
| 1391 | if (b.debug_incremental) { | |
| 1392 | try zig_args.append("--debug-incremental"); | |
| 1393 | } | |
| 1394 | ||
| 1395 | if (b.verbose_air) try zig_args.append("--verbose-air"); | |
| 1396 | if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); | |
| 1397 | if (b.verbose_llvm_bc) |path| try zig_args.append(b.fmt("--verbose-llvm-bc={s}", .{path})); | |
| 1398 | if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); | |
| 1399 | if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); | |
| 1400 | if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); | |
| 1401 | if (b.graph.time_report) try zig_args.append("--time-report"); | |
| 1402 | ||
| 1403 | if (compile.generated_asm != null) try zig_args.append("-femit-asm"); | |
| 1404 | if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); | |
| 1405 | if (compile.generated_docs != null) try zig_args.append("-femit-docs"); | |
| 1406 | if (compile.generated_implib != null) try zig_args.append("-femit-implib"); | |
| 1407 | if (compile.generated_llvm_bc != null) try zig_args.append("-femit-llvm-bc"); | |
| 1408 | if (compile.generated_llvm_ir != null) try zig_args.append("-femit-llvm-ir"); | |
| 1409 | if (compile.generated_h != null) try zig_args.append("-femit-h"); | |
| 1410 | ||
| 1411 | try addFlag(&zig_args, "formatted-panics", compile.formatted_panics); | |
| 1412 | ||
| 1413 | switch (compile.compress_debug_sections) { | |
| 1414 | .none => {}, | |
| 1415 | .zlib => try zig_args.append("--compress-debug-sections=zlib"), | |
| 1416 | .zstd => try zig_args.append("--compress-debug-sections=zstd"), | |
| 1417 | } | |
| 1418 | ||
| 1419 | if (compile.link_eh_frame_hdr) { | |
| 1420 | try zig_args.append("--eh-frame-hdr"); | |
| 1421 | } | |
| 1422 | if (compile.link_emit_relocs) { | |
| 1423 | try zig_args.append("--emit-relocs"); | |
| 1424 | } | |
| 1425 | if (compile.link_function_sections) { | |
| 1426 | try zig_args.append("-ffunction-sections"); | |
| 1427 | } | |
| 1428 | if (compile.link_data_sections) { | |
| 1429 | try zig_args.append("-fdata-sections"); | |
| 1430 | } | |
| 1431 | if (compile.link_gc_sections) |x| { | |
| 1432 | try zig_args.append(if (x) "--gc-sections" else "--no-gc-sections"); | |
| 1433 | } | |
| 1434 | if (!compile.linker_dynamicbase) { | |
| 1435 | try zig_args.append("--no-dynamicbase"); | |
| 1436 | } | |
| 1437 | if (compile.linker_allow_shlib_undefined) |x| { | |
| 1438 | try zig_args.append(if (x) "-fallow-shlib-undefined" else "-fno-allow-shlib-undefined"); | |
| 1439 | } | |
| 1440 | if (compile.link_z_notext) { | |
| 1441 | try zig_args.append("-z"); | |
| 1442 | try zig_args.append("notext"); | |
| 1443 | } | |
| 1444 | if (!compile.link_z_relro) { | |
| 1445 | try zig_args.append("-z"); | |
| 1446 | try zig_args.append("norelro"); | |
| 1447 | } | |
| 1448 | if (compile.link_z_lazy) { | |
| 1449 | try zig_args.append("-z"); | |
| 1450 | try zig_args.append("lazy"); | |
| 1451 | } | |
| 1452 | if (compile.link_z_common_page_size) |size| { | |
| 1453 | try zig_args.append("-z"); | |
| 1454 | try zig_args.append(b.fmt("common-page-size={d}", .{size})); | |
| 1455 | } | |
| 1456 | if (compile.link_z_max_page_size) |size| { | |
| 1457 | try zig_args.append("-z"); | |
| 1458 | try zig_args.append(b.fmt("max-page-size={d}", .{size})); | |
| 1459 | } | |
| 1460 | if (compile.link_z_defs) { | |
| 1461 | try zig_args.append("-z"); | |
| 1462 | try zig_args.append("defs"); | |
| 1463 | } | |
| 1464 | ||
| 1465 | if (compile.libc_file) |libc_file| { | |
| 1466 | try zig_args.append("--libc"); | |
| 1467 | try zig_args.append(libc_file.getPath2(b, step)); | |
| 1468 | } else if (b.libc_file) |libc_file| { | |
| 1469 | try zig_args.append("--libc"); | |
| 1470 | try zig_args.append(libc_file); | |
| 1471 | } | |
| 1472 | ||
| 1473 | try zig_args.append("--cache-dir"); | |
| 1474 | try zig_args.append(b.cache_root.path orelse "."); | |
| 1475 | ||
| 1476 | try zig_args.append("--global-cache-dir"); | |
| 1477 | try zig_args.append(b.graph.global_cache_root.path orelse "."); | |
| 1478 | ||
| 1479 | if (b.graph.debug_compiler_runtime_libs) |mode| | |
| 1480 | try zig_args.append(b.fmt("--debug-rt={t}", .{mode})); | |
| 1481 | ||
| 1482 | try zig_args.append("--name"); | |
| 1483 | try zig_args.append(compile.name); | |
| 1484 | ||
| 1485 | if (compile.linkage) |some| switch (some) { | |
| 1486 | .dynamic => try zig_args.append("-dynamic"), | |
| 1487 | .static => try zig_args.append("-static"), | |
| 1488 | }; | |
| 1489 | if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic) { | |
| 1490 | if (compile.version) |version| { | |
| 1491 | try zig_args.append("--version"); | |
| 1492 | try zig_args.append(b.fmt("{f}", .{version})); | |
| 1493 | } | |
| 1494 | ||
| 1495 | if (compile.rootModuleTarget().os.tag.isDarwin()) { | |
| 1496 | const install_name = compile.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ | |
| 1497 | compile.rootModuleTarget().libPrefix(), | |
| 1498 | compile.name, | |
| 1499 | compile.rootModuleTarget().dynamicLibSuffix(), | |
| 1500 | }); | |
| 1501 | try zig_args.append("-install_name"); | |
| 1502 | try zig_args.append(install_name); | |
| 1503 | } | |
| 1504 | } | |
| 1505 | ||
| 1506 | if (compile.entitlements) |entitlements| { | |
| 1507 | try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); | |
| 1508 | } | |
| 1509 | if (compile.pagezero_size) |pagezero_size| { | |
| 1510 | const size = try std.fmt.allocPrint(arena, "{x}", .{pagezero_size}); | |
| 1511 | try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); | |
| 1512 | } | |
| 1513 | if (compile.headerpad_size) |headerpad_size| { | |
| 1514 | const size = try std.fmt.allocPrint(arena, "{x}", .{headerpad_size}); | |
| 1515 | try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); | |
| 1516 | } | |
| 1517 | if (compile.headerpad_max_install_names) { | |
| 1518 | try zig_args.append("-headerpad_max_install_names"); | |
| 1519 | } | |
| 1520 | if (compile.dead_strip_dylibs) { | |
| 1521 | try zig_args.append("-dead_strip_dylibs"); | |
| 1522 | } | |
| 1523 | if (compile.force_load_objc) { | |
| 1524 | try zig_args.append("-ObjC"); | |
| 1525 | } | |
| 1526 | if (compile.discard_local_symbols) { | |
| 1527 | try zig_args.append("--discard-all"); | |
| 1528 | } | |
| 1529 | ||
| 1530 | try addFlag(&zig_args, "compiler-rt", compile.bundle_compiler_rt); | |
| 1531 | try addFlag(&zig_args, "ubsan-rt", compile.bundle_ubsan_rt); | |
| 1532 | try addFlag(&zig_args, "dll-export-fns", compile.dll_export_fns); | |
| 1533 | if (compile.rdynamic) { | |
| 1534 | try zig_args.append("-rdynamic"); | |
| 1535 | } | |
| 1536 | if (compile.import_memory) { | |
| 1537 | try zig_args.append("--import-memory"); | |
| 1538 | } | |
| 1539 | if (compile.export_memory) { | |
| 1540 | try zig_args.append("--export-memory"); | |
| 1541 | } | |
| 1542 | if (compile.import_symbols) { | |
| 1543 | try zig_args.append("--import-symbols"); | |
| 1544 | } | |
| 1545 | if (compile.import_table) { | |
| 1546 | try zig_args.append("--import-table"); | |
| 1547 | } | |
| 1548 | if (compile.export_table) { | |
| 1549 | try zig_args.append("--export-table"); | |
| 1550 | } | |
| 1551 | if (compile.initial_memory) |initial_memory| { | |
| 1552 | try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); | |
| 1553 | } | |
| 1554 | if (compile.max_memory) |max_memory| { | |
| 1555 | try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); | |
| 1556 | } | |
| 1557 | if (compile.shared_memory) { | |
| 1558 | try zig_args.append("--shared-memory"); | |
| 1559 | } | |
| 1560 | if (compile.global_base) |global_base| { | |
| 1561 | try zig_args.append(b.fmt("--global-base={d}", .{global_base})); | |
| 1562 | } | |
| 1563 | ||
| 1564 | if (compile.wasi_exec_model) |model| { | |
| 1565 | try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); | |
| 1566 | } | |
| 1567 | if (compile.linker_script) |linker_script| { | |
| 1568 | try zig_args.append("--script"); | |
| 1569 | try zig_args.append(linker_script.getPath2(b, step)); | |
| 1570 | } | |
| 1571 | ||
| 1572 | if (compile.version_script) |version_script| { | |
| 1573 | try zig_args.append("--version-script"); | |
| 1574 | try zig_args.append(version_script.getPath2(b, step)); | |
| 1575 | } | |
| 1576 | if (compile.linker_allow_undefined_version) |x| { | |
| 1577 | try zig_args.append(if (x) "--undefined-version" else "--no-undefined-version"); | |
| 1578 | } | |
| 1579 | ||
| 1580 | if (compile.linker_enable_new_dtags) |enabled| { | |
| 1581 | try zig_args.append(if (enabled) "--enable-new-dtags" else "--disable-new-dtags"); | |
| 1582 | } | |
| 1583 | ||
| 1584 | if (compile.kind == .@"test") { | |
| 1585 | if (compile.exec_cmd_args) |exec_cmd_args| { | |
| 1586 | for (exec_cmd_args) |cmd_arg| { | |
| 1587 | if (cmd_arg) |arg| { | |
| 1588 | try zig_args.append("--test-cmd"); | |
| 1589 | try zig_args.append(arg); | |
| 1590 | } else { | |
| 1591 | try zig_args.append("--test-cmd-bin"); | |
| 1592 | } | |
| 1593 | } | |
| 1594 | } | |
| 1595 | } | |
| 1596 | ||
| 1597 | if (b.sysroot) |sysroot| { | |
| 1598 | try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); | |
| 1599 | } | |
| 1600 | ||
| 1601 | // -I and -L arguments that appear after the last --mod argument apply to all modules. | |
| 1602 | const cwd: Io.Dir = .cwd(); | |
| 1603 | const io = b.graph.io; | |
| 1604 | ||
| 1605 | for (b.search_prefixes.items) |search_prefix| { | |
| 1606 | var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { | |
| 1607 | return step.fail("unable to open prefix directory '{s}': {s}", .{ | |
| 1608 | search_prefix, @errorName(err), | |
| 1609 | }); | |
| 1610 | }; | |
| 1611 | defer prefix_dir.close(io); | |
| 1612 | ||
| 1613 | // Avoid passing -L and -I flags for nonexistent directories. | |
| 1614 | // This prevents a warning, that should probably be upgraded to an error in Zig's | |
| 1615 | // CLI parsing code, when the linker sees an -L directory that does not exist. | |
| 1616 | ||
| 1617 | if (prefix_dir.access(io, "lib", .{})) |_| { | |
| 1618 | try zig_args.appendSlice(&.{ | |
| 1619 | "-L", b.pathJoin(&.{ search_prefix, "lib" }), | |
| 1620 | }); | |
| 1621 | } else |err| switch (err) { | |
| 1622 | error.FileNotFound => {}, | |
| 1623 | else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ | |
| 1624 | search_prefix, @errorName(e), | |
| 1625 | }), | |
| 1626 | } | |
| 1627 | ||
| 1628 | if (prefix_dir.access(io, "include", .{})) |_| { | |
| 1629 | try zig_args.appendSlice(&.{ | |
| 1630 | "-I", b.pathJoin(&.{ search_prefix, "include" }), | |
| 1631 | }); | |
| 1632 | } else |err| switch (err) { | |
| 1633 | error.FileNotFound => {}, | |
| 1634 | else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ | |
| 1635 | search_prefix, @errorName(e), | |
| 1636 | }), | |
| 1637 | } | |
| 1638 | } | |
| 1639 | ||
| 1640 | if (compile.rc_includes != .any) { | |
| 1641 | try zig_args.append("-rcincludes"); | |
| 1642 | try zig_args.append(@tagName(compile.rc_includes)); | |
| 1643 | } | |
| 1644 | ||
| 1645 | try addFlag(&zig_args, "each-lib-rpath", compile.each_lib_rpath); | |
| 1646 | ||
| 1647 | if (compile.build_id orelse b.build_id) |build_id| { | |
| 1648 | try zig_args.append(switch (build_id) { | |
| 1649 | .hexstring => |hs| b.fmt("--build-id=0x{x}", .{hs.toSlice()}), | |
| 1650 | .none, .fast, .uuid, .sha1, .md5 => b.fmt("--build-id={s}", .{@tagName(build_id)}), | |
| 1651 | }); | |
| 1652 | } | |
| 1653 | ||
| 1654 | const opt_zig_lib_dir = if (compile.zig_lib_dir) |dir| | |
| 1655 | dir.getPath2(b, step) | |
| 1656 | else if (b.graph.zig_lib_directory.path) |_| | |
| 1657 | b.fmt("{f}", .{b.graph.zig_lib_directory}) | |
| 1658 | else | |
| 1659 | null; | |
| 1660 | ||
| 1661 | if (opt_zig_lib_dir) |zig_lib_dir| { | |
| 1662 | try zig_args.append("--zig-lib-dir"); | |
| 1663 | try zig_args.append(zig_lib_dir); | |
| 1664 | } | |
| 1665 | ||
| 1666 | try addFlag(&zig_args, "PIE", compile.pie); | |
| 1667 | ||
| 1668 | if (compile.lto) |lto| { | |
| 1669 | try zig_args.append(switch (lto) { | |
| 1670 | .full => "-flto=full", | |
| 1671 | .thin => "-flto=thin", | |
| 1672 | .none => "-fno-lto", | |
| 1673 | }); | |
| 1674 | } | |
| 1675 | ||
| 1676 | try addFlag(&zig_args, "sanitize-coverage-trace-pc-guard", compile.sanitize_coverage_trace_pc_guard); | |
| 1677 | ||
| 1678 | if (compile.subsystem) |subsystem| { | |
| 1679 | try zig_args.append("--subsystem"); | |
| 1680 | try zig_args.append(@tagName(subsystem)); | |
| 1681 | } | |
| 1682 | ||
| 1683 | if (compile.mingw_unicode_entry_point) { | |
| 1684 | try zig_args.append("-municode"); | |
| 1685 | } | |
| 1686 | ||
| 1687 | if (compile.error_limit) |err_limit| try zig_args.appendSlice(&.{ | |
| 1688 | "--error-limit", b.fmt("{d}", .{err_limit}), | |
| 1689 | }); | |
| 1690 | ||
| 1691 | try addFlag(&zig_args, "incremental", b.graph.incremental); | |
| 1692 | ||
| 1693 | try zig_args.append("--listen=-"); | |
| 1694 | ||
| 1695 | // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux | |
| 1696 | // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and | |
| 1697 | // pass that to zig, e.g. via 'zig build-lib @args.rsp' | |
| 1698 | // See @file syntax here: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html | |
| 1699 | var args_length: usize = 0; | |
| 1700 | for (zig_args.items) |arg| { | |
| 1701 | args_length += arg.len + 1; // +1 to account for null terminator | |
| 1702 | } | |
| 1703 | if (args_length >= 30 * 1024) { | |
| 1704 | try b.cache_root.handle.createDirPath(io, "args"); | |
| 1705 | ||
| 1706 | const args_to_escape = zig_args.items[2..]; | |
| 1707 | var escaped_args = try std.array_list.Managed([]const u8).initCapacity(arena, args_to_escape.len); | |
| 1708 | arg_blk: for (args_to_escape) |arg| { | |
| 1709 | for (arg, 0..) |c, arg_idx| { | |
| 1710 | if (c == '\\' or c == '"') { | |
| 1711 | // Slow path for arguments that need to be escaped. We'll need to allocate and copy | |
| 1712 | var escaped: std.ArrayList(u8) = .empty; | |
| 1713 | try escaped.ensureTotalCapacityPrecise(arena, arg.len + 1); | |
| 1714 | try escaped.appendSlice(arena, arg[0..arg_idx]); | |
| 1715 | for (arg[arg_idx..]) |to_escape| { | |
| 1716 | if (to_escape == '\\' or to_escape == '"') try escaped.append(arena, '\\'); | |
| 1717 | try escaped.append(arena, to_escape); | |
| 1718 | } | |
| 1719 | escaped_args.appendAssumeCapacity(escaped.items); | |
| 1720 | continue :arg_blk; | |
| 1721 | } | |
| 1722 | } | |
| 1723 | escaped_args.appendAssumeCapacity(arg); // no escaping needed so just use original argument | |
| 1724 | } | |
| 1725 | ||
| 1726 | // Write the args to zig-cache/args/<SHA256 hash of args> to avoid conflicts with | |
| 1727 | // other zig build commands running in parallel. | |
| 1728 | const partially_quoted = try std.mem.join(arena, "\" \"", escaped_args.items); | |
| 1729 | const args = try std.mem.concat(arena, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); | |
| 1730 | ||
| 1731 | var args_hash: [Sha256.digest_length]u8 = undefined; | |
| 1732 | Sha256.hash(args, &args_hash, .{}); | |
| 1733 | var args_hex_hash: [Sha256.digest_length * 2]u8 = undefined; | |
| 1734 | _ = try std.fmt.bufPrint(&args_hex_hash, "{x}", .{&args_hash}); | |
| 1735 | ||
| 1736 | const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; | |
| 1737 | if (b.cache_root.handle.access(io, args_file, .{})) |_| { | |
| 1738 | // The args file is already present from a previous run. | |
| 1739 | } else |err| switch (err) { | |
| 1740 | error.FileNotFound => { | |
| 1741 | var af = b.cache_root.handle.createFileAtomic(io, args_file, .{ | |
| 1742 | .replace = false, | |
| 1743 | .make_path = true, | |
| 1744 | }) catch |e| return step.fail("failed creating tmp args file {f}{s}: {t}", .{ | |
| 1745 | b.cache_root, args_file, e, | |
| 1746 | }); | |
| 1747 | defer af.deinit(io); | |
| 1748 | ||
| 1749 | af.file.writeStreamingAll(io, args) catch |e| { | |
| 1750 | return step.fail("failed writing args data to tmp file {f}{s}: {t}", .{ | |
| 1751 | b.cache_root, args_file, e, | |
| 1752 | }); | |
| 1753 | }; | |
| 1754 | // Note we can't clean up this file, not even after build | |
| 1755 | // success, because that might interfere with another build | |
| 1756 | // process that needs the same file. | |
| 1757 | af.link(io) catch |e| switch (e) { | |
| 1758 | error.PathAlreadyExists => { | |
| 1759 | // The args file was created by another concurrent build process. | |
| 1760 | }, | |
| 1761 | else => |other_err| return step.fail("failed linking tmp file {f}{s}: {t}", .{ | |
| 1762 | b.cache_root, args_file, other_err, | |
| 1763 | }), | |
| 1764 | }; | |
| 1765 | }, | |
| 1766 | else => |other_err| return other_err, | |
| 1767 | } | |
| 1768 | ||
| 1769 | const resolved_args_file = try mem.concat(arena, u8, &.{ | |
| 1770 | "@", | |
| 1771 | try b.cache_root.join(arena, &.{args_file}), | |
| 1772 | }); | |
| 1773 | ||
| 1774 | zig_args.shrinkRetainingCapacity(2); | |
| 1775 | try zig_args.append(resolved_args_file); | |
| 1776 | } | |
| 1777 | ||
| 1778 | return try zig_args.toOwnedSlice(); | |
| 1779 | } | |
| 1780 | ||
| 1781 | fn make(step: *Step, options: Step.MakeOptions) !void { | |
| 1782 | const b = step.owner; | |
| 1783 | const compile: *Compile = @fieldParentPtr("step", step); | |
| 1784 | ||
| 1785 | const zig_args = try getZigArgs(compile, false); | |
| 1786 | ||
| 1787 | const maybe_output_dir = step.evalZigProcess( | |
| 1788 | zig_args, | |
| 1789 | options.progress_node, | |
| 1790 | (b.graph.incremental == true) and (options.watch or options.web_server != null), | |
| 1791 | options.web_server, | |
| 1792 | options.gpa, | |
| 1793 | ) catch |err| switch (err) { | |
| 1794 | error.NeedCompileErrorCheck => { | |
| 1795 | assert(compile.expect_errors != null); | |
| 1796 | try checkCompileErrors(compile); | |
| 1797 | return; | |
| 1798 | }, | |
| 1799 | else => |e| return e, | |
| 1800 | }; | |
| 1801 | ||
| 1802 | // Update generated files | |
| 1803 | if (maybe_output_dir) |output_dir| { | |
| 1804 | if (compile.emit_directory) |lp| { | |
| 1805 | lp.path = b.fmt("{f}", .{output_dir}); | |
| 1806 | } | |
| 1807 | ||
| 1808 | // zig fmt: off | |
| 1809 | if (compile.generated_bin) |lp| lp.path = compile.outputPath(output_dir, .bin); | |
| 1810 | if (compile.generated_pdb) |lp| lp.path = compile.outputPath(output_dir, .pdb); | |
| 1811 | // hack for stage2_x86_64 + coff | |
| 1812 | if (compile.generated_compiler_rt_dyn_lib) |lp| lp.path = compile.outputPath(output_dir, .compiler_rt_dyn_lib); | |
| 1813 | if (compile.generated_implib) |lp| lp.path = compile.outputPath(output_dir, .implib); | |
| 1814 | if (compile.generated_h) |lp| lp.path = compile.outputPath(output_dir, .h); | |
| 1815 | if (compile.generated_docs) |lp| lp.path = compile.outputPath(output_dir, .docs); | |
| 1816 | if (compile.generated_asm) |lp| lp.path = compile.outputPath(output_dir, .@"asm"); | |
| 1817 | if (compile.generated_llvm_ir) |lp| lp.path = compile.outputPath(output_dir, .llvm_ir); | |
| 1818 | if (compile.generated_llvm_bc) |lp| lp.path = compile.outputPath(output_dir, .llvm_bc); | |
| 1819 | // zig fmt: on | |
| 1820 | } | |
| 1821 | ||
| 1822 | if (compile.kind == .lib and compile.linkage != null and compile.linkage.? == .dynamic and | |
| 1823 | compile.version != null and compile.generated_bin != null and | |
| 1824 | std.Build.wantSharedLibSymLinks(compile.rootModuleTarget())) | |
| 1825 | { | |
| 1826 | try doAtomicSymLinks( | |
| 1827 | step, | |
| 1828 | compile.getEmittedBin().getPath2(b, step), | |
| 1829 | compile.major_only_filename.?, | |
| 1830 | compile.name_only_filename.?, | |
| 1831 | ); | |
| 1832 | } | |
| 1833 | } | |
| 1834 | 854 | fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifact) []const u8 { |
| 1835 | 855 | const arena = c.step.owner.graph.arena; |
| 1836 | 856 | const name = ea.cacheName(arena, .{ |
| ... | ... | @@ -1847,100 +867,6 @@ fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifa |
| 1847 | 867 | return out_dir.joinString(arena, name) catch @panic("OOM"); |
| 1848 | 868 | } |
| 1849 | 869 | |
| 1850 | pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { | |
| 1851 | c.step.result_error_msgs.clearRetainingCapacity(); | |
| 1852 | c.step.result_stderr = ""; | |
| 1853 | ||
| 1854 | c.step.result_error_bundle.deinit(gpa); | |
| 1855 | c.step.result_error_bundle = std.zig.ErrorBundle.empty; | |
| 1856 | ||
| 1857 | if (c.step.result_failed_command) |cmd| { | |
| 1858 | gpa.free(cmd); | |
| 1859 | c.step.result_failed_command = null; | |
| 1860 | } | |
| 1861 | ||
| 1862 | const zig_args = try getZigArgs(c, true); | |
| 1863 | const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); | |
| 1864 | return maybe_output_bin_path.?; | |
| 1865 | } | |
| 1866 | ||
| 1867 | pub fn doAtomicSymLinks( | |
| 1868 | step: *Step, | |
| 1869 | output_path: []const u8, | |
| 1870 | filename_major_only: []const u8, | |
| 1871 | filename_name_only: []const u8, | |
| 1872 | ) !void { | |
| 1873 | const b = step.owner; | |
| 1874 | const io = b.graph.io; | |
| 1875 | const out_dir = fs.path.dirname(output_path) orelse "."; | |
| 1876 | const out_basename = fs.path.basename(output_path); | |
| 1877 | // sym link for libfoo.so.1 to libfoo.so.1.2.3 | |
| 1878 | const major_only_path = b.pathJoin(&.{ out_dir, filename_major_only }); | |
| 1879 | const cwd: Io.Dir = .cwd(); | |
| 1880 | cwd.symLinkAtomic(io, out_basename, major_only_path, .{}) catch |err| { | |
| 1881 | return step.fail("unable to symlink {s} -> {s}: {s}", .{ | |
| 1882 | major_only_path, out_basename, @errorName(err), | |
| 1883 | }); | |
| 1884 | }; | |
| 1885 | // sym link for libfoo.so to libfoo.so.1 | |
| 1886 | const name_only_path = b.pathJoin(&.{ out_dir, filename_name_only }); | |
| 1887 | cwd.symLinkAtomic(io, filename_major_only, name_only_path, .{}) catch |err| { | |
| 1888 | return step.fail("Unable to symlink {s} -> {s}: {s}", .{ | |
| 1889 | name_only_path, filename_major_only, @errorName(err), | |
| 1890 | }); | |
| 1891 | }; | |
| 1892 | } | |
| 1893 | ||
| 1894 | fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { | |
| 1895 | const pkg_config_exe = b.graph.environ_map.get("PKG_CONFIG") orelse "pkg-config"; | |
| 1896 | const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .ignore); | |
| 1897 | var list = std.array_list.Managed(PkgConfigPkg).init(b.allocator); | |
| 1898 | errdefer list.deinit(); | |
| 1899 | var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); | |
| 1900 | while (line_it.next()) |line| { | |
| 1901 | if (mem.trim(u8, line, " \t").len == 0) continue; | |
| 1902 | var tok_it = mem.tokenizeAny(u8, line, " \t"); | |
| 1903 | try list.append(PkgConfigPkg{ | |
| 1904 | .name = tok_it.next() orelse return error.PkgConfigInvalidOutput, | |
| 1905 | .desc = tok_it.rest(), | |
| 1906 | }); | |
| 1907 | } | |
| 1908 | return list.toOwnedSlice(); | |
| 1909 | } | |
| 1910 | ||
| 1911 | fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { | |
| 1912 | if (b.pkg_config_pkg_list) |res| { | |
| 1913 | return res; | |
| 1914 | } | |
| 1915 | var code: u8 = undefined; | |
| 1916 | if (execPkgConfigList(b, &code)) |list| { | |
| 1917 | b.pkg_config_pkg_list = list; | |
| 1918 | return list; | |
| 1919 | } else |err| { | |
| 1920 | const result = switch (err) { | |
| 1921 | error.ProcessTerminated => error.PkgConfigCrashed, | |
| 1922 | error.ExecNotSupported => error.PkgConfigFailed, | |
| 1923 | error.ExitCodeFailure => error.PkgConfigFailed, | |
| 1924 | error.FileNotFound => error.PkgConfigNotInstalled, | |
| 1925 | error.InvalidName => error.PkgConfigNotInstalled, | |
| 1926 | error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, | |
| 1927 | else => return err, | |
| 1928 | }; | |
| 1929 | b.pkg_config_pkg_list = result; | |
| 1930 | return result; | |
| 1931 | } | |
| 1932 | } | |
| 1933 | ||
| 1934 | fn addFlag(args: *std.array_list.Managed([]const u8), comptime name: []const u8, opt: ?bool) !void { | |
| 1935 | const cond = opt orelse return; | |
| 1936 | try args.ensureUnusedCapacity(1); | |
| 1937 | if (cond) { | |
| 1938 | args.appendAssumeCapacity("-f" ++ name); | |
| 1939 | } else { | |
| 1940 | args.appendAssumeCapacity("-fno-" ++ name); | |
| 1941 | } | |
| 1942 | } | |
| 1943 | ||
| 1944 | 870 | fn checkCompileErrors(compile: *Compile) !void { |
| 1945 | 871 | // Clear this field so that it does not get printed by the build runner. |
| 1946 | 872 | const actual_eb = compile.step.result_error_bundle; |
lib/std/Build/Step/ConfigHeader.zig+2-2
| ... | ... | @@ -47,7 +47,7 @@ max_bytes: usize, |
| 47 | 47 | include_path: []const u8, |
| 48 | 48 | include_guard_override: ?[]const u8, |
| 49 | 49 | |
| 50 | pub const base_id: Step.Id = .config_header; | |
| 50 | pub const base_tag: Step.Tag = .config_header; | |
| 51 | 51 | |
| 52 | 52 | pub const Options = struct { |
| 53 | 53 | style: Style = .blank, |
| ... | ... | @@ -88,7 +88,7 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { |
| 88 | 88 | |
| 89 | 89 | config_header.* = .{ |
| 90 | 90 | .step = .init(.{ |
| 91 | .id = base_id, | |
| 91 | .tag = base_tag, | |
| 92 | 92 | .name = name, |
| 93 | 93 | .owner = owner, |
| 94 | 94 | .makeFn = make, |
lib/std/Build/Step/Fail.zig+2-2
| ... | ... | @@ -6,14 +6,14 @@ const Fail = @This(); |
| 6 | 6 | step: Step, |
| 7 | 7 | error_msg: []const u8, |
| 8 | 8 | |
| 9 | pub const base_id: Step.Id = .fail; | |
| 9 | pub const base_tag: Step.Tag = .fail; | |
| 10 | 10 | |
| 11 | 11 | pub fn create(owner: *std.Build, error_msg: []const u8) *Fail { |
| 12 | 12 | const fail = owner.allocator.create(Fail) catch @panic("OOM"); |
| 13 | 13 | |
| 14 | 14 | fail.* = .{ |
| 15 | 15 | .step = Step.init(.{ |
| 16 | .id = base_id, | |
| 16 | .tag = base_tag, | |
| 17 | 17 | .name = "fail", |
| 18 | 18 | .owner = owner, |
| 19 | 19 | .makeFn = make, |
lib/std/Build/Step/Fmt.zig+2-2
| ... | ... | @@ -10,7 +10,7 @@ paths: []const []const u8, |
| 10 | 10 | exclude_paths: []const []const u8, |
| 11 | 11 | check: bool, |
| 12 | 12 | |
| 13 | pub const base_id: Step.Id = .fmt; | |
| 13 | pub const base_tag: Step.Tag = .fmt; | |
| 14 | 14 | |
| 15 | 15 | pub const Options = struct { |
| 16 | 16 | paths: []const []const u8 = &.{}, |
| ... | ... | @@ -24,7 +24,7 @@ pub fn create(owner: *std.Build, options: Options) *Fmt { |
| 24 | 24 | const name = if (options.check) "zig fmt --check" else "zig fmt"; |
| 25 | 25 | fmt.* = .{ |
| 26 | 26 | .step = Step.init(.{ |
| 27 | .id = base_id, | |
| 27 | .tag = base_tag, | |
| 28 | 28 | .name = name, |
| 29 | 29 | .owner = owner, |
| 30 | 30 | .makeFn = make, |
lib/std/Build/Step/InstallArtifact.zig+2-99
| ... | ... | @@ -33,7 +33,7 @@ const DylibSymlinkInfo = struct { |
| 33 | 33 | name_only_filename: []const u8, |
| 34 | 34 | }; |
| 35 | 35 | |
| 36 | pub const base_id: Step.Id = .install_artifact; | |
| 36 | pub const base_tag: Step.Tag = .install_artifact; | |
| 37 | 37 | |
| 38 | 38 | pub const Options = struct { |
| 39 | 39 | /// Which installation directory to put the main output file into. |
| ... | ... | @@ -69,10 +69,9 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins |
| 69 | 69 | }; |
| 70 | 70 | install_artifact.* = .{ |
| 71 | 71 | .step = Step.init(.{ |
| 72 | .id = base_id, | |
| 72 | .tag = base_tag, | |
| 73 | 73 | .name = owner.fmt("install {s}", .{artifact.name}), |
| 74 | 74 | .owner = owner, |
| 75 | .makeFn = make, | |
| 76 | 75 | }), |
| 77 | 76 | .dest_dir = dest_dir, |
| 78 | 77 | .pdb_dir = switch (options.pdb_dir) { |
| ... | ... | @@ -126,99 +125,3 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins |
| 126 | 125 | |
| 127 | 126 | return install_artifact; |
| 128 | 127 | } |
| 129 | ||
| 130 | fn make(step: *Step, options: Step.MakeOptions) !void { | |
| 131 | _ = options; | |
| 132 | const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); | |
| 133 | const b = step.owner; | |
| 134 | const io = b.graph.io; | |
| 135 | ||
| 136 | var all_cached = true; | |
| 137 | ||
| 138 | if (install_artifact.dest_dir) |dest_dir| { | |
| 139 | const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path); | |
| 140 | const p = try step.installFile(install_artifact.emitted_bin.?, full_dest_path); | |
| 141 | all_cached = all_cached and p == .fresh; | |
| 142 | ||
| 143 | if (install_artifact.dylib_symlinks) |dls| { | |
| 144 | try Step.Compile.doAtomicSymLinks(step, full_dest_path, dls.major_only_filename, dls.name_only_filename); | |
| 145 | } | |
| 146 | ||
| 147 | install_artifact.artifact.installed_path = full_dest_path; | |
| 148 | } | |
| 149 | ||
| 150 | if (install_artifact.compiler_rt_dyn_lib_dir) |compiler_rt_dir| { | |
| 151 | const full_compiler_rt_path = b.getInstallPath(compiler_rt_dir, install_artifact.emitted_compiler_rt_dyn_lib.?.basename(b, step)); | |
| 152 | const p = try step.installFile(install_artifact.emitted_compiler_rt_dyn_lib.?, full_compiler_rt_path); | |
| 153 | all_cached = all_cached and p == .fresh; | |
| 154 | } | |
| 155 | ||
| 156 | if (install_artifact.implib_dir) |implib_dir| { | |
| 157 | const full_implib_path = b.getInstallPath(implib_dir, install_artifact.emitted_implib.?.basename(b, step)); | |
| 158 | const p = try step.installFile(install_artifact.emitted_implib.?, full_implib_path); | |
| 159 | all_cached = all_cached and p == .fresh; | |
| 160 | } | |
| 161 | ||
| 162 | if (install_artifact.pdb_dir) |pdb_dir| { | |
| 163 | const full_pdb_path = b.getInstallPath(pdb_dir, install_artifact.emitted_pdb.?.basename(b, step)); | |
| 164 | const p = try step.installFile(install_artifact.emitted_pdb.?, full_pdb_path); | |
| 165 | all_cached = all_cached and p == .fresh; | |
| 166 | } | |
| 167 | ||
| 168 | if (install_artifact.h_dir) |h_dir| { | |
| 169 | if (install_artifact.emitted_h) |emitted_h| { | |
| 170 | const full_h_path = b.getInstallPath(h_dir, emitted_h.basename(b, step)); | |
| 171 | const p = try step.installFile(emitted_h, full_h_path); | |
| 172 | all_cached = all_cached and p == .fresh; | |
| 173 | } | |
| 174 | ||
| 175 | for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) { | |
| 176 | .file => |file| { | |
| 177 | const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path); | |
| 178 | const p = try step.installFile(file.source, full_h_path); | |
| 179 | all_cached = all_cached and p == .fresh; | |
| 180 | }, | |
| 181 | .directory => |dir| { | |
| 182 | const src_dir_path = dir.source.getPath3(b, step); | |
| 183 | const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path); | |
| 184 | ||
| 185 | var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { | |
| 186 | return step.fail("unable to open source directory '{f}': {s}", .{ | |
| 187 | src_dir_path, @errorName(err), | |
| 188 | }); | |
| 189 | }; | |
| 190 | defer src_dir.close(io); | |
| 191 | ||
| 192 | var it = try src_dir.walk(b.allocator); | |
| 193 | next_entry: while (try it.next(io)) |entry| { | |
| 194 | for (dir.options.exclude_extensions) |ext| { | |
| 195 | if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; | |
| 196 | } | |
| 197 | if (dir.options.include_extensions) |incs| { | |
| 198 | for (incs) |inc| { | |
| 199 | if (std.mem.endsWith(u8, entry.path, inc)) break; | |
| 200 | } else { | |
| 201 | continue :next_entry; | |
| 202 | } | |
| 203 | } | |
| 204 | ||
| 205 | const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path }); | |
| 206 | switch (entry.kind) { | |
| 207 | .directory => { | |
| 208 | try Step.handleVerbose(b, .inherit, &.{ "install", "-d", full_dest_path }); | |
| 209 | const p = try step.installDir(full_dest_path); | |
| 210 | all_cached = all_cached and p == .existed; | |
| 211 | }, | |
| 212 | .file => { | |
| 213 | const p = try step.installFile(try dir.source.join(b.allocator, entry.path), full_dest_path); | |
| 214 | all_cached = all_cached and p == .fresh; | |
| 215 | }, | |
| 216 | else => continue, | |
| 217 | } | |
| 218 | } | |
| 219 | }, | |
| 220 | }; | |
| 221 | } | |
| 222 | ||
| 223 | step.result_cached = all_cached; | |
| 224 | } |
lib/std/Build/Step/InstallDir.zig+2-2
| ... | ... | @@ -8,7 +8,7 @@ const InstallDir = @This(); |
| 8 | 8 | step: Step, |
| 9 | 9 | options: Options, |
| 10 | 10 | |
| 11 | pub const base_id: Step.Id = .install_dir; | |
| 11 | pub const base_tag: Step.Tag = .install_dir; | |
| 12 | 12 | |
| 13 | 13 | pub const Options = struct { |
| 14 | 14 | source_dir: LazyPath, |
| ... | ... | @@ -44,7 +44,7 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir { |
| 44 | 44 | const install_dir = owner.allocator.create(InstallDir) catch @panic("OOM"); |
| 45 | 45 | install_dir.* = .{ |
| 46 | 46 | .step = Step.init(.{ |
| 47 | .id = base_id, | |
| 47 | .tag = base_tag, | |
| 48 | 48 | .name = owner.fmt("install {s}/", .{options.source_dir.getDisplayName()}), |
| 49 | 49 | .owner = owner, |
| 50 | 50 | .makeFn = make, |
lib/std/Build/Step/InstallFile.zig+2-2
| ... | ... | @@ -5,7 +5,7 @@ const InstallDir = std.Build.InstallDir; |
| 5 | 5 | const InstallFile = @This(); |
| 6 | 6 | const assert = std.debug.assert; |
| 7 | 7 | |
| 8 | pub const base_id: Step.Id = .install_file; | |
| 8 | pub const base_tag: Step.Tag = .install_file; | |
| 9 | 9 | |
| 10 | 10 | step: Step, |
| 11 | 11 | source: LazyPath, |
| ... | ... | @@ -22,7 +22,7 @@ pub fn create( |
| 22 | 22 | const install_file = owner.allocator.create(InstallFile) catch @panic("OOM"); |
| 23 | 23 | install_file.* = .{ |
| 24 | 24 | .step = Step.init(.{ |
| 25 | .id = base_id, | |
| 25 | .tag = base_tag, | |
| 26 | 26 | .name = owner.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), |
| 27 | 27 | .owner = owner, |
| 28 | 28 | .makeFn = make, |
lib/std/Build/Step/ObjCopy.zig+2-2
| ... | ... | @@ -10,7 +10,7 @@ const elf = std.elf; |
| 10 | 10 | const fs = std.fs; |
| 11 | 11 | const sort = std.sort; |
| 12 | 12 | |
| 13 | pub const base_id: Step.Id = .objcopy; | |
| 13 | pub const base_tag: Step.Tag = .objcopy; | |
| 14 | 14 | |
| 15 | 15 | pub const RawFormat = enum { |
| 16 | 16 | bin, |
| ... | ... | @@ -111,7 +111,7 @@ pub fn create( |
| 111 | 111 | const objcopy = owner.allocator.create(ObjCopy) catch @panic("OOM"); |
| 112 | 112 | objcopy.* = ObjCopy{ |
| 113 | 113 | .step = Step.init(.{ |
| 114 | .id = base_id, | |
| 114 | .tag = base_tag, | |
| 115 | 115 | .name = owner.fmt("objcopy {s}", .{input_file.getDisplayName()}), |
| 116 | 116 | .owner = owner, |
| 117 | 117 | .makeFn = make, |
lib/std/Build/Step/Options.zig+2-2
| ... | ... | @@ -8,7 +8,7 @@ const Step = std.Build.Step; |
| 8 | 8 | const GeneratedFile = std.Build.GeneratedFile; |
| 9 | 9 | const LazyPath = std.Build.LazyPath; |
| 10 | 10 | |
| 11 | pub const base_id: Step.Id = .options; | |
| 11 | pub const base_tag: Step.Tag = .options; | |
| 12 | 12 | |
| 13 | 13 | step: Step, |
| 14 | 14 | generated_file: GeneratedFile, |
| ... | ... | @@ -21,7 +21,7 @@ pub fn create(owner: *std.Build) *Options { |
| 21 | 21 | const options = owner.allocator.create(Options) catch @panic("OOM"); |
| 22 | 22 | options.* = .{ |
| 23 | 23 | .step = .init(.{ |
| 24 | .id = base_id, | |
| 24 | .tag = base_tag, | |
| 25 | 25 | .name = "options", |
| 26 | 26 | .owner = owner, |
| 27 | 27 | .makeFn = make, |
lib/std/Build/Step/Run.zig+2-2114
| ... | ... | @@ -12,7 +12,7 @@ const EnvMap = std.process.Environ.Map; |
| 12 | 12 | const assert = std.debug.assert; |
| 13 | 13 | const Path = std.Build.Cache.Path; |
| 14 | 14 | |
| 15 | pub const base_id: Step.Id = .run; | |
| 15 | pub const base_tag: Step.Tag = .run; | |
| 16 | 16 | |
| 17 | 17 | step: Step, |
| 18 | 18 | |
| ... | ... | @@ -88,12 +88,6 @@ dep_output_file: ?*Output, |
| 88 | 88 | |
| 89 | 89 | has_side_effects: bool, |
| 90 | 90 | |
| 91 | /// If this is a Zig unit test binary, this tracks the names of the unit | |
| 92 | /// tests that are also fuzz tests. Indexes cannot be used as they may | |
| 93 | /// change between reruns. | |
| 94 | fuzz_tests: std.ArrayList([]const u8), | |
| 95 | cached_test_metadata: ?CachedTestMetadata = null, | |
| 96 | ||
| 97 | 91 | /// Populated during the fuzz phase if this run step corresponds to a unit test |
| 98 | 92 | /// executable that contains fuzz tests. |
| 99 | 93 | rebuilt_executable: ?Path, |
| ... | ... | @@ -209,10 +203,9 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { |
| 209 | 203 | const run = owner.allocator.create(Run) catch @panic("OOM"); |
| 210 | 204 | run.* = .{ |
| 211 | 205 | .step = .init(.{ |
| 212 | .id = base_id, | |
| 206 | .tag = base_tag, | |
| 213 | 207 | .name = name, |
| 214 | 208 | .owner = owner, |
| 215 | .makeFn = make, | |
| 216 | 209 | }), |
| 217 | 210 | .argv = .empty, |
| 218 | 211 | .cwd = null, |
| ... | ... | @@ -229,7 +222,6 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { |
| 229 | 222 | .captured_stderr = null, |
| 230 | 223 | .dep_output_file = null, |
| 231 | 224 | .has_side_effects = false, |
| 232 | .fuzz_tests = .empty, | |
| 233 | 225 | .rebuilt_executable = null, |
| 234 | 226 | .producer = null, |
| 235 | 227 | }; |
| ... | ... | @@ -702,2107 +694,3 @@ pub fn addFileInput(self: *Run, file_input: std.Build.LazyPath) void { |
| 702 | 694 | file_input.addStepDependencies(&self.step); |
| 703 | 695 | self.file_inputs.append(self.step.owner.allocator, file_input.dupe(self.step.owner)) catch @panic("OOM"); |
| 704 | 696 | } |
| 705 | ||
| 706 | /// Returns whether the Run step has side effects *other than* updating the output arguments. | |
| 707 | fn hasSideEffects(run: Run) bool { | |
| 708 | if (run.has_side_effects) return true; | |
| 709 | return switch (run.stdio) { | |
| 710 | .infer_from_args => !run.hasAnyOutputArgs(), | |
| 711 | .inherit => true, | |
| 712 | .check => false, | |
| 713 | .zig_test => false, | |
| 714 | }; | |
| 715 | } | |
| 716 | ||
| 717 | fn hasAnyOutputArgs(run: Run) bool { | |
| 718 | if (run.captured_stdout != null) return true; | |
| 719 | if (run.captured_stderr != null) return true; | |
| 720 | for (run.argv.items) |arg| switch (arg) { | |
| 721 | .output_file, .output_directory => return true, | |
| 722 | else => continue, | |
| 723 | }; | |
| 724 | return false; | |
| 725 | } | |
| 726 | ||
| 727 | fn checksContainStdout(checks: []const StdIo.Check) bool { | |
| 728 | for (checks) |check| switch (check) { | |
| 729 | .expect_stderr_exact, | |
| 730 | .expect_stderr_match, | |
| 731 | .expect_term, | |
| 732 | => continue, | |
| 733 | ||
| 734 | .expect_stdout_exact, | |
| 735 | .expect_stdout_match, | |
| 736 | => return true, | |
| 737 | }; | |
| 738 | return false; | |
| 739 | } | |
| 740 | ||
| 741 | fn checksContainStderr(checks: []const StdIo.Check) bool { | |
| 742 | for (checks) |check| switch (check) { | |
| 743 | .expect_stdout_exact, | |
| 744 | .expect_stdout_match, | |
| 745 | .expect_term, | |
| 746 | => continue, | |
| 747 | ||
| 748 | .expect_stderr_exact, | |
| 749 | .expect_stderr_match, | |
| 750 | => return true, | |
| 751 | }; | |
| 752 | return false; | |
| 753 | } | |
| 754 | ||
| 755 | /// If `path` is cwd-relative, make it relative to the cwd of the child instead. | |
| 756 | /// | |
| 757 | /// Whenever a path is included in the argv of a child, it should be put through this function first | |
| 758 | /// to make sure the child doesn't see paths relative to a cwd other than its own. | |
| 759 | fn convertPathArg(run: *Run, path: Build.Cache.Path) []const u8 { | |
| 760 | const b = run.step.owner; | |
| 761 | const graph = b.graph; | |
| 762 | const arena = graph.arena; | |
| 763 | ||
| 764 | const path_str = path.toString(arena) catch @panic("OOM"); | |
| 765 | if (Dir.path.isAbsolute(path_str)) { | |
| 766 | // Absolute paths don't need changing. | |
| 767 | return path_str; | |
| 768 | } | |
| 769 | const child_cwd_rel: []const u8 = rel: { | |
| 770 | const child_lazy_cwd = run.cwd orelse break :rel path_str; | |
| 771 | const child_cwd = child_lazy_cwd.getPath3(b, &run.step).toString(arena) catch @panic("OOM"); | |
| 772 | // Convert it from relative to *our* cwd, to relative to the *child's* cwd. | |
| 773 | break :rel Dir.path.relative(arena, graph.cache.cwd, &graph.environ_map, child_cwd, path_str) catch @panic("OOM"); | |
| 774 | }; | |
| 775 | // Not every path can be made relative, e.g. if the path and the child cwd are on different | |
| 776 | // disk designators on Windows. In that case, `relative` will return an absolute path which we can | |
| 777 | // just return. | |
| 778 | if (Dir.path.isAbsolute(child_cwd_rel)) return child_cwd_rel; | |
| 779 | ||
| 780 | // We're not done yet. In some cases this path must be prefixed with './': | |
| 781 | // * On POSIX, the executable name cannot be a single component like 'foo' | |
| 782 | // * Some executables might treat a leading '-' like a flag, which we must avoid | |
| 783 | // There's no harm in it, so just *always* apply this prefix. | |
| 784 | return Dir.path.join(arena, &.{ ".", child_cwd_rel }) catch @panic("OOM"); | |
| 785 | } | |
| 786 | ||
| 787 | const IndexedOutput = struct { | |
| 788 | index: usize, | |
| 789 | tag: @typeInfo(Arg).@"union".tag_type.?, | |
| 790 | output: *Output, | |
| 791 | }; | |
| 792 | fn make(step: *Step, options: Step.MakeOptions) !void { | |
| 793 | const b = step.owner; | |
| 794 | const io = b.graph.io; | |
| 795 | const arena = b.allocator; | |
| 796 | const run: *Run = @fieldParentPtr("step", step); | |
| 797 | const has_side_effects = run.hasSideEffects(); | |
| 798 | ||
| 799 | var argv_list = std.array_list.Managed([]const u8).init(arena); | |
| 800 | var output_placeholders = std.array_list.Managed(IndexedOutput).init(arena); | |
| 801 | ||
| 802 | var man = b.graph.cache.obtain(); | |
| 803 | defer man.deinit(); | |
| 804 | ||
| 805 | if (run.environ_map) |environ_map| { | |
| 806 | for (environ_map.keys(), environ_map.values()) |key, value| { | |
| 807 | man.hash.addBytes(key); | |
| 808 | man.hash.addBytes(value); | |
| 809 | } | |
| 810 | } | |
| 811 | ||
| 812 | man.hash.add(run.color); | |
| 813 | man.hash.add(run.disable_zig_progress); | |
| 814 | ||
| 815 | for (run.argv.items) |arg| { | |
| 816 | switch (arg) { | |
| 817 | .bytes => |bytes| { | |
| 818 | try argv_list.append(bytes); | |
| 819 | man.hash.addBytes(bytes); | |
| 820 | }, | |
| 821 | .lazy_path => |file| { | |
| 822 | const file_path = file.lazy_path.getPath3(b, step); | |
| 823 | try argv_list.append(b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); | |
| 824 | man.hash.addBytes(file.prefix); | |
| 825 | _ = try man.addFilePath(file_path, null); | |
| 826 | }, | |
| 827 | .decorated_directory => |dd| { | |
| 828 | const file_path = dd.lazy_path.getPath3(b, step); | |
| 829 | const resolved_arg = b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix }); | |
| 830 | try argv_list.append(resolved_arg); | |
| 831 | man.hash.addBytes(resolved_arg); | |
| 832 | }, | |
| 833 | .file_content => |file_plp| { | |
| 834 | const file_path = file_plp.lazy_path.getPath3(b, step); | |
| 835 | ||
| 836 | var result: std.Io.Writer.Allocating = .init(arena); | |
| 837 | errdefer result.deinit(); | |
| 838 | result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; | |
| 839 | ||
| 840 | const file = file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}) catch |err| { | |
| 841 | return step.fail( | |
| 842 | "unable to open input file '{f}': {t}", | |
| 843 | .{ file_path, err }, | |
| 844 | ); | |
| 845 | }; | |
| 846 | defer file.close(io); | |
| 847 | ||
| 848 | var buf: [1024]u8 = undefined; | |
| 849 | var file_reader = file.reader(io, &buf); | |
| 850 | _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { | |
| 851 | error.ReadFailed => return step.fail( | |
| 852 | "failed to read from '{f}': {t}", | |
| 853 | .{ file_path, file_reader.err.? }, | |
| 854 | ), | |
| 855 | error.WriteFailed => return error.OutOfMemory, | |
| 856 | }; | |
| 857 | ||
| 858 | try argv_list.append(result.written()); | |
| 859 | man.hash.addBytes(file_plp.prefix); | |
| 860 | _ = try man.addFilePath(file_path, null); | |
| 861 | }, | |
| 862 | .artifact => |pa| { | |
| 863 | const artifact = pa.artifact; | |
| 864 | ||
| 865 | if (artifact.rootModuleTarget().os.tag == .windows) { | |
| 866 | // On Windows we don't have rpaths so we have to add .dll search paths to PATH | |
| 867 | run.addPathForDynLibs(artifact); | |
| 868 | } | |
| 869 | const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; | |
| 870 | ||
| 871 | try argv_list.append(b.fmt("{s}{s}", .{ | |
| 872 | pa.prefix, | |
| 873 | run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), | |
| 874 | })); | |
| 875 | ||
| 876 | _ = try man.addFile(file_path, null); | |
| 877 | }, | |
| 878 | .output_file, .output_directory => |output| { | |
| 879 | man.hash.addBytes(output.prefix); | |
| 880 | man.hash.addBytes(output.basename); | |
| 881 | // Add a placeholder into the argument list because we need the | |
| 882 | // manifest hash to be updated with all arguments before the | |
| 883 | // object directory is computed. | |
| 884 | try output_placeholders.append(.{ | |
| 885 | .index = argv_list.items.len, | |
| 886 | .tag = arg, | |
| 887 | .output = output, | |
| 888 | }); | |
| 889 | _ = try argv_list.addOne(); | |
| 890 | }, | |
| 891 | } | |
| 892 | } | |
| 893 | ||
| 894 | switch (run.stdin) { | |
| 895 | .bytes => |bytes| { | |
| 896 | man.hash.addBytes(bytes); | |
| 897 | }, | |
| 898 | .lazy_path => |lazy_path| { | |
| 899 | const file_path = lazy_path.getPath2(b, step); | |
| 900 | _ = try man.addFile(file_path, null); | |
| 901 | }, | |
| 902 | .none => {}, | |
| 903 | } | |
| 904 | ||
| 905 | if (run.captured_stdout) |captured| { | |
| 906 | man.hash.addBytes(captured.output.basename); | |
| 907 | man.hash.add(captured.trim_whitespace); | |
| 908 | } | |
| 909 | ||
| 910 | if (run.captured_stderr) |captured| { | |
| 911 | man.hash.addBytes(captured.output.basename); | |
| 912 | man.hash.add(captured.trim_whitespace); | |
| 913 | } | |
| 914 | ||
| 915 | hashStdIo(&man.hash, run.stdio); | |
| 916 | ||
| 917 | for (run.file_inputs.items) |lazy_path| { | |
| 918 | _ = try man.addFile(lazy_path.getPath2(b, step), null); | |
| 919 | } | |
| 920 | ||
| 921 | if (run.cwd) |cwd| { | |
| 922 | const cwd_path = cwd.getPath3(b, step); | |
| 923 | _ = man.hash.addBytes(try cwd_path.toString(arena)); | |
| 924 | } | |
| 925 | ||
| 926 | if (!has_side_effects and try step.cacheHitAndWatch(&man)) { | |
| 927 | // cache hit, skip running command | |
| 928 | const digest = man.final(); | |
| 929 | ||
| 930 | try populateGeneratedPaths( | |
| 931 | arena, | |
| 932 | output_placeholders.items, | |
| 933 | run.captured_stdout, | |
| 934 | run.captured_stderr, | |
| 935 | b.cache_root, | |
| 936 | &digest, | |
| 937 | ); | |
| 938 | ||
| 939 | step.result_cached = true; | |
| 940 | return; | |
| 941 | } | |
| 942 | ||
| 943 | const dep_output_file = run.dep_output_file orelse { | |
| 944 | // We already know the final output paths, use them directly. | |
| 945 | const digest = if (has_side_effects) | |
| 946 | man.hash.final() | |
| 947 | else | |
| 948 | man.final(); | |
| 949 | ||
| 950 | try populateGeneratedPaths( | |
| 951 | arena, | |
| 952 | output_placeholders.items, | |
| 953 | run.captured_stdout, | |
| 954 | run.captured_stderr, | |
| 955 | b.cache_root, | |
| 956 | &digest, | |
| 957 | ); | |
| 958 | ||
| 959 | const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; | |
| 960 | for (output_placeholders.items) |placeholder| { | |
| 961 | const output_sub_path = b.pathJoin(&.{ output_dir_path, placeholder.output.basename }); | |
| 962 | const output_sub_dir_path = switch (placeholder.tag) { | |
| 963 | .output_file => Dir.path.dirname(output_sub_path).?, | |
| 964 | .output_directory => output_sub_path, | |
| 965 | else => unreachable, | |
| 966 | }; | |
| 967 | b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { | |
| 968 | return step.fail("unable to make path '{f}{s}': {s}", .{ | |
| 969 | b.cache_root, output_sub_dir_path, @errorName(err), | |
| 970 | }); | |
| 971 | }; | |
| 972 | const arg_output_path = run.convertPathArg(.{ | |
| 973 | .root_dir = .cwd(), | |
| 974 | .sub_path = placeholder.output.generated_file.getPath(), | |
| 975 | }); | |
| 976 | argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) | |
| 977 | arg_output_path | |
| 978 | else | |
| 979 | b.fmt("{s}{s}", .{ placeholder.output.prefix, arg_output_path }); | |
| 980 | } | |
| 981 | ||
| 982 | try runCommand(run, argv_list.items, has_side_effects, output_dir_path, options, null); | |
| 983 | if (!has_side_effects) try step.writeManifestAndWatch(&man); | |
| 984 | return; | |
| 985 | }; | |
| 986 | ||
| 987 | // We do not know the final output paths yet, use temp paths to run the command. | |
| 988 | var rand_int: u64 = undefined; | |
| 989 | io.random(@ptrCast(&rand_int)); | |
| 990 | const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); | |
| 991 | ||
| 992 | for (output_placeholders.items) |placeholder| { | |
| 993 | const output_components = .{ tmp_dir_path, placeholder.output.basename }; | |
| 994 | const output_sub_path = b.pathJoin(&output_components); | |
| 995 | const output_sub_dir_path = switch (placeholder.tag) { | |
| 996 | .output_file => Dir.path.dirname(output_sub_path).?, | |
| 997 | .output_directory => output_sub_path, | |
| 998 | else => unreachable, | |
| 999 | }; | |
| 1000 | b.cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { | |
| 1001 | return step.fail("unable to make path '{f}{s}': {s}", .{ | |
| 1002 | b.cache_root, output_sub_dir_path, @errorName(err), | |
| 1003 | }); | |
| 1004 | }; | |
| 1005 | const raw_output_path: Build.Cache.Path = .{ | |
| 1006 | .root_dir = b.cache_root, | |
| 1007 | .sub_path = b.pathJoin(&output_components), | |
| 1008 | }; | |
| 1009 | placeholder.output.generated_file.path = raw_output_path.toString(b.graph.arena) catch @panic("OOM"); | |
| 1010 | argv_list.items[placeholder.index] = b.fmt("{s}{s}", .{ | |
| 1011 | placeholder.output.prefix, | |
| 1012 | run.convertPathArg(raw_output_path), | |
| 1013 | }); | |
| 1014 | } | |
| 1015 | ||
| 1016 | try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, options, null); | |
| 1017 | ||
| 1018 | const dep_file_dir = Dir.cwd(); | |
| 1019 | const dep_file_basename = dep_output_file.generated_file.getPath2(b, step); | |
| 1020 | if (has_side_effects) | |
| 1021 | try man.addDepFile(dep_file_dir, dep_file_basename) | |
| 1022 | else | |
| 1023 | try man.addDepFilePost(dep_file_dir, dep_file_basename); | |
| 1024 | ||
| 1025 | const digest = if (has_side_effects) | |
| 1026 | man.hash.final() | |
| 1027 | else | |
| 1028 | man.final(); | |
| 1029 | ||
| 1030 | const any_output = output_placeholders.items.len > 0 or | |
| 1031 | run.captured_stdout != null or run.captured_stderr != null; | |
| 1032 | ||
| 1033 | // Rename into place | |
| 1034 | if (any_output) { | |
| 1035 | const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; | |
| 1036 | ||
| 1037 | b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |err| switch (err) { | |
| 1038 | Dir.RenameError.DirNotEmpty => { | |
| 1039 | b.cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { | |
| 1040 | return step.fail("unable to remove dir '{f}'{s}: {t}", .{ | |
| 1041 | b.cache_root, tmp_dir_path, del_err, | |
| 1042 | }); | |
| 1043 | }; | |
| 1044 | b.cache_root.handle.rename(tmp_dir_path, b.cache_root.handle, o_sub_path, io) catch |retry_err| { | |
| 1045 | return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ | |
| 1046 | b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, retry_err, | |
| 1047 | }); | |
| 1048 | }; | |
| 1049 | }, | |
| 1050 | else => return step.fail("unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ | |
| 1051 | b.cache_root, tmp_dir_path, b.cache_root, o_sub_path, err, | |
| 1052 | }), | |
| 1053 | }; | |
| 1054 | } | |
| 1055 | ||
| 1056 | if (!has_side_effects) try step.writeManifestAndWatch(&man); | |
| 1057 | ||
| 1058 | try populateGeneratedPaths( | |
| 1059 | arena, | |
| 1060 | output_placeholders.items, | |
| 1061 | run.captured_stdout, | |
| 1062 | run.captured_stderr, | |
| 1063 | b.cache_root, | |
| 1064 | &digest, | |
| 1065 | ); | |
| 1066 | } | |
| 1067 | ||
| 1068 | pub fn rerunInFuzzMode( | |
| 1069 | run: *Run, | |
| 1070 | fuzz: *std.Build.Fuzz, | |
| 1071 | prog_node: std.Progress.Node, | |
| 1072 | ) !void { | |
| 1073 | const step = &run.step; | |
| 1074 | const b = step.owner; | |
| 1075 | const io = b.graph.io; | |
| 1076 | const arena = b.allocator; | |
| 1077 | var argv_list: std.ArrayList([]const u8) = .empty; | |
| 1078 | for (run.argv.items) |arg| { | |
| 1079 | switch (arg) { | |
| 1080 | .bytes => |bytes| { | |
| 1081 | try argv_list.append(arena, bytes); | |
| 1082 | }, | |
| 1083 | .lazy_path => |file| { | |
| 1084 | const file_path = file.lazy_path.getPath3(b, step); | |
| 1085 | try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, run.convertPathArg(file_path) })); | |
| 1086 | }, | |
| 1087 | .decorated_directory => |dd| { | |
| 1088 | const file_path = dd.lazy_path.getPath3(b, step); | |
| 1089 | try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, run.convertPathArg(file_path), dd.suffix })); | |
| 1090 | }, | |
| 1091 | .file_content => |file_plp| { | |
| 1092 | const file_path = file_plp.lazy_path.getPath3(b, step); | |
| 1093 | ||
| 1094 | var result: std.Io.Writer.Allocating = .init(arena); | |
| 1095 | errdefer result.deinit(); | |
| 1096 | result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; | |
| 1097 | ||
| 1098 | const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); | |
| 1099 | defer file.close(io); | |
| 1100 | ||
| 1101 | var buf: [1024]u8 = undefined; | |
| 1102 | var file_reader = file.reader(io, &buf); | |
| 1103 | _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { | |
| 1104 | error.ReadFailed => return file_reader.err.?, | |
| 1105 | error.WriteFailed => return error.OutOfMemory, | |
| 1106 | }; | |
| 1107 | ||
| 1108 | try argv_list.append(arena, result.written()); | |
| 1109 | }, | |
| 1110 | .artifact => |pa| { | |
| 1111 | const artifact = pa.artifact; | |
| 1112 | const file_path: []const u8 = p: { | |
| 1113 | if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); | |
| 1114 | break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; | |
| 1115 | }; | |
| 1116 | try argv_list.append(arena, b.fmt("{s}{s}", .{ | |
| 1117 | pa.prefix, | |
| 1118 | run.convertPathArg(.{ .root_dir = .cwd(), .sub_path = file_path }), | |
| 1119 | })); | |
| 1120 | }, | |
| 1121 | .output_file, .output_directory => unreachable, | |
| 1122 | } | |
| 1123 | } | |
| 1124 | ||
| 1125 | if (run.step.result_failed_command) |cmd| { | |
| 1126 | fuzz.gpa.free(cmd); | |
| 1127 | run.step.result_failed_command = null; | |
| 1128 | } | |
| 1129 | ||
| 1130 | const has_side_effects = false; | |
| 1131 | var rand_int: u64 = undefined; | |
| 1132 | io.random(@ptrCast(&rand_int)); | |
| 1133 | const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); | |
| 1134 | try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{ | |
| 1135 | .progress_node = prog_node, | |
| 1136 | .watch = undefined, // not used by `runCommand` | |
| 1137 | .web_server = null, // only needed for time reports | |
| 1138 | .unit_test_timeout_ns = null, // don't time out fuzz tests for now | |
| 1139 | .gpa = fuzz.gpa, | |
| 1140 | }, .{ | |
| 1141 | .fuzz = fuzz, | |
| 1142 | }); | |
| 1143 | } | |
| 1144 | ||
| 1145 | fn populateGeneratedPaths( | |
| 1146 | arena: std.mem.Allocator, | |
| 1147 | output_placeholders: []const IndexedOutput, | |
| 1148 | captured_stdout: ?*CapturedStdIo, | |
| 1149 | captured_stderr: ?*CapturedStdIo, | |
| 1150 | cache_root: Build.Cache.Directory, | |
| 1151 | digest: *const Build.Cache.HexDigest, | |
| 1152 | ) !void { | |
| 1153 | for (output_placeholders) |placeholder| { | |
| 1154 | placeholder.output.generated_file.path = try cache_root.join(arena, &.{ | |
| 1155 | "o", digest, placeholder.output.basename, | |
| 1156 | }); | |
| 1157 | } | |
| 1158 | ||
| 1159 | if (captured_stdout) |captured| { | |
| 1160 | captured.output.generated_file.path = try cache_root.join(arena, &.{ | |
| 1161 | "o", digest, captured.output.basename, | |
| 1162 | }); | |
| 1163 | } | |
| 1164 | ||
| 1165 | if (captured_stderr) |captured| { | |
| 1166 | captured.output.generated_file.path = try cache_root.join(arena, &.{ | |
| 1167 | "o", digest, captured.output.basename, | |
| 1168 | }); | |
| 1169 | } | |
| 1170 | } | |
| 1171 | ||
| 1172 | fn formatTerm(term: ?process.Child.Term, w: *std.Io.Writer) std.Io.Writer.Error!void { | |
| 1173 | if (term) |t| switch (t) { | |
| 1174 | .exited => |code| try w.print("exited with code {d}", .{code}), | |
| 1175 | .signal => |sig| try w.print("terminated with signal {t}", .{sig}), | |
| 1176 | .stopped => |sig| try w.print("stopped with signal {t}", .{sig}), | |
| 1177 | .unknown => |code| try w.print("terminated for unknown reason with code {d}", .{code}), | |
| 1178 | } else { | |
| 1179 | try w.writeAll("exited with any code"); | |
| 1180 | } | |
| 1181 | } | |
| 1182 | fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTerm) { | |
| 1183 | return .{ .data = term }; | |
| 1184 | } | |
| 1185 | ||
| 1186 | fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool { | |
| 1187 | return if (expected) |e| switch (e) { | |
| 1188 | .exited => |expected_code| switch (actual) { | |
| 1189 | .exited => |actual_code| expected_code == actual_code, | |
| 1190 | else => false, | |
| 1191 | }, | |
| 1192 | .signal => |expected_sig| switch (actual) { | |
| 1193 | .signal => |actual_sig| expected_sig == actual_sig, | |
| 1194 | else => false, | |
| 1195 | }, | |
| 1196 | .stopped => |expected_sig| switch (actual) { | |
| 1197 | .stopped => |actual_sig| expected_sig == actual_sig, | |
| 1198 | else => false, | |
| 1199 | }, | |
| 1200 | .unknown => |expected_code| switch (actual) { | |
| 1201 | .unknown => |actual_code| expected_code == actual_code, | |
| 1202 | else => false, | |
| 1203 | }, | |
| 1204 | } else switch (actual) { | |
| 1205 | .exited => true, | |
| 1206 | else => false, | |
| 1207 | }; | |
| 1208 | } | |
| 1209 | ||
| 1210 | const FuzzContext = struct { | |
| 1211 | fuzz: *std.Build.Fuzz, | |
| 1212 | }; | |
| 1213 | ||
| 1214 | fn runCommand( | |
| 1215 | run: *Run, | |
| 1216 | argv: []const []const u8, | |
| 1217 | has_side_effects: bool, | |
| 1218 | output_dir_path: []const u8, | |
| 1219 | options: Step.MakeOptions, | |
| 1220 | fuzz_context: ?FuzzContext, | |
| 1221 | ) !void { | |
| 1222 | const step = &run.step; | |
| 1223 | const b = step.owner; | |
| 1224 | const arena = b.allocator; | |
| 1225 | const gpa = options.gpa; | |
| 1226 | const io = b.graph.io; | |
| 1227 | ||
| 1228 | const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; | |
| 1229 | ||
| 1230 | try step.handleChildProcUnsupported(); | |
| 1231 | try Step.handleVerbose2(step.owner, cwd, run.environ_map, argv); | |
| 1232 | ||
| 1233 | const allow_skip = switch (run.stdio) { | |
| 1234 | .check, .zig_test => run.skip_foreign_checks, | |
| 1235 | else => false, | |
| 1236 | }; | |
| 1237 | ||
| 1238 | var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); | |
| 1239 | defer interp_argv.deinit(); | |
| 1240 | ||
| 1241 | var environ_map: EnvMap = env: { | |
| 1242 | const orig = run.environ_map orelse &b.graph.environ_map; | |
| 1243 | break :env try orig.clone(gpa); | |
| 1244 | }; | |
| 1245 | defer environ_map.deinit(); | |
| 1246 | ||
| 1247 | const opt_generic_result = spawnChildAndCollect(run, argv, &environ_map, has_side_effects, options, fuzz_context) catch |err| term: { | |
| 1248 | // InvalidExe: cpu arch mismatch | |
| 1249 | // FileNotFound: can happen with a wrong dynamic linker path | |
| 1250 | if (err == error.InvalidExe or err == error.FileNotFound) interpret: { | |
| 1251 | // TODO: learn the target from the binary directly rather than from | |
| 1252 | // relying on it being a Compile step. This will make this logic | |
| 1253 | // work even for the edge case that the binary was produced by a | |
| 1254 | // third party. | |
| 1255 | const exe = switch (run.argv.items[0]) { | |
| 1256 | .artifact => |exe| exe.artifact, | |
| 1257 | else => break :interpret, | |
| 1258 | }; | |
| 1259 | switch (exe.kind) { | |
| 1260 | .exe, .@"test" => {}, | |
| 1261 | else => break :interpret, | |
| 1262 | } | |
| 1263 | ||
| 1264 | const root_target = exe.rootModuleTarget(); | |
| 1265 | const need_cross_libc = exe.is_linking_libc and | |
| 1266 | (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); | |
| 1267 | const other_target = exe.root_module.resolved_target.?.result; | |
| 1268 | switch (std.zig.system.getExternalExecutor(io, &b.graph.host.result, &other_target, .{ | |
| 1269 | .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, | |
| 1270 | .link_libc = exe.is_linking_libc, | |
| 1271 | })) { | |
| 1272 | .native, .rosetta => { | |
| 1273 | if (allow_skip) return error.MakeSkipped; | |
| 1274 | break :interpret; | |
| 1275 | }, | |
| 1276 | .wine => |bin_name| { | |
| 1277 | if (b.enable_wine) { | |
| 1278 | try interp_argv.append(bin_name); | |
| 1279 | try interp_argv.appendSlice(argv); | |
| 1280 | ||
| 1281 | // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but | |
| 1282 | // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. | |
| 1283 | if (environ_map.get("WINEDEBUG") == null) { | |
| 1284 | try environ_map.put("WINEDEBUG", "-all"); | |
| 1285 | } | |
| 1286 | } else { | |
| 1287 | return failForeign(run, "-fwine", argv[0], exe); | |
| 1288 | } | |
| 1289 | }, | |
| 1290 | .qemu => |bin_name| { | |
| 1291 | if (b.enable_qemu) { | |
| 1292 | try interp_argv.append(bin_name); | |
| 1293 | ||
| 1294 | if (need_cross_libc) { | |
| 1295 | if (b.libc_runtimes_dir) |dir| { | |
| 1296 | try interp_argv.append("-L"); | |
| 1297 | try interp_argv.append(b.pathJoin(&.{ | |
| 1298 | dir, | |
| 1299 | try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( | |
| 1300 | b.allocator, | |
| 1301 | root_target.cpu.arch, | |
| 1302 | root_target.os.tag, | |
| 1303 | root_target.abi, | |
| 1304 | ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( | |
| 1305 | b.allocator, | |
| 1306 | root_target.cpu.arch, | |
| 1307 | root_target.abi, | |
| 1308 | ) else unreachable, | |
| 1309 | })); | |
| 1310 | } else return failForeign(run, "--libc-runtimes", argv[0], exe); | |
| 1311 | } | |
| 1312 | ||
| 1313 | try interp_argv.appendSlice(argv); | |
| 1314 | } else return failForeign(run, "-fqemu", argv[0], exe); | |
| 1315 | }, | |
| 1316 | .darling => |bin_name| { | |
| 1317 | if (b.enable_darling) { | |
| 1318 | try interp_argv.append(bin_name); | |
| 1319 | try interp_argv.appendSlice(argv); | |
| 1320 | } else { | |
| 1321 | return failForeign(run, "-fdarling", argv[0], exe); | |
| 1322 | } | |
| 1323 | }, | |
| 1324 | .wasmtime => |bin_name| { | |
| 1325 | if (b.enable_wasmtime) { | |
| 1326 | try interp_argv.append(bin_name); | |
| 1327 | try interp_argv.append("--dir=."); | |
| 1328 | // Wasmtime doeesn't inherit environment variables from the parent process | |
| 1329 | // by default. '-S inherit-env' was added in Wasmtime version 20. | |
| 1330 | try interp_argv.append("-Sinherit-env"); | |
| 1331 | try interp_argv.append(argv[0]); | |
| 1332 | try interp_argv.appendSlice(argv[1..]); | |
| 1333 | } else { | |
| 1334 | return failForeign(run, "-fwasmtime", argv[0], exe); | |
| 1335 | } | |
| 1336 | }, | |
| 1337 | .bad_dl => |foreign_dl| { | |
| 1338 | if (allow_skip) return error.MakeSkipped; | |
| 1339 | ||
| 1340 | const host_dl = b.graph.host.result.dynamic_linker.get() orelse "(none)"; | |
| 1341 | ||
| 1342 | return step.fail( | |
| 1343 | \\the host system is unable to execute binaries from the target | |
| 1344 | \\ because the host dynamic linker is '{s}', | |
| 1345 | \\ while the target dynamic linker is '{s}'. | |
| 1346 | \\ consider setting the dynamic linker or enabling skip_foreign_checks in the Run step | |
| 1347 | , .{ host_dl, foreign_dl }); | |
| 1348 | }, | |
| 1349 | .bad_os_or_cpu => { | |
| 1350 | if (allow_skip) return error.MakeSkipped; | |
| 1351 | ||
| 1352 | const host_name = try b.graph.host.result.zigTriple(b.allocator); | |
| 1353 | const foreign_name = try root_target.zigTriple(b.allocator); | |
| 1354 | ||
| 1355 | return step.fail("the host system ({s}) is unable to execute binaries from the target ({s})", .{ | |
| 1356 | host_name, foreign_name, | |
| 1357 | }); | |
| 1358 | }, | |
| 1359 | } | |
| 1360 | ||
| 1361 | if (root_target.os.tag == .windows) { | |
| 1362 | // On Windows we don't have rpaths so we have to add .dll search paths to PATH | |
| 1363 | run.addPathForDynLibs(exe); | |
| 1364 | } | |
| 1365 | ||
| 1366 | gpa.free(step.result_failed_command.?); | |
| 1367 | step.result_failed_command = null; | |
| 1368 | try Step.handleVerbose2(step.owner, cwd, run.environ_map, interp_argv.items); | |
| 1369 | ||
| 1370 | break :term spawnChildAndCollect(run, interp_argv.items, &environ_map, has_side_effects, options, fuzz_context) catch |e| { | |
| 1371 | if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; | |
| 1372 | if (e == error.MakeFailed) return error.MakeFailed; // error already reported | |
| 1373 | return step.fail("unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); | |
| 1374 | }; | |
| 1375 | } | |
| 1376 | if (err == error.MakeFailed) return error.MakeFailed; // error already reported | |
| 1377 | ||
| 1378 | return step.fail("failed to spawn and capture stdio from {s}: {t}", .{ argv[0], err }); | |
| 1379 | }; | |
| 1380 | ||
| 1381 | const generic_result = opt_generic_result orelse { | |
| 1382 | assert(run.stdio == .zig_test); | |
| 1383 | // Specific errors have already been reported, and test results are populated. All we need | |
| 1384 | // to do is report step failure if any test failed. | |
| 1385 | if (!step.test_results.isSuccess()) return error.MakeFailed; | |
| 1386 | return; | |
| 1387 | }; | |
| 1388 | ||
| 1389 | assert(fuzz_context == null); | |
| 1390 | assert(run.stdio != .zig_test); | |
| 1391 | ||
| 1392 | // Capture stdout and stderr to GeneratedFile objects. | |
| 1393 | const Stream = struct { | |
| 1394 | captured: ?*CapturedStdIo, | |
| 1395 | bytes: ?[]const u8, | |
| 1396 | }; | |
| 1397 | for ([_]Stream{ | |
| 1398 | .{ | |
| 1399 | .captured = run.captured_stdout, | |
| 1400 | .bytes = generic_result.stdout, | |
| 1401 | }, | |
| 1402 | .{ | |
| 1403 | .captured = run.captured_stderr, | |
| 1404 | .bytes = generic_result.stderr, | |
| 1405 | }, | |
| 1406 | }) |stream| { | |
| 1407 | if (stream.captured) |captured| { | |
| 1408 | const output_components = .{ output_dir_path, captured.output.basename }; | |
| 1409 | const output_path = try b.cache_root.join(arena, &output_components); | |
| 1410 | captured.output.generated_file.path = output_path; | |
| 1411 | ||
| 1412 | const sub_path = b.pathJoin(&output_components); | |
| 1413 | const sub_path_dirname = Dir.path.dirname(sub_path).?; | |
| 1414 | b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { | |
| 1415 | return step.fail("unable to make path '{f}{s}': {s}", .{ | |
| 1416 | b.cache_root, sub_path_dirname, @errorName(err), | |
| 1417 | }); | |
| 1418 | }; | |
| 1419 | const data = switch (captured.trim_whitespace) { | |
| 1420 | .none => stream.bytes.?, | |
| 1421 | .all => mem.trim(u8, stream.bytes.?, &std.ascii.whitespace), | |
| 1422 | .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), | |
| 1423 | .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), | |
| 1424 | }; | |
| 1425 | b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { | |
| 1426 | return step.fail("unable to write file '{f}{s}': {s}", .{ | |
| 1427 | b.cache_root, sub_path, @errorName(err), | |
| 1428 | }); | |
| 1429 | }; | |
| 1430 | } | |
| 1431 | } | |
| 1432 | ||
| 1433 | switch (run.stdio) { | |
| 1434 | .zig_test => unreachable, | |
| 1435 | .check => |checks| for (checks.items) |check| switch (check) { | |
| 1436 | .expect_stderr_exact => |expected_bytes| { | |
| 1437 | if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) { | |
| 1438 | return step.fail( | |
| 1439 | \\========= expected this stderr: ========= | |
| 1440 | \\{s} | |
| 1441 | \\========= but found: ==================== | |
| 1442 | \\{s} | |
| 1443 | , .{ | |
| 1444 | expected_bytes, | |
| 1445 | generic_result.stderr.?, | |
| 1446 | }); | |
| 1447 | } | |
| 1448 | }, | |
| 1449 | .expect_stderr_match => |match| { | |
| 1450 | if (mem.find(u8, generic_result.stderr.?, match) == null) { | |
| 1451 | return step.fail( | |
| 1452 | \\========= expected to find in stderr: ========= | |
| 1453 | \\{s} | |
| 1454 | \\========= but stderr does not contain it: ===== | |
| 1455 | \\{s} | |
| 1456 | , .{ | |
| 1457 | match, | |
| 1458 | generic_result.stderr.?, | |
| 1459 | }); | |
| 1460 | } | |
| 1461 | }, | |
| 1462 | .expect_stdout_exact => |expected_bytes| { | |
| 1463 | if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) { | |
| 1464 | return step.fail( | |
| 1465 | \\========= expected this stdout: ========= | |
| 1466 | \\{s} | |
| 1467 | \\========= but found: ==================== | |
| 1468 | \\{s} | |
| 1469 | , .{ | |
| 1470 | expected_bytes, | |
| 1471 | generic_result.stdout.?, | |
| 1472 | }); | |
| 1473 | } | |
| 1474 | }, | |
| 1475 | .expect_stdout_match => |match| { | |
| 1476 | if (mem.find(u8, generic_result.stdout.?, match) == null) { | |
| 1477 | return step.fail( | |
| 1478 | \\========= expected to find in stdout: ========= | |
| 1479 | \\{s} | |
| 1480 | \\========= but stdout does not contain it: ===== | |
| 1481 | \\{s} | |
| 1482 | , .{ | |
| 1483 | match, | |
| 1484 | generic_result.stdout.?, | |
| 1485 | }); | |
| 1486 | } | |
| 1487 | }, | |
| 1488 | .expect_term => |expected_term| { | |
| 1489 | if (!termMatches(expected_term, generic_result.term)) { | |
| 1490 | return step.fail("process {f} (expected {f})", .{ | |
| 1491 | fmtTerm(generic_result.term), | |
| 1492 | fmtTerm(expected_term), | |
| 1493 | }); | |
| 1494 | } | |
| 1495 | }, | |
| 1496 | }, | |
| 1497 | else => { | |
| 1498 | // On failure, report captured stderr like normal standard error output. | |
| 1499 | const bad_exit = switch (generic_result.term) { | |
| 1500 | .exited => |code| code != 0, | |
| 1501 | .signal, .stopped, .unknown => true, | |
| 1502 | }; | |
| 1503 | if (bad_exit) { | |
| 1504 | if (generic_result.stderr) |bytes| { | |
| 1505 | run.step.result_stderr = bytes; | |
| 1506 | } | |
| 1507 | } | |
| 1508 | ||
| 1509 | try step.handleChildProcessTerm(generic_result.term); | |
| 1510 | }, | |
| 1511 | } | |
| 1512 | } | |
| 1513 | ||
| 1514 | const EvalGenericResult = struct { | |
| 1515 | term: process.Child.Term, | |
| 1516 | stdout: ?[]const u8, | |
| 1517 | stderr: ?[]const u8, | |
| 1518 | }; | |
| 1519 | ||
| 1520 | fn spawnChildAndCollect( | |
| 1521 | run: *Run, | |
| 1522 | argv: []const []const u8, | |
| 1523 | environ_map: *EnvMap, | |
| 1524 | has_side_effects: bool, | |
| 1525 | options: Step.MakeOptions, | |
| 1526 | fuzz_context: ?FuzzContext, | |
| 1527 | ) !?EvalGenericResult { | |
| 1528 | const b = run.step.owner; | |
| 1529 | const graph = b.graph; | |
| 1530 | const io = graph.io; | |
| 1531 | ||
| 1532 | if (fuzz_context != null) { | |
| 1533 | assert(!has_side_effects); | |
| 1534 | assert(run.stdio == .zig_test); | |
| 1535 | } | |
| 1536 | ||
| 1537 | const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; | |
| 1538 | ||
| 1539 | // If an error occurs, it's caused by this command: | |
| 1540 | assert(run.step.result_failed_command == null); | |
| 1541 | run.step.result_failed_command = try Step.allocPrintCmd(options.gpa, child_cwd, .{ | |
| 1542 | .child = environ_map, | |
| 1543 | .parent = &graph.environ_map, | |
| 1544 | }, argv); | |
| 1545 | ||
| 1546 | var spawn_options: process.SpawnOptions = .{ | |
| 1547 | .argv = argv, | |
| 1548 | .cwd = child_cwd, | |
| 1549 | .environ_map = environ_map, | |
| 1550 | .request_resource_usage_statistics = true, | |
| 1551 | .stdin = if (run.stdin != .none) s: { | |
| 1552 | assert(run.stdio != .inherit); | |
| 1553 | break :s .pipe; | |
| 1554 | } else switch (run.stdio) { | |
| 1555 | .infer_from_args => if (has_side_effects) .inherit else .ignore, | |
| 1556 | .inherit => .inherit, | |
| 1557 | .check => .ignore, | |
| 1558 | .zig_test => .pipe, | |
| 1559 | }, | |
| 1560 | .stdout = if (run.captured_stdout != null) .pipe else switch (run.stdio) { | |
| 1561 | .infer_from_args => if (has_side_effects) .inherit else .ignore, | |
| 1562 | .inherit => .inherit, | |
| 1563 | .check => |checks| if (checksContainStdout(checks.items)) .pipe else .ignore, | |
| 1564 | .zig_test => .pipe, | |
| 1565 | }, | |
| 1566 | .stderr = if (run.captured_stderr != null) .pipe else switch (run.stdio) { | |
| 1567 | .infer_from_args => if (has_side_effects) .inherit else .pipe, | |
| 1568 | .inherit => .inherit, | |
| 1569 | .check => .pipe, | |
| 1570 | .zig_test => .pipe, | |
| 1571 | }, | |
| 1572 | }; | |
| 1573 | ||
| 1574 | if (run.stdio == .zig_test) { | |
| 1575 | const started: Io.Clock.Timestamp = .now(io, .awake); | |
| 1576 | const result = evalZigTest(run, spawn_options, options, fuzz_context) catch |err| switch (err) { | |
| 1577 | error.Canceled => |e| return e, | |
| 1578 | else => |e| e, | |
| 1579 | }; | |
| 1580 | run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); | |
| 1581 | try result; | |
| 1582 | return null; | |
| 1583 | } else { | |
| 1584 | const inherit = spawn_options.stdout == .inherit or spawn_options.stderr == .inherit; | |
| 1585 | if (!run.disable_zig_progress and !inherit) { | |
| 1586 | spawn_options.progress_node = options.progress_node; | |
| 1587 | } | |
| 1588 | const terminal_mode: Io.Terminal.Mode = if (inherit) m: { | |
| 1589 | const stderr = try io.lockStderr(&.{}, graph.stderr_mode); | |
| 1590 | break :m stderr.terminal_mode; | |
| 1591 | } else .no_color; | |
| 1592 | defer if (inherit) io.unlockStderr(); | |
| 1593 | try setColorEnvironmentVariables(run, environ_map, terminal_mode); | |
| 1594 | ||
| 1595 | const started: Io.Clock.Timestamp = .now(io, .awake); | |
| 1596 | const result = evalGeneric(run, spawn_options) catch |err| switch (err) { | |
| 1597 | error.Canceled => |e| return e, | |
| 1598 | else => |e| e, | |
| 1599 | }; | |
| 1600 | run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); | |
| 1601 | return try result; | |
| 1602 | } | |
| 1603 | } | |
| 1604 | ||
| 1605 | fn setColorEnvironmentVariables(run: *Run, environ_map: *EnvMap, terminal_mode: Io.Terminal.Mode) !void { | |
| 1606 | color: switch (run.color) { | |
| 1607 | .manual => {}, | |
| 1608 | .enable => { | |
| 1609 | try environ_map.put("CLICOLOR_FORCE", "1"); | |
| 1610 | _ = environ_map.swapRemove("NO_COLOR"); | |
| 1611 | }, | |
| 1612 | .disable => { | |
| 1613 | try environ_map.put("NO_COLOR", "1"); | |
| 1614 | _ = environ_map.swapRemove("CLICOLOR_FORCE"); | |
| 1615 | }, | |
| 1616 | .inherit => switch (terminal_mode) { | |
| 1617 | .no_color, .windows_api => continue :color .disable, | |
| 1618 | .escape_codes => continue :color .enable, | |
| 1619 | }, | |
| 1620 | .auto => { | |
| 1621 | const capture_stderr = run.captured_stderr != null or switch (run.stdio) { | |
| 1622 | .check => |checks| checksContainStderr(checks.items), | |
| 1623 | .infer_from_args, .inherit, .zig_test => false, | |
| 1624 | }; | |
| 1625 | if (capture_stderr) { | |
| 1626 | continue :color .disable; | |
| 1627 | } else { | |
| 1628 | continue :color .inherit; | |
| 1629 | } | |
| 1630 | }, | |
| 1631 | } | |
| 1632 | } | |
| 1633 | ||
| 1634 | const StdioPollEnum = enum { stdout, stderr }; | |
| 1635 | ||
| 1636 | fn evalZigTest( | |
| 1637 | run: *Run, | |
| 1638 | spawn_options: process.SpawnOptions, | |
| 1639 | options: Step.MakeOptions, | |
| 1640 | fuzz_context: ?FuzzContext, | |
| 1641 | ) !void { | |
| 1642 | if (fuzz_context != null) { | |
| 1643 | try evalFuzzTest(run, spawn_options, options, fuzz_context.?); | |
| 1644 | return; | |
| 1645 | } | |
| 1646 | ||
| 1647 | const step_owner = run.step.owner; | |
| 1648 | const gpa = step_owner.allocator; | |
| 1649 | const arena = step_owner.allocator; | |
| 1650 | const io = step_owner.graph.io; | |
| 1651 | ||
| 1652 | // We will update this every time a child runs. | |
| 1653 | run.step.result_peak_rss = 0; | |
| 1654 | ||
| 1655 | var test_results: Step.TestResults = .{ | |
| 1656 | .test_count = 0, | |
| 1657 | .skip_count = 0, | |
| 1658 | .fail_count = 0, | |
| 1659 | .crash_count = 0, | |
| 1660 | .timeout_count = 0, | |
| 1661 | .leak_count = 0, | |
| 1662 | .log_err_count = 0, | |
| 1663 | }; | |
| 1664 | var test_metadata: ?TestMetadata = null; | |
| 1665 | ||
| 1666 | while (true) { | |
| 1667 | var child = try process.spawn(io, spawn_options); | |
| 1668 | var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; | |
| 1669 | var multi_reader: Io.File.MultiReader = undefined; | |
| 1670 | multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); | |
| 1671 | var child_killed = false; | |
| 1672 | defer if (!child_killed) { | |
| 1673 | child.kill(io); | |
| 1674 | multi_reader.deinit(); | |
| 1675 | run.step.result_peak_rss = @max( | |
| 1676 | run.step.result_peak_rss, | |
| 1677 | child.resource_usage_statistics.getMaxRss() orelse 0, | |
| 1678 | ); | |
| 1679 | }; | |
| 1680 | ||
| 1681 | switch (try waitZigTest( | |
| 1682 | run, | |
| 1683 | &child, | |
| 1684 | options, | |
| 1685 | &multi_reader, | |
| 1686 | &test_metadata, | |
| 1687 | &test_results, | |
| 1688 | )) { | |
| 1689 | .write_failed => |err| { | |
| 1690 | // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured | |
| 1691 | // all available stderr to make our error output as useful as possible. | |
| 1692 | const stderr_fr = multi_reader.fileReader(1); | |
| 1693 | while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) { | |
| 1694 | error.ReadFailed => return stderr_fr.err.?, | |
| 1695 | error.EndOfStream => {}, | |
| 1696 | } | |
| 1697 | run.step.result_stderr = try arena.dupe(u8, stderr_fr.interface.buffered()); | |
| 1698 | ||
| 1699 | // Clean up everything and wait for the child to exit. | |
| 1700 | child.stdin.?.close(io); | |
| 1701 | child.stdin = null; | |
| 1702 | multi_reader.deinit(); | |
| 1703 | child_killed = true; | |
| 1704 | const term = try child.wait(io); | |
| 1705 | run.step.result_peak_rss = @max( | |
| 1706 | run.step.result_peak_rss, | |
| 1707 | child.resource_usage_statistics.getMaxRss() orelse 0, | |
| 1708 | ); | |
| 1709 | ||
| 1710 | // The individual unit test results are irrelevant: the test runner itself broke! | |
| 1711 | // Fail immediately without populating `s.test_results`. | |
| 1712 | return run.step.fail("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) }); | |
| 1713 | }, | |
| 1714 | .no_poll => |no_poll| { | |
| 1715 | // This might be a success (we requested exit and the child dutifully closed stdout) or | |
| 1716 | // a crash of some kind. Either way, the child will terminate by itself -- wait for it. | |
| 1717 | const stderr_reader = multi_reader.reader(1); | |
| 1718 | const stderr_owned = try arena.dupe(u8, stderr_reader.buffered()); | |
| 1719 | ||
| 1720 | // Clean up everything and wait for the child to exit. | |
| 1721 | child.stdin.?.close(io); | |
| 1722 | child.stdin = null; | |
| 1723 | multi_reader.deinit(); | |
| 1724 | child_killed = true; | |
| 1725 | const term = try child.wait(io); | |
| 1726 | run.step.result_peak_rss = @max( | |
| 1727 | run.step.result_peak_rss, | |
| 1728 | child.resource_usage_statistics.getMaxRss() orelse 0, | |
| 1729 | ); | |
| 1730 | ||
| 1731 | if (no_poll.active_test_index) |test_index| { | |
| 1732 | // A test was running, so this is definitely a crash. Report it against that | |
| 1733 | // test, and continue to the next test. | |
| 1734 | test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed; | |
| 1735 | test_results.crash_count += 1; | |
| 1736 | try run.step.addError("'{s}' {f}{s}{s}", .{ | |
| 1737 | test_metadata.?.testName(test_index), | |
| 1738 | fmtTerm(term), | |
| 1739 | if (stderr_owned.len != 0) " with stderr:\n" else "", | |
| 1740 | std.mem.trim(u8, stderr_owned, "\n"), | |
| 1741 | }); | |
| 1742 | continue; | |
| 1743 | } | |
| 1744 | ||
| 1745 | // Report an error if the child terminated uncleanly or if we were still trying to run more tests. | |
| 1746 | run.step.result_stderr = stderr_owned; | |
| 1747 | const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32); | |
| 1748 | if (!tests_done or !termMatches(.{ .exited = 0 }, term)) { | |
| 1749 | // The individual unit test results are irrelevant: the test runner itself broke! | |
| 1750 | // Fail immediately without populating `s.test_results`. | |
| 1751 | return run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); | |
| 1752 | } | |
| 1753 | ||
| 1754 | // We're done with all of the tests! Commit the test results and return. | |
| 1755 | run.step.test_results = test_results; | |
| 1756 | if (test_metadata) |tm| { | |
| 1757 | run.cached_test_metadata = tm.toCachedTestMetadata(); | |
| 1758 | if (options.web_server) |ws| { | |
| 1759 | if (run.step.owner.graph.time_report) { | |
| 1760 | ws.updateTimeReportRunTest( | |
| 1761 | run, | |
| 1762 | &run.cached_test_metadata.?, | |
| 1763 | tm.ns_per_test, | |
| 1764 | ); | |
| 1765 | } | |
| 1766 | } | |
| 1767 | } | |
| 1768 | return; | |
| 1769 | }, | |
| 1770 | .timeout => |timeout| { | |
| 1771 | const stderr_reader = multi_reader.reader(1); | |
| 1772 | const stderr = stderr_reader.buffered(); | |
| 1773 | stderr_reader.tossBuffered(); | |
| 1774 | if (timeout.active_test_index) |test_index| { | |
| 1775 | // A test was running. Report the timeout against that test, and continue on to | |
| 1776 | // the next test. | |
| 1777 | test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed; | |
| 1778 | test_results.timeout_count += 1; | |
| 1779 | try run.step.addError("'{s}' timed out after {f}{s}{s}", .{ | |
| 1780 | test_metadata.?.testName(test_index), | |
| 1781 | Io.Duration{ .nanoseconds = timeout.ns_elapsed }, | |
| 1782 | if (stderr.len != 0) " with stderr:\n" else "", | |
| 1783 | std.mem.trim(u8, stderr, "\n"), | |
| 1784 | }); | |
| 1785 | continue; | |
| 1786 | } | |
| 1787 | // Just log an error and let the child be killed. | |
| 1788 | run.step.result_stderr = try arena.dupe(u8, stderr); | |
| 1789 | // The individual unit test results in `results` are irrelevant: the test runner | |
| 1790 | // is broken! Fail immediately without populating `s.test_results`. | |
| 1791 | return run.step.fail("test runner failed to respond for {f}", .{Io.Duration{ .nanoseconds = timeout.ns_elapsed }}); | |
| 1792 | }, | |
| 1793 | } | |
| 1794 | comptime unreachable; | |
| 1795 | } | |
| 1796 | } | |
| 1797 | ||
| 1798 | /// Reads stdout of a Zig test process until a termination condition is reached: | |
| 1799 | /// * A write fails, indicating the child unexpectedly closed stdin | |
| 1800 | /// * A test (or a response from the test runner) times out | |
| 1801 | /// * The wait fails, indicating the child closed stdout and stderr | |
| 1802 | fn waitZigTest( | |
| 1803 | run: *Run, | |
| 1804 | child: *process.Child, | |
| 1805 | options: Step.MakeOptions, | |
| 1806 | multi_reader: *Io.File.MultiReader, | |
| 1807 | opt_metadata: *?TestMetadata, | |
| 1808 | results: *Step.TestResults, | |
| 1809 | ) !union(enum) { | |
| 1810 | write_failed: anyerror, | |
| 1811 | no_poll: struct { | |
| 1812 | active_test_index: ?u32, | |
| 1813 | ns_elapsed: u64, | |
| 1814 | }, | |
| 1815 | timeout: struct { | |
| 1816 | active_test_index: ?u32, | |
| 1817 | ns_elapsed: u64, | |
| 1818 | }, | |
| 1819 | } { | |
| 1820 | const gpa = run.step.owner.allocator; | |
| 1821 | const arena = run.step.owner.allocator; | |
| 1822 | const io = run.step.owner.graph.io; | |
| 1823 | ||
| 1824 | var sub_prog_node: ?std.Progress.Node = null; | |
| 1825 | defer if (sub_prog_node) |n| n.end(); | |
| 1826 | ||
| 1827 | if (opt_metadata.*) |*md| { | |
| 1828 | // Previous unit test process died or was killed; we're continuing where it left off | |
| 1829 | requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; | |
| 1830 | } else { | |
| 1831 | // Running unit tests normally | |
| 1832 | run.fuzz_tests.clearRetainingCapacity(); | |
| 1833 | sendMessage(io, child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err }; | |
| 1834 | } | |
| 1835 | ||
| 1836 | var active_test_index: ?u32 = null; | |
| 1837 | ||
| 1838 | var last_update: Io.Clock.Timestamp = .now(io, .awake); | |
| 1839 | ||
| 1840 | // This timeout is used when we're waiting on the test runner itself rather than a user-specified | |
| 1841 | // test. For instance, if the test runner leaves this much time between us requesting a test to | |
| 1842 | // start and it acknowledging the test starting, we terminate the child and raise an error. This | |
| 1843 | // *should* never happen, but could in theory be caused by some very unlucky IB in a test. | |
| 1844 | const response_timeout: Io.Clock.Duration = t: { | |
| 1845 | const ns = @max(options.unit_test_timeout_ns orelse 0, 60 * std.time.ns_per_s); | |
| 1846 | break :t .{ .clock = .awake, .raw = .fromNanoseconds(ns) }; | |
| 1847 | }; | |
| 1848 | const test_timeout: ?Io.Clock.Duration = if (options.unit_test_timeout_ns) |ns| .{ | |
| 1849 | .clock = .awake, | |
| 1850 | .raw = .fromNanoseconds(ns), | |
| 1851 | } else null; | |
| 1852 | ||
| 1853 | const stdout = multi_reader.reader(0); | |
| 1854 | const stderr = multi_reader.reader(1); | |
| 1855 | const Header = std.zig.Server.Message.Header; | |
| 1856 | ||
| 1857 | while (true) { | |
| 1858 | const timeout: Io.Timeout = t: { | |
| 1859 | const opt_duration = if (active_test_index == null) response_timeout else test_timeout; | |
| 1860 | const duration = opt_duration orelse break :t .none; | |
| 1861 | break :t .{ .deadline = last_update.addDuration(duration) }; | |
| 1862 | }; | |
| 1863 | ||
| 1864 | // This block is exited when `stdout` contains enough bytes for a `Header`. | |
| 1865 | header_ready: { | |
| 1866 | if (stdout.buffered().len >= @sizeOf(Header)) { | |
| 1867 | // We already have one, no need to poll! | |
| 1868 | break :header_ready; | |
| 1869 | } | |
| 1870 | ||
| 1871 | multi_reader.fill(64, timeout) catch |err| switch (err) { | |
| 1872 | error.Timeout => return .{ .timeout = .{ | |
| 1873 | .active_test_index = active_test_index, | |
| 1874 | .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), | |
| 1875 | } }, | |
| 1876 | error.EndOfStream => return .{ .no_poll = .{ | |
| 1877 | .active_test_index = active_test_index, | |
| 1878 | .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), | |
| 1879 | } }, | |
| 1880 | else => |e| return e, | |
| 1881 | }; | |
| 1882 | ||
| 1883 | continue; | |
| 1884 | } | |
| 1885 | // There is definitely a header available now -- read it. | |
| 1886 | const header = stdout.takeStruct(Header, .little) catch unreachable; | |
| 1887 | ||
| 1888 | while (stdout.buffered().len < header.bytes_len) { | |
| 1889 | multi_reader.fill(64, timeout) catch |err| switch (err) { | |
| 1890 | error.Timeout => return .{ .timeout = .{ | |
| 1891 | .active_test_index = active_test_index, | |
| 1892 | .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), | |
| 1893 | } }, | |
| 1894 | error.EndOfStream => return .{ .no_poll = .{ | |
| 1895 | .active_test_index = active_test_index, | |
| 1896 | .ns_elapsed = @intCast(last_update.untilNow(io).raw.nanoseconds), | |
| 1897 | } }, | |
| 1898 | else => |e| return e, | |
| 1899 | }; | |
| 1900 | } | |
| 1901 | ||
| 1902 | const body = stdout.take(header.bytes_len) catch unreachable; | |
| 1903 | var body_r: std.Io.Reader = .fixed(body); | |
| 1904 | switch (header.tag) { | |
| 1905 | .zig_version => { | |
| 1906 | if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail( | |
| 1907 | "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", | |
| 1908 | .{ builtin.zig_version_string, body }, | |
| 1909 | ); | |
| 1910 | }, | |
| 1911 | .test_metadata => { | |
| 1912 | // `metadata` would only be populated if we'd already seen a `test_metadata`, but we | |
| 1913 | // only request it once (and importantly, we don't re-request it if we kill and | |
| 1914 | // restart the test runner). | |
| 1915 | assert(opt_metadata.* == null); | |
| 1916 | ||
| 1917 | const tm_hdr = body_r.takeStruct(std.zig.Server.Message.TestMetadata, .little) catch unreachable; | |
| 1918 | results.test_count = tm_hdr.tests_len; | |
| 1919 | ||
| 1920 | const names = try arena.alloc(u32, results.test_count); | |
| 1921 | for (names) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; | |
| 1922 | ||
| 1923 | const expected_panic_msgs = try arena.alloc(u32, results.test_count); | |
| 1924 | for (expected_panic_msgs) |*dest| dest.* = body_r.takeInt(u32, .little) catch unreachable; | |
| 1925 | ||
| 1926 | const string_bytes = body_r.take(tm_hdr.string_bytes_len) catch unreachable; | |
| 1927 | ||
| 1928 | options.progress_node.setEstimatedTotalItems(names.len); | |
| 1929 | opt_metadata.* = .{ | |
| 1930 | .string_bytes = try arena.dupe(u8, string_bytes), | |
| 1931 | .ns_per_test = try arena.alloc(u64, results.test_count), | |
| 1932 | .names = names, | |
| 1933 | .expected_panic_msgs = expected_panic_msgs, | |
| 1934 | .next_index = 0, | |
| 1935 | .prog_node = options.progress_node, | |
| 1936 | }; | |
| 1937 | @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64)); | |
| 1938 | ||
| 1939 | active_test_index = null; | |
| 1940 | last_update = .now(io, .awake); | |
| 1941 | ||
| 1942 | requestNextTest(io, child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err }; | |
| 1943 | }, | |
| 1944 | .test_started => { | |
| 1945 | active_test_index = opt_metadata.*.?.next_index - 1; | |
| 1946 | last_update = .now(io, .awake); | |
| 1947 | }, | |
| 1948 | .test_results => { | |
| 1949 | const md = &opt_metadata.*.?; | |
| 1950 | ||
| 1951 | const tr_hdr = body_r.takeStruct(std.zig.Server.Message.TestResults, .little) catch unreachable; | |
| 1952 | assert(tr_hdr.index == active_test_index); | |
| 1953 | ||
| 1954 | switch (tr_hdr.flags.status) { | |
| 1955 | .pass => {}, | |
| 1956 | .skip => results.skip_count +|= 1, | |
| 1957 | .fail => results.fail_count +|= 1, | |
| 1958 | } | |
| 1959 | const leak_count = tr_hdr.flags.leak_count; | |
| 1960 | const log_err_count = tr_hdr.flags.log_err_count; | |
| 1961 | results.leak_count +|= leak_count; | |
| 1962 | results.log_err_count +|= log_err_count; | |
| 1963 | ||
| 1964 | if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index)); | |
| 1965 | ||
| 1966 | if (tr_hdr.flags.status == .fail) { | |
| 1967 | const name = md.testName(tr_hdr.index); | |
| 1968 | const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); | |
| 1969 | stderr.tossBuffered(); | |
| 1970 | if (stderr_bytes.len == 0) { | |
| 1971 | try run.step.addError("'{s}' failed without output", .{name}); | |
| 1972 | } else { | |
| 1973 | try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes }); | |
| 1974 | } | |
| 1975 | } else if (leak_count > 0) { | |
| 1976 | const name = md.testName(tr_hdr.index); | |
| 1977 | const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); | |
| 1978 | stderr.tossBuffered(); | |
| 1979 | try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes }); | |
| 1980 | } else if (log_err_count > 0) { | |
| 1981 | const name = md.testName(tr_hdr.index); | |
| 1982 | const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n"); | |
| 1983 | stderr.tossBuffered(); | |
| 1984 | try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes }); | |
| 1985 | } | |
| 1986 | ||
| 1987 | active_test_index = null; | |
| 1988 | ||
| 1989 | const now: Io.Clock.Timestamp = .now(io, .awake); | |
| 1990 | md.ns_per_test[tr_hdr.index] = @intCast(last_update.durationTo(now).raw.nanoseconds); | |
| 1991 | last_update = now; | |
| 1992 | ||
| 1993 | requestNextTest(io, child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err }; | |
| 1994 | }, | |
| 1995 | else => {}, // ignore other messages | |
| 1996 | } | |
| 1997 | } | |
| 1998 | } | |
| 1999 | ||
| 2000 | const FuzzTestRunner = struct { | |
| 2001 | run: *Run, | |
| 2002 | ctx: FuzzContext, | |
| 2003 | coverage_id: ?u64, | |
| 2004 | ||
| 2005 | instances: []Instance, | |
| 2006 | /// The indexes of this are layed out such that it is effectively an array | |
| 2007 | /// of `[instances.len][3]Io.Operation.Storage` of stdin, stdout, stderr. | |
| 2008 | batch: Io.Batch, | |
| 2009 | /// LIFO. Stream of message bodies trailed by PendingBroadcastFooter. | |
| 2010 | pending_broadcasts: std.ArrayList(u8), | |
| 2011 | broadcast: std.ArrayList(u8), | |
| 2012 | broadcast_undelivered: u32, | |
| 2013 | ||
| 2014 | const Instance = struct { | |
| 2015 | child: process.Child, | |
| 2016 | message: std.ArrayListAligned(u8, .@"4"), | |
| 2017 | broadcast_written: usize, | |
| 2018 | stderr: std.ArrayList(u8), | |
| 2019 | stdin_vec: [1][]u8, | |
| 2020 | stdout_vec: [1][]u8, | |
| 2021 | stderr_vec: [1][]u8, | |
| 2022 | progress_node: std.Progress.Node, | |
| 2023 | ||
| 2024 | fn messageHeader(instance: *Instance) InHeader { | |
| 2025 | assert(instance.message.items.len >= @sizeOf(InHeader)); | |
| 2026 | const header_ptr: *InHeader = @ptrCast(instance.message.items); | |
| 2027 | var header = header_ptr.*; | |
| 2028 | if (std.builtin.Endian.native != .little) { | |
| 2029 | std.mem.byteSwapAllFields(InHeader, &header); | |
| 2030 | } | |
| 2031 | return header; | |
| 2032 | } | |
| 2033 | }; | |
| 2034 | ||
| 2035 | const PendingBroadcastFooter = struct { | |
| 2036 | from_id: u32, | |
| 2037 | body_len: u32, | |
| 2038 | }; | |
| 2039 | ||
| 2040 | const InHeader = std.zig.Server.Message.Header; | |
| 2041 | const OutHeader = std.zig.Client.Message.Header; | |
| 2042 | ||
| 2043 | const stdin_i = 0; | |
| 2044 | const stdout_i = 1; | |
| 2045 | const stderr_i = 2; | |
| 2046 | ||
| 2047 | fn init( | |
| 2048 | run: *Run, | |
| 2049 | ctx: FuzzContext, | |
| 2050 | progress_node: std.Progress.Node, | |
| 2051 | spawn_options: process.SpawnOptions, | |
| 2052 | ) !FuzzTestRunner { | |
| 2053 | const step_owner = run.step.owner; | |
| 2054 | const gpa = step_owner.allocator; | |
| 2055 | const io = step_owner.graph.io; | |
| 2056 | ||
| 2057 | const n_instances = switch (ctx.fuzz.mode) { | |
| 2058 | .forever => step_owner.graph.max_jobs orelse @min( | |
| 2059 | std.Thread.getCpuCount() catch 1, | |
| 2060 | (std.math.maxInt(u32) - 2) / 3, | |
| 2061 | ), | |
| 2062 | .limit => 1, | |
| 2063 | }; | |
| 2064 | const instances = try gpa.alloc(Instance, n_instances); | |
| 2065 | errdefer gpa.free(instances); | |
| 2066 | const batch_storage = try gpa.alloc(Io.Operation.Storage, instances.len * 3); | |
| 2067 | errdefer gpa.free(batch_storage); | |
| 2068 | ||
| 2069 | @memset(instances, .{ | |
| 2070 | .child = undefined, | |
| 2071 | .message = .empty, | |
| 2072 | .broadcast_written = undefined, | |
| 2073 | .stderr = .empty, | |
| 2074 | .stdin_vec = undefined, | |
| 2075 | .stdout_vec = undefined, | |
| 2076 | .stderr_vec = undefined, | |
| 2077 | .progress_node = undefined, | |
| 2078 | }); | |
| 2079 | for (0.., instances) |id, *instance| { | |
| 2080 | errdefer for (instances[0..id]) |*spawned| { | |
| 2081 | spawned.child.kill(io); | |
| 2082 | spawned.progress_node.end(); | |
| 2083 | }; | |
| 2084 | instance.child = try process.spawn(io, spawn_options); | |
| 2085 | instance.progress_node = progress_node.start("starting fuzzer", 0); | |
| 2086 | } | |
| 2087 | ||
| 2088 | return .{ | |
| 2089 | .run = run, | |
| 2090 | .ctx = ctx, | |
| 2091 | .coverage_id = null, | |
| 2092 | ||
| 2093 | .instances = instances, | |
| 2094 | .batch = .init(batch_storage), | |
| 2095 | .pending_broadcasts = .empty, | |
| 2096 | .broadcast = .empty, | |
| 2097 | .broadcast_undelivered = 0, | |
| 2098 | }; | |
| 2099 | } | |
| 2100 | ||
| 2101 | fn deinit(f: *FuzzTestRunner) void { | |
| 2102 | const step_owner = f.run.step.owner; | |
| 2103 | const gpa = step_owner.allocator; | |
| 2104 | const io = step_owner.graph.io; | |
| 2105 | ||
| 2106 | f.batch.cancel(io); | |
| 2107 | gpa.free(f.batch.storage); | |
| 2108 | var total_rss: usize = 0; | |
| 2109 | for (f.instances) |*instance| { | |
| 2110 | instance.child.kill(io); | |
| 2111 | instance.message.deinit(gpa); | |
| 2112 | instance.stderr.deinit(gpa); | |
| 2113 | instance.progress_node.end(); | |
| 2114 | total_rss += instance.child.resource_usage_statistics.getMaxRss() orelse 0; | |
| 2115 | } | |
| 2116 | f.run.step.result_peak_rss = @max(f.run.step.result_peak_rss, total_rss); | |
| 2117 | gpa.free(f.instances); | |
| 2118 | } | |
| 2119 | ||
| 2120 | fn startInstances(f: *FuzzTestRunner) !void { | |
| 2121 | const step_owner = f.run.step.owner; | |
| 2122 | const io = step_owner.graph.io; | |
| 2123 | ||
| 2124 | for (0.., f.instances) |id, *instance| { | |
| 2125 | const id32: u32 = @intCast(id); | |
| 2126 | (switch (f.ctx.fuzz.mode) { | |
| 2127 | .forever => sendRunFuzzTestMessage( | |
| 2128 | io, | |
| 2129 | instance.child.stdin.?, | |
| 2130 | f.run.fuzz_tests.items, | |
| 2131 | .forever, | |
| 2132 | id32, | |
| 2133 | ), | |
| 2134 | .limit => |limit| sendRunFuzzTestMessage( | |
| 2135 | io, | |
| 2136 | instance.child.stdin.?, | |
| 2137 | f.run.fuzz_tests.items, | |
| 2138 | .iterations, | |
| 2139 | limit.amount, | |
| 2140 | ), | |
| 2141 | }) catch |write_err| { | |
| 2142 | // The runner unexpectedly closed stdin, which means it crashed during initialization. | |
| 2143 | // Clean up everything and wait for the child to exit. | |
| 2144 | instance.child.stdin.?.close(io); | |
| 2145 | instance.child.stdin = null; | |
| 2146 | const term = try instance.child.wait(io); | |
| 2147 | return f.run.step.fail( | |
| 2148 | "unable to write stdin ({t}); test process unexpectedly {f}", | |
| 2149 | .{ write_err, fmtTerm(term) }, | |
| 2150 | ); | |
| 2151 | }; | |
| 2152 | ||
| 2153 | try f.addStdoutRead(id32, @sizeOf(InHeader)); | |
| 2154 | try f.addStderrRead(id32); | |
| 2155 | } | |
| 2156 | } | |
| 2157 | ||
| 2158 | fn listen(f: *FuzzTestRunner) !void { | |
| 2159 | const step_owner = f.run.step.owner; | |
| 2160 | const io = step_owner.graph.io; | |
| 2161 | ||
| 2162 | while (true) { | |
| 2163 | try f.batch.awaitConcurrent(io, .none); | |
| 2164 | while (f.batch.next()) |completion| { | |
| 2165 | const id = completion.index / 3; | |
| 2166 | const result = completion.result; | |
| 2167 | switch (completion.index % 3) { | |
| 2168 | 0 => try f.completeStdinWrite(id, result.file_write_streaming catch |e| switch (e) { | |
| 2169 | // Avoid calling `instanceEos` until EndOfStream is seen with stderr so | |
| 2170 | // that all stderr is collected. | |
| 2171 | error.BrokenPipe => continue, | |
| 2172 | else => |write_e| return write_e, | |
| 2173 | }), | |
| 2174 | 1 => try f.completeStdoutRead(id, result.file_read_streaming catch |e| switch (e) { | |
| 2175 | // Avoid calling `instanceEos` until EndOfStream is seen with stderr so | |
| 2176 | // that all stderr is collected. | |
| 2177 | error.EndOfStream => continue, | |
| 2178 | else => |read_e| return read_e, | |
| 2179 | }), | |
| 2180 | 2 => try f.completeStderrRead(id, result.file_read_streaming catch |e| switch (e) { | |
| 2181 | error.EndOfStream => return f.instanceEos(id), | |
| 2182 | else => |read_e| return read_e, | |
| 2183 | }), | |
| 2184 | else => unreachable, | |
| 2185 | } | |
| 2186 | } | |
| 2187 | } | |
| 2188 | } | |
| 2189 | ||
| 2190 | fn completeStdoutRead(f: *FuzzTestRunner, id: u32, n: usize) !void { | |
| 2191 | const step_owner = f.run.step.owner; | |
| 2192 | const gpa = step_owner.allocator; | |
| 2193 | const io = step_owner.graph.io; | |
| 2194 | const instance = &f.instances[id]; | |
| 2195 | ||
| 2196 | instance.message.items.len += n; | |
| 2197 | const total_read = instance.message.items.len; | |
| 2198 | if (total_read < @sizeOf(InHeader)) { | |
| 2199 | try f.addStdoutRead(id, @sizeOf(InHeader)); | |
| 2200 | return; | |
| 2201 | } | |
| 2202 | ||
| 2203 | const header = instance.messageHeader(); | |
| 2204 | const body = instance.message.items[@sizeOf(InHeader)..]; | |
| 2205 | if (body.len != header.bytes_len) { | |
| 2206 | try f.addStdoutRead(id, @sizeOf(InHeader) + header.bytes_len); | |
| 2207 | return; | |
| 2208 | } | |
| 2209 | ||
| 2210 | switch (header.tag) { | |
| 2211 | .zig_version => { | |
| 2212 | if (!std.mem.eql(u8, builtin.zig_version_string, body)) return f.run.step.fail( | |
| 2213 | "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", | |
| 2214 | .{ builtin.zig_version_string, body }, | |
| 2215 | ); | |
| 2216 | }, | |
| 2217 | .coverage_id => { | |
| 2218 | var body_r: Io.Reader = .fixed(body); | |
| 2219 | f.coverage_id = body_r.takeInt(u64, .little) catch unreachable; | |
| 2220 | const cumulative_runs = body_r.takeInt(u64, .little) catch unreachable; | |
| 2221 | const cumulative_unique = body_r.takeInt(u64, .little) catch unreachable; | |
| 2222 | const cumulative_coverage = body_r.takeInt(u64, .little) catch unreachable; | |
| 2223 | ||
| 2224 | const fuzz = f.ctx.fuzz; | |
| 2225 | fuzz.queue_mutex.lockUncancelable(io); | |
| 2226 | defer fuzz.queue_mutex.unlock(io); | |
| 2227 | try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{ | |
| 2228 | .id = f.coverage_id.?, | |
| 2229 | .cumulative = .{ | |
| 2230 | .runs = cumulative_runs, | |
| 2231 | .unique = cumulative_unique, | |
| 2232 | .coverage = cumulative_coverage, | |
| 2233 | }, | |
| 2234 | .run = f.run, | |
| 2235 | } }); | |
| 2236 | fuzz.queue_cond.signal(io); | |
| 2237 | }, | |
| 2238 | .fuzz_start_addr => { | |
| 2239 | var body_r: Io.Reader = .fixed(body); | |
| 2240 | const fuzz = f.ctx.fuzz; | |
| 2241 | const addr = body_r.takeInt(u64, .little) catch unreachable; | |
| 2242 | ||
| 2243 | fuzz.queue_mutex.lockUncancelable(io); | |
| 2244 | defer fuzz.queue_mutex.unlock(io); | |
| 2245 | try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{ | |
| 2246 | .addr = addr, | |
| 2247 | .coverage_id = f.coverage_id.?, | |
| 2248 | } }); | |
| 2249 | fuzz.queue_cond.signal(io); | |
| 2250 | }, | |
| 2251 | .fuzz_test_change => { | |
| 2252 | const test_i = std.mem.readInt(u32, body[0..4], .little); | |
| 2253 | instance.progress_node.setName(f.run.fuzz_tests.items[test_i]); | |
| 2254 | }, | |
| 2255 | .broadcast_fuzz_input => { | |
| 2256 | if (f.instances.len == 1) { | |
| 2257 | // No other processes to broadcast to. | |
| 2258 | } else if (f.broadcast_undelivered == 0) { | |
| 2259 | try f.instanceBroadcast(id, body); | |
| 2260 | } else { | |
| 2261 | const footer: PendingBroadcastFooter = .{ | |
| 2262 | .from_id = id, | |
| 2263 | .body_len = @intCast(body.len), | |
| 2264 | }; | |
| 2265 | // There is another broadcast in progress so add this one to the queue. | |
| 2266 | const size = @sizeOf(PendingBroadcastFooter) + body.len; | |
| 2267 | try f.pending_broadcasts.ensureUnusedCapacity(gpa, size); | |
| 2268 | f.pending_broadcasts.appendSliceAssumeCapacity(body); | |
| 2269 | f.pending_broadcasts.appendSliceAssumeCapacity(@ptrCast(&footer)); | |
| 2270 | } | |
| 2271 | }, | |
| 2272 | else => {}, // ignore other messages | |
| 2273 | } | |
| 2274 | ||
| 2275 | instance.message.clearRetainingCapacity(); | |
| 2276 | try f.addStdoutRead(id, @sizeOf(InHeader)); | |
| 2277 | } | |
| 2278 | ||
| 2279 | fn completeStderrRead(f: *FuzzTestRunner, id: u32, n: usize) !void { | |
| 2280 | const instance = &f.instances[id]; | |
| 2281 | instance.stderr.items.len += n; | |
| 2282 | try f.addStderrRead(id); | |
| 2283 | } | |
| 2284 | ||
| 2285 | fn completeStdinWrite(f: *FuzzTestRunner, id: u32, n: usize) !void { | |
| 2286 | const instance = &f.instances[id]; | |
| 2287 | ||
| 2288 | instance.broadcast_written += n; | |
| 2289 | if (instance.broadcast_written == f.broadcast.items.len) { | |
| 2290 | f.broadcast_undelivered -= 1; | |
| 2291 | if (f.broadcast_undelivered == 0) { | |
| 2292 | try f.broadcastComplete(); | |
| 2293 | } | |
| 2294 | } else { | |
| 2295 | f.addStdinWrite(id); | |
| 2296 | } | |
| 2297 | } | |
| 2298 | ||
| 2299 | fn addStdoutRead(f: *FuzzTestRunner, id: u32, end: usize) !void { | |
| 2300 | const step_owner = f.run.step.owner; | |
| 2301 | const gpa = step_owner.allocator; | |
| 2302 | const instance = &f.instances[id]; | |
| 2303 | ||
| 2304 | try instance.message.ensureTotalCapacity(gpa, end); | |
| 2305 | const start = instance.message.items.len; | |
| 2306 | instance.stdout_vec = .{instance.message.allocatedSlice()[start..end]}; | |
| 2307 | f.batch.addAt(id * 3 + stdout_i, .{ .file_read_streaming = .{ | |
| 2308 | .file = instance.child.stdout.?, | |
| 2309 | .data = &instance.stdout_vec, | |
| 2310 | } }); | |
| 2311 | } | |
| 2312 | ||
| 2313 | fn addStderrRead(f: *FuzzTestRunner, id: u32) !void { | |
| 2314 | const step_owner = f.run.step.owner; | |
| 2315 | const gpa = step_owner.allocator; | |
| 2316 | const instance = &f.instances[id]; | |
| 2317 | ||
| 2318 | try instance.stderr.ensureUnusedCapacity(gpa, 1); | |
| 2319 | instance.stderr_vec = .{instance.stderr.unusedCapacitySlice()}; | |
| 2320 | f.batch.addAt(id * 3 + stderr_i, .{ .file_read_streaming = .{ | |
| 2321 | .file = instance.child.stderr.?, | |
| 2322 | .data = &instance.stderr_vec, | |
| 2323 | } }); | |
| 2324 | } | |
| 2325 | ||
| 2326 | fn addStdinWrite(f: *FuzzTestRunner, id: u32) void { | |
| 2327 | const instance = &f.instances[id]; | |
| 2328 | ||
| 2329 | assert(f.broadcast.items.len != instance.broadcast_written); | |
| 2330 | instance.stdin_vec = .{f.broadcast.items[instance.broadcast_written..]}; | |
| 2331 | f.batch.addAt(id * 3 + stdin_i, .{ .file_write_streaming = .{ | |
| 2332 | .file = instance.child.stdin.?, | |
| 2333 | .data = &instance.stdin_vec, | |
| 2334 | } }); | |
| 2335 | } | |
| 2336 | ||
| 2337 | fn instanceEos(f: *FuzzTestRunner, id: u32) !void { | |
| 2338 | const step_owner = f.run.step.owner; | |
| 2339 | const io = step_owner.graph.io; | |
| 2340 | const instance = &f.instances[id]; | |
| 2341 | ||
| 2342 | instance.child.stdin.?.close(io); | |
| 2343 | instance.child.stdin = null; | |
| 2344 | const term = try instance.child.wait(io); | |
| 2345 | if (!termMatches(.{ .exited = 0 }, term)) { | |
| 2346 | f.run.step.result_stderr = try f.mergedStderr(); | |
| 2347 | try f.saveCrash(id, term); | |
| 2348 | return f.run.step.fail("test process unexpectedly {f}", .{fmtTerm(term)}); | |
| 2349 | } | |
| 2350 | } | |
| 2351 | ||
| 2352 | fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { | |
| 2353 | const step = &f.run.step; | |
| 2354 | const b = step.owner; | |
| 2355 | const io = b.graph.io; | |
| 2356 | ||
| 2357 | if (f.coverage_id == null) return; | |
| 2358 | ||
| 2359 | // Search for the input file corresponding to the instance | |
| 2360 | const InputHeader = Build.abi.fuzz.MmapInputHeader; | |
| 2361 | var in_r_buf: [@sizeOf(InputHeader)]u8 = undefined; | |
| 2362 | var in_r: Io.File.Reader = undefined; | |
| 2363 | var in_f: Io.File = undefined; | |
| 2364 | var in_name_buf: [12]u8 = undefined; | |
| 2365 | var in_name: []const u8 = undefined; | |
| 2366 | var i: u32 = 0; | |
| 2367 | const header: InputHeader = while (true) : ({ | |
| 2368 | if (i == std.math.maxInt(u32)) return; | |
| 2369 | i += 1; | |
| 2370 | }) { | |
| 2371 | const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; | |
| 2372 | in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; | |
| 2373 | in_f = b.cache_root.handle.openFile(io, in_name, .{ | |
| 2374 | .lock = .exclusive, | |
| 2375 | .lock_nonblocking = true, | |
| 2376 | }) catch |e| switch (e) { | |
| 2377 | error.FileNotFound => return, | |
| 2378 | error.WouldBlock => continue, // Can not be from | |
| 2379 | // the crashed instance since it is still locked. | |
| 2380 | else => return step.fail("failed to open file '{f}{s}': {t}", .{ | |
| 2381 | b.cache_root, in_name, e, | |
| 2382 | }), | |
| 2383 | }; | |
| 2384 | ||
| 2385 | in_r = in_f.readerStreaming(io, &in_r_buf); | |
| 2386 | const header = in_r.interface.takeStruct(InputHeader, .little) catch |e| { | |
| 2387 | in_f.close(io); | |
| 2388 | switch (e) { | |
| 2389 | error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ | |
| 2390 | b.cache_root, in_name, in_r.err.?, | |
| 2391 | }), | |
| 2392 | error.EndOfStream => continue, | |
| 2393 | } | |
| 2394 | }; | |
| 2395 | ||
| 2396 | if (header.pc_digest == f.coverage_id.? and | |
| 2397 | header.instance_id == id and | |
| 2398 | header.test_i < f.run.fuzz_tests.items.len) | |
| 2399 | { | |
| 2400 | break header; | |
| 2401 | } | |
| 2402 | ||
| 2403 | in_f.close(io); | |
| 2404 | }; | |
| 2405 | defer in_f.close(io); | |
| 2406 | ||
| 2407 | // Save it to a seperate file | |
| 2408 | const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; | |
| 2409 | const out = b.cache_root.handle.createFile(io, crash_name, .{ | |
| 2410 | .lock = .exclusive, // Multiple run steps could have found a crash at the same time | |
| 2411 | }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ | |
| 2412 | b.cache_root, crash_name, e, | |
| 2413 | }); | |
| 2414 | defer out.close(io); | |
| 2415 | ||
| 2416 | var out_w_buf: [512]u8 = undefined; | |
| 2417 | var out_w = out.writerStreaming(io, &out_w_buf); | |
| 2418 | _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { | |
| 2419 | error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ | |
| 2420 | b.cache_root, in_name, in_r.err.?, | |
| 2421 | }), | |
| 2422 | error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ | |
| 2423 | b.cache_root, crash_name, out_w.err.?, | |
| 2424 | }), | |
| 2425 | }; | |
| 2426 | ||
| 2427 | return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ | |
| 2428 | f.run.fuzz_tests.items[header.test_i], | |
| 2429 | fmtTerm(term), | |
| 2430 | b.cache_root, | |
| 2431 | crash_name, | |
| 2432 | }); | |
| 2433 | } | |
| 2434 | ||
| 2435 | fn instanceBroadcast(f: *FuzzTestRunner, from_id: u32, bytes: []const u8) !void { | |
| 2436 | assert(f.instances.len > 1); | |
| 2437 | assert(f.broadcast_undelivered == 0); // no other broadcast is progress | |
| 2438 | assert(f.broadcast.items.len == 0); | |
| 2439 | assert(from_id < f.instances.len); | |
| 2440 | ||
| 2441 | const step_owner = f.run.step.owner; | |
| 2442 | const gpa = step_owner.allocator; | |
| 2443 | ||
| 2444 | var out_header: OutHeader = .{ | |
| 2445 | .tag = .new_fuzz_input, | |
| 2446 | .bytes_len = @intCast(bytes.len), | |
| 2447 | }; | |
| 2448 | if (std.builtin.Endian.native != .little) { | |
| 2449 | std.mem.byteSwapAllFields(OutHeader, &out_header); | |
| 2450 | } | |
| 2451 | try f.broadcast.ensureTotalCapacity(gpa, @sizeOf(OutHeader) + bytes.len); | |
| 2452 | f.broadcast.appendSliceAssumeCapacity(@ptrCast(&out_header)); | |
| 2453 | f.broadcast.appendSliceAssumeCapacity(bytes); | |
| 2454 | ||
| 2455 | f.broadcast_undelivered = @intCast(f.instances.len - 1); | |
| 2456 | for (0.., f.instances) |to_id, *instance| { | |
| 2457 | if (to_id == from_id) continue; | |
| 2458 | instance.broadcast_written = 0; | |
| 2459 | f.addStdinWrite(@intCast(to_id)); | |
| 2460 | } | |
| 2461 | } | |
| 2462 | ||
| 2463 | fn broadcastComplete(f: *FuzzTestRunner) !void { | |
| 2464 | assert(f.instances.len > 1); | |
| 2465 | assert(f.broadcast_undelivered == 0); | |
| 2466 | f.broadcast.clearRetainingCapacity(); | |
| 2467 | ||
| 2468 | const pending = &f.pending_broadcasts; | |
| 2469 | if (pending.items.len != 0) { | |
| 2470 | // Another broadcast is pending; copy it over to `broadcast` | |
| 2471 | ||
| 2472 | const footer_len = @sizeOf(PendingBroadcastFooter); | |
| 2473 | const footer_bytes = pending.items[pending.items.len - footer_len ..]; | |
| 2474 | const footer: *align(1) PendingBroadcastFooter = @ptrCast(footer_bytes); | |
| 2475 | pending.items.len -= footer_len; | |
| 2476 | ||
| 2477 | const body = pending.items[pending.items.len - footer.body_len ..]; | |
| 2478 | try f.instanceBroadcast(footer.from_id, body); | |
| 2479 | pending.items.len -= body.len; | |
| 2480 | } | |
| 2481 | } | |
| 2482 | ||
| 2483 | fn mergedStderr(f: *FuzzTestRunner) std.mem.Allocator.Error![]const u8 { | |
| 2484 | const step_owner = f.run.step.owner; | |
| 2485 | const arena = step_owner.allocator; | |
| 2486 | ||
| 2487 | // Collect any available stderr | |
| 2488 | while (f.batch.next()) |completion| { | |
| 2489 | if (completion.index % 3 != 2) continue; | |
| 2490 | const len = completion.result.file_read_streaming catch continue; | |
| 2491 | f.instances[completion.index / 3].stderr.items.len += len; | |
| 2492 | } | |
| 2493 | ||
| 2494 | var stderr_len: usize = 0; | |
| 2495 | for (f.instances) |*instance| stderr_len += instance.stderr.items.len; | |
| 2496 | const stderr = try arena.alloc(u8, stderr_len); | |
| 2497 | ||
| 2498 | stderr_len = 0; | |
| 2499 | for (f.instances) |*instance| { | |
| 2500 | @memcpy(stderr[stderr_len..][0..instance.stderr.items.len], instance.stderr.items); | |
| 2501 | stderr_len += instance.stderr.items.len; | |
| 2502 | } | |
| 2503 | return stderr; | |
| 2504 | } | |
| 2505 | }; | |
| 2506 | ||
| 2507 | fn evalFuzzTest( | |
| 2508 | run: *Run, | |
| 2509 | spawn_options: process.SpawnOptions, | |
| 2510 | options: Step.MakeOptions, | |
| 2511 | fuzz_context: FuzzContext, | |
| 2512 | ) !void { | |
| 2513 | var f: FuzzTestRunner = try .init(run, fuzz_context, options.progress_node, spawn_options); | |
| 2514 | defer f.deinit(); | |
| 2515 | try f.startInstances(); | |
| 2516 | try f.listen(); | |
| 2517 | } | |
| 2518 | ||
| 2519 | const TestMetadata = struct { | |
| 2520 | names: []const u32, | |
| 2521 | ns_per_test: []u64, | |
| 2522 | expected_panic_msgs: []const u32, | |
| 2523 | string_bytes: []const u8, | |
| 2524 | next_index: u32, | |
| 2525 | prog_node: std.Progress.Node, | |
| 2526 | ||
| 2527 | fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { | |
| 2528 | return .{ | |
| 2529 | .names = tm.names, | |
| 2530 | .string_bytes = tm.string_bytes, | |
| 2531 | }; | |
| 2532 | } | |
| 2533 | ||
| 2534 | fn testName(tm: TestMetadata, index: u32) []const u8 { | |
| 2535 | return tm.toCachedTestMetadata().testName(index); | |
| 2536 | } | |
| 2537 | }; | |
| 2538 | ||
| 2539 | pub const CachedTestMetadata = struct { | |
| 2540 | names: []const u32, | |
| 2541 | string_bytes: []const u8, | |
| 2542 | ||
| 2543 | pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { | |
| 2544 | return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); | |
| 2545 | } | |
| 2546 | }; | |
| 2547 | ||
| 2548 | fn requestNextTest(io: Io, in: Io.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { | |
| 2549 | while (metadata.next_index < metadata.names.len) { | |
| 2550 | const i = metadata.next_index; | |
| 2551 | metadata.next_index += 1; | |
| 2552 | ||
| 2553 | if (metadata.expected_panic_msgs[i] != 0) continue; | |
| 2554 | ||
| 2555 | const name = metadata.testName(i); | |
| 2556 | if (sub_prog_node.*) |n| n.end(); | |
| 2557 | sub_prog_node.* = metadata.prog_node.start(name, 0); | |
| 2558 | ||
| 2559 | try sendRunTestMessage(io, in, .run_test, i); | |
| 2560 | return; | |
| 2561 | } else { | |
| 2562 | metadata.next_index = std.math.maxInt(u32); // indicate that all tests are done | |
| 2563 | try sendMessage(io, in, .exit); | |
| 2564 | } | |
| 2565 | } | |
| 2566 | ||
| 2567 | fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { | |
| 2568 | const header: std.zig.Client.Message.Header = .{ | |
| 2569 | .tag = tag, | |
| 2570 | .bytes_len = 0, | |
| 2571 | }; | |
| 2572 | var w = file.writerStreaming(io, &.{}); | |
| 2573 | w.interface.writeStruct(header, .little) catch |err| switch (err) { | |
| 2574 | error.WriteFailed => return w.err.?, | |
| 2575 | }; | |
| 2576 | } | |
| 2577 | ||
| 2578 | fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, index: u32) !void { | |
| 2579 | const header: std.zig.Client.Message.Header = .{ | |
| 2580 | .tag = tag, | |
| 2581 | .bytes_len = 4, | |
| 2582 | }; | |
| 2583 | var w = file.writerStreaming(io, &.{}); | |
| 2584 | w.interface.writeStruct(header, .little) catch |err| switch (err) { | |
| 2585 | error.WriteFailed => return w.err.?, | |
| 2586 | }; | |
| 2587 | w.interface.writeInt(u32, index, .little) catch |err| switch (err) { | |
| 2588 | error.WriteFailed => return w.err.?, | |
| 2589 | }; | |
| 2590 | } | |
| 2591 | ||
| 2592 | fn sendRunFuzzTestMessage( | |
| 2593 | io: Io, | |
| 2594 | file: Io.File, | |
| 2595 | test_names: []const []const u8, | |
| 2596 | kind: std.Build.abi.fuzz.LimitKind, | |
| 2597 | amount_or_instance: u64, | |
| 2598 | ) !void { | |
| 2599 | const header: std.zig.Client.Message.Header = .{ | |
| 2600 | .tag = .start_fuzzing, | |
| 2601 | .bytes_len = 1 + 8 + 4 + count: { | |
| 2602 | var c: u32 = @intCast(test_names.len * 4); | |
| 2603 | for (test_names) |name| { | |
| 2604 | c += @intCast(name.len); | |
| 2605 | } | |
| 2606 | break :count c; | |
| 2607 | }, | |
| 2608 | }; | |
| 2609 | var w = file.writerStreaming(io, &.{}); | |
| 2610 | w.interface.writeStruct(header, .little) catch |err| switch (err) { | |
| 2611 | error.WriteFailed => return w.err.?, | |
| 2612 | }; | |
| 2613 | w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) { | |
| 2614 | error.WriteFailed => return w.err.?, | |
| 2615 | }; | |
| 2616 | w.interface.writeInt(u64, amount_or_instance, .little) catch |err| switch (err) { | |
| 2617 | error.WriteFailed => return w.err.?, | |
| 2618 | }; | |
| 2619 | w.interface.writeInt(u32, @intCast(test_names.len), .little) catch |err| switch (err) { | |
| 2620 | error.WriteFailed => return w.err.?, | |
| 2621 | }; | |
| 2622 | for (test_names) |test_name| { | |
| 2623 | w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) { | |
| 2624 | error.WriteFailed => return w.err.?, | |
| 2625 | }; | |
| 2626 | w.interface.writeAll(test_name) catch |err| switch (err) { | |
| 2627 | error.WriteFailed => return w.err.?, | |
| 2628 | }; | |
| 2629 | } | |
| 2630 | } | |
| 2631 | ||
| 2632 | fn evalGeneric(run: *Run, spawn_options: process.SpawnOptions) !EvalGenericResult { | |
| 2633 | const b = run.step.owner; | |
| 2634 | const io = b.graph.io; | |
| 2635 | const arena = b.allocator; | |
| 2636 | const gpa = b.allocator; | |
| 2637 | ||
| 2638 | var child = try process.spawn(io, spawn_options); | |
| 2639 | defer child.kill(io); | |
| 2640 | ||
| 2641 | switch (run.stdin) { | |
| 2642 | .bytes => |bytes| { | |
| 2643 | child.stdin.?.writeStreamingAll(io, bytes) catch |err| { | |
| 2644 | return run.step.fail("unable to write stdin: {t}", .{err}); | |
| 2645 | }; | |
| 2646 | child.stdin.?.close(io); | |
| 2647 | child.stdin = null; | |
| 2648 | }, | |
| 2649 | .lazy_path => |lazy_path| { | |
| 2650 | const path = lazy_path.getPath3(b, &run.step); | |
| 2651 | const file = path.root_dir.handle.openFile(io, path.subPathOrDot(), .{}) catch |err| { | |
| 2652 | return run.step.fail("unable to open stdin file: {t}", .{err}); | |
| 2653 | }; | |
| 2654 | defer file.close(io); | |
| 2655 | // TODO https://github.com/ziglang/zig/issues/23955 | |
| 2656 | var read_buffer: [1024]u8 = undefined; | |
| 2657 | var file_reader = file.reader(io, &read_buffer); | |
| 2658 | var write_buffer: [1024]u8 = undefined; | |
| 2659 | var stdin_writer = child.stdin.?.writerStreaming(io, &write_buffer); | |
| 2660 | _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { | |
| 2661 | error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{ | |
| 2662 | path, file_reader.err.?, | |
| 2663 | }), | |
| 2664 | error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ | |
| 2665 | stdin_writer.err.?, | |
| 2666 | }), | |
| 2667 | }; | |
| 2668 | stdin_writer.interface.flush() catch |err| switch (err) { | |
| 2669 | error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{ | |
| 2670 | stdin_writer.err.?, | |
| 2671 | }), | |
| 2672 | }; | |
| 2673 | child.stdin.?.close(io); | |
| 2674 | child.stdin = null; | |
| 2675 | }, | |
| 2676 | .none => {}, | |
| 2677 | } | |
| 2678 | ||
| 2679 | var stdout_bytes: ?[]const u8 = null; | |
| 2680 | var stderr_bytes: ?[]const u8 = null; | |
| 2681 | ||
| 2682 | if (child.stdout) |stdout| { | |
| 2683 | if (child.stderr) |stderr| { | |
| 2684 | var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined; | |
| 2685 | var multi_reader: Io.File.MultiReader = undefined; | |
| 2686 | multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ stdout, stderr }); | |
| 2687 | defer multi_reader.deinit(); | |
| 2688 | ||
| 2689 | const stdout_reader = multi_reader.reader(0); | |
| 2690 | const stderr_reader = multi_reader.reader(1); | |
| 2691 | ||
| 2692 | while (multi_reader.fill(64, .none)) |_| { | |
| 2693 | if (run.stdio_limit.toInt()) |limit| { | |
| 2694 | if (stdout_reader.buffered().len > limit) | |
| 2695 | return error.StdoutStreamTooLong; | |
| 2696 | if (stderr_reader.buffered().len > limit) | |
| 2697 | return error.StderrStreamTooLong; | |
| 2698 | } | |
| 2699 | } else |err| switch (err) { | |
| 2700 | error.Timeout => unreachable, | |
| 2701 | error.EndOfStream => {}, | |
| 2702 | else => |e| return e, | |
| 2703 | } | |
| 2704 | ||
| 2705 | try multi_reader.checkAnyError(); | |
| 2706 | ||
| 2707 | // TODO: this string can leak since alloc below can return error. | |
| 2708 | stdout_bytes = try multi_reader.toOwnedSlice(0); | |
| 2709 | // TODO: this string can leak since its allocated using gpa and `try child.wait(io)` below can fail. | |
| 2710 | stderr_bytes = try multi_reader.toOwnedSlice(1); | |
| 2711 | } else { | |
| 2712 | var stdout_reader = stdout.readerStreaming(io, &.{}); | |
| 2713 | stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { | |
| 2714 | error.OutOfMemory => |e| return e, | |
| 2715 | error.ReadFailed => return stdout_reader.err.?, | |
| 2716 | error.StreamTooLong => return error.StdoutStreamTooLong, | |
| 2717 | }; | |
| 2718 | } | |
| 2719 | } else if (child.stderr) |stderr| { | |
| 2720 | var stderr_reader = stderr.readerStreaming(io, &.{}); | |
| 2721 | stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { | |
| 2722 | error.OutOfMemory => |e| return e, | |
| 2723 | error.ReadFailed => return stderr_reader.err.?, | |
| 2724 | error.StreamTooLong => return error.StderrStreamTooLong, | |
| 2725 | }; | |
| 2726 | } | |
| 2727 | ||
| 2728 | if (stderr_bytes) |bytes| if (bytes.len > 0) { | |
| 2729 | // Treat stderr as an error message. | |
| 2730 | const stderr_is_diagnostic = run.captured_stderr == null and switch (run.stdio) { | |
| 2731 | .check => |checks| !checksContainStderr(checks.items), | |
| 2732 | else => true, | |
| 2733 | }; | |
| 2734 | if (stderr_is_diagnostic) { | |
| 2735 | run.step.result_stderr = bytes; | |
| 2736 | } | |
| 2737 | }; | |
| 2738 | ||
| 2739 | run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; | |
| 2740 | ||
| 2741 | return .{ | |
| 2742 | .term = try child.wait(io), | |
| 2743 | .stdout = stdout_bytes, | |
| 2744 | .stderr = stderr_bytes, | |
| 2745 | }; | |
| 2746 | } | |
| 2747 | ||
| 2748 | fn addPathForDynLibs(run: *Run, artifact: *Step.Compile) void { | |
| 2749 | const b = run.step.owner; | |
| 2750 | const compiles = artifact.getCompileDependencies(true); | |
| 2751 | for (compiles) |compile| { | |
| 2752 | if (compile.root_module.resolved_target.?.result.os.tag == .windows and | |
| 2753 | compile.isDynamicLibrary()) | |
| 2754 | { | |
| 2755 | addPathDir(run, Dir.path.dirname(compile.getEmittedBin().getPath2(b, &run.step)).?); | |
| 2756 | } | |
| 2757 | } | |
| 2758 | } | |
| 2759 | ||
| 2760 | fn failForeign( | |
| 2761 | run: *Run, | |
| 2762 | suggested_flag: []const u8, | |
| 2763 | argv0: []const u8, | |
| 2764 | exe: *Step.Compile, | |
| 2765 | ) error{ MakeFailed, MakeSkipped, OutOfMemory } { | |
| 2766 | switch (run.stdio) { | |
| 2767 | .check, .zig_test => { | |
| 2768 | if (run.skip_foreign_checks) | |
| 2769 | return error.MakeSkipped; | |
| 2770 | ||
| 2771 | const b = run.step.owner; | |
| 2772 | const host_name = try b.graph.host.result.zigTriple(b.allocator); | |
| 2773 | const foreign_name = try exe.rootModuleTarget().zigTriple(b.allocator); | |
| 2774 | ||
| 2775 | return run.step.fail( | |
| 2776 | \\unable to spawn foreign binary '{s}' ({s}) on host system ({s}) | |
| 2777 | \\ consider using {s} or enabling skip_foreign_checks in the Run step | |
| 2778 | , .{ argv0, foreign_name, host_name, suggested_flag }); | |
| 2779 | }, | |
| 2780 | else => { | |
| 2781 | return run.step.fail("unable to spawn foreign binary '{s}'", .{argv0}); | |
| 2782 | }, | |
| 2783 | } | |
| 2784 | } | |
| 2785 | ||
| 2786 | fn hashStdIo(hh: *std.Build.Cache.HashHelper, stdio: StdIo) void { | |
| 2787 | switch (stdio) { | |
| 2788 | .infer_from_args, .inherit, .zig_test => {}, | |
| 2789 | .check => |checks| for (checks.items) |check| { | |
| 2790 | hh.add(@as(std.meta.Tag(StdIo.Check), check)); | |
| 2791 | switch (check) { | |
| 2792 | .expect_stderr_exact, | |
| 2793 | .expect_stderr_match, | |
| 2794 | .expect_stdout_exact, | |
| 2795 | .expect_stdout_match, | |
| 2796 | => |s| hh.addBytes(s), | |
| 2797 | ||
| 2798 | .expect_term => |term| { | |
| 2799 | hh.add(@as(std.meta.Tag(process.Child.Term), term)); | |
| 2800 | switch (term) { | |
| 2801 | inline .exited, .signal, .stopped => |x| hh.add(x), | |
| 2802 | .unknown => |x| hh.add(x), | |
| 2803 | } | |
| 2804 | }, | |
| 2805 | } | |
| 2806 | }, | |
| 2807 | } | |
| 2808 | } |
lib/std/Build/Step/TranslateC.zig+2-2
| ... | ... | @@ -6,7 +6,7 @@ const mem = std.mem; |
| 6 | 6 | |
| 7 | 7 | const TranslateC = @This(); |
| 8 | 8 | |
| 9 | pub const base_id: Step.Id = .translate_c; | |
| 9 | pub const base_tag: Step.Tag = .translate_c; | |
| 10 | 10 | |
| 11 | 11 | step: Step, |
| 12 | 12 | source: std.Build.LazyPath, |
| ... | ... | @@ -31,7 +31,7 @@ pub fn create(owner: *std.Build, options: Options) *TranslateC { |
| 31 | 31 | const source = options.root_source_file.dupe(owner); |
| 32 | 32 | translate_c.* = .{ |
| 33 | 33 | .step = Step.init(.{ |
| 34 | .id = base_id, | |
| 34 | .tag = base_tag, | |
| 35 | 35 | .name = "translate-c", |
| 36 | 36 | .owner = owner, |
| 37 | 37 | .makeFn = make, |
lib/std/Build/Step/UpdateSourceFiles.zig+2-2
| ... | ... | @@ -14,7 +14,7 @@ const ArrayList = std.ArrayList; |
| 14 | 14 | step: Step, |
| 15 | 15 | output_source_files: std.ArrayList(OutputSourceFile), |
| 16 | 16 | |
| 17 | pub const base_id: Step.Id = .update_source_files; | |
| 17 | pub const base_tag: Step.Tag = .update_source_files; | |
| 18 | 18 | |
| 19 | 19 | pub const OutputSourceFile = struct { |
| 20 | 20 | contents: Contents, |
| ... | ... | @@ -30,7 +30,7 @@ pub fn create(owner: *std.Build) *UpdateSourceFiles { |
| 30 | 30 | const usf = owner.allocator.create(UpdateSourceFiles) catch @panic("OOM"); |
| 31 | 31 | usf.* = .{ |
| 32 | 32 | .step = Step.init(.{ |
| 33 | .id = base_id, | |
| 33 | .tag = base_tag, | |
| 34 | 34 | .name = "UpdateSourceFiles", |
| 35 | 35 | .owner = owner, |
| 36 | 36 | .makeFn = make, |
lib/std/Build/Step/WriteFile.zig+2-209
| ... | ... | @@ -18,7 +18,7 @@ directories: std.ArrayList(Directory), |
| 18 | 18 | generated_directory: std.Build.GeneratedFile, |
| 19 | 19 | mode: Mode = .whole_cached, |
| 20 | 20 | |
| 21 | pub const base_id: Step.Id = .write_file; | |
| 21 | pub const base_tag: Step.Tag = .write_file; | |
| 22 | 22 | |
| 23 | 23 | pub const Mode = union(enum) { |
| 24 | 24 | /// Default mode. Integrates with the cache system. The directory should be |
| ... | ... | @@ -89,10 +89,9 @@ pub fn create(owner: *std.Build) *WriteFile { |
| 89 | 89 | const write_file = owner.allocator.create(WriteFile) catch @panic("OOM"); |
| 90 | 90 | write_file.* = .{ |
| 91 | 91 | .step = Step.init(.{ |
| 92 | .id = base_id, | |
| 92 | .tag = base_tag, | |
| 93 | 93 | .name = "WriteFile", |
| 94 | 94 | .owner = owner, |
| 95 | .makeFn = make, | |
| 96 | 95 | }), |
| 97 | 96 | .files = .empty, |
| 98 | 97 | .directories = .empty, |
| ... | ... | @@ -191,209 +190,3 @@ fn maybeUpdateName(write_file: *WriteFile) void { |
| 191 | 190 | } |
| 192 | 191 | } |
| 193 | 192 | } |
| 194 | ||
| 195 | fn make(step: *Step, options: Step.MakeOptions) !void { | |
| 196 | _ = options; | |
| 197 | const b = step.owner; | |
| 198 | const graph = b.graph; | |
| 199 | const io = graph.io; | |
| 200 | const arena = b.allocator; | |
| 201 | const gpa = graph.cache.gpa; | |
| 202 | const write_file: *WriteFile = @fieldParentPtr("step", step); | |
| 203 | ||
| 204 | const open_dir_cache = try arena.alloc(Io.Dir, write_file.directories.items.len); | |
| 205 | var open_dirs_count: usize = 0; | |
| 206 | defer Io.Dir.closeMany(io, open_dir_cache[0..open_dirs_count]); | |
| 207 | ||
| 208 | switch (write_file.mode) { | |
| 209 | .whole_cached => { | |
| 210 | step.clearWatchInputs(); | |
| 211 | ||
| 212 | // The cache is used here not really as a way to speed things up - because writing | |
| 213 | // the data to a file would probably be very fast - but as a way to find a canonical | |
| 214 | // location to put build artifacts. | |
| 215 | ||
| 216 | // If, for example, a hard-coded path was used as the location to put WriteFile | |
| 217 | // files, then two WriteFiles executing in parallel might clobber each other. | |
| 218 | ||
| 219 | var man = b.graph.cache.obtain(); | |
| 220 | defer man.deinit(); | |
| 221 | ||
| 222 | for (write_file.files.items) |file| { | |
| 223 | man.hash.addBytes(file.sub_path); | |
| 224 | ||
| 225 | switch (file.contents) { | |
| 226 | .bytes => |bytes| { | |
| 227 | man.hash.addBytes(bytes); | |
| 228 | }, | |
| 229 | .copy => |lazy_path| { | |
| 230 | const path = lazy_path.getPath3(b, step); | |
| 231 | _ = try man.addFilePath(path, null); | |
| 232 | try step.addWatchInput(lazy_path); | |
| 233 | }, | |
| 234 | } | |
| 235 | } | |
| 236 | ||
| 237 | for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { | |
| 238 | man.hash.addBytes(dir.sub_path); | |
| 239 | for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); | |
| 240 | if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); | |
| 241 | ||
| 242 | const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); | |
| 243 | const src_dir_path = dir.source.getPath3(b, step); | |
| 244 | ||
| 245 | var src_dir = src_dir_path.root_dir.handle.openDir(io, src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { | |
| 246 | return step.fail("unable to open source directory '{f}': {s}", .{ | |
| 247 | src_dir_path, @errorName(err), | |
| 248 | }); | |
| 249 | }; | |
| 250 | open_dir_cache_elem.* = src_dir; | |
| 251 | open_dirs_count += 1; | |
| 252 | ||
| 253 | var it = try src_dir.walk(gpa); | |
| 254 | defer it.deinit(); | |
| 255 | while (try it.next(io)) |entry| { | |
| 256 | if (!dir.options.pathIncluded(entry.path)) continue; | |
| 257 | ||
| 258 | switch (entry.kind) { | |
| 259 | .directory => { | |
| 260 | if (need_derived_inputs) { | |
| 261 | const entry_path = try src_dir_path.join(arena, entry.path); | |
| 262 | try step.addDirectoryWatchInputFromPath(entry_path); | |
| 263 | } | |
| 264 | }, | |
| 265 | .file => { | |
| 266 | const entry_path = try src_dir_path.join(arena, entry.path); | |
| 267 | _ = try man.addFilePath(entry_path, null); | |
| 268 | }, | |
| 269 | else => continue, | |
| 270 | } | |
| 271 | } | |
| 272 | } | |
| 273 | ||
| 274 | if (try step.cacheHit(&man)) { | |
| 275 | const digest = man.final(); | |
| 276 | write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); | |
| 277 | assert(step.result_cached); | |
| 278 | return; | |
| 279 | } | |
| 280 | ||
| 281 | const digest = man.final(); | |
| 282 | const cache_path = "o" ++ Dir.path.sep_str ++ digest; | |
| 283 | ||
| 284 | write_file.generated_directory.path = try b.cache_root.join(arena, &.{cache_path}); | |
| 285 | ||
| 286 | try operate(write_file, open_dir_cache, .{ | |
| 287 | .root_dir = b.cache_root, | |
| 288 | .sub_path = cache_path, | |
| 289 | }); | |
| 290 | ||
| 291 | try step.writeManifest(&man); | |
| 292 | }, | |
| 293 | .tmp => { | |
| 294 | step.result_cached = false; | |
| 295 | ||
| 296 | var rand_int: u64 = undefined; | |
| 297 | io.random(@ptrCast(&rand_int)); | |
| 298 | const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); | |
| 299 | ||
| 300 | write_file.generated_directory.path = try b.cache_root.join(arena, &.{tmp_dir_sub_path}); | |
| 301 | ||
| 302 | try operate(write_file, open_dir_cache, .{ | |
| 303 | .root_dir = b.cache_root, | |
| 304 | .sub_path = tmp_dir_sub_path, | |
| 305 | }); | |
| 306 | }, | |
| 307 | .mutate => |lp| { | |
| 308 | step.result_cached = false; | |
| 309 | const root_path = try lp.getPath4(b, step); | |
| 310 | write_file.generated_directory.path = try root_path.toString(arena); | |
| 311 | try operate(write_file, open_dir_cache, root_path); | |
| 312 | }, | |
| 313 | } | |
| 314 | } | |
| 315 | ||
| 316 | fn operate(write_file: *WriteFile, open_dir_cache: []const Io.Dir, root_path: std.Build.Cache.Path) !void { | |
| 317 | const step = &write_file.step; | |
| 318 | const b = step.owner; | |
| 319 | const io = b.graph.io; | |
| 320 | const gpa = b.graph.cache.gpa; | |
| 321 | const arena = b.allocator; | |
| 322 | ||
| 323 | var cache_dir = root_path.root_dir.handle.createDirPathOpen(io, root_path.sub_path, .{}) catch |err| | |
| 324 | return step.fail("unable to make path {f}: {t}", .{ root_path, err }); | |
| 325 | defer cache_dir.close(io); | |
| 326 | ||
| 327 | for (write_file.files.items) |file| { | |
| 328 | if (Dir.path.dirname(file.sub_path)) |dirname| { | |
| 329 | cache_dir.createDirPath(io, dirname) catch |err| { | |
| 330 | return step.fail("unable to make path '{f}{c}{s}': {t}", .{ | |
| 331 | root_path, Dir.path.sep, dirname, err, | |
| 332 | }); | |
| 333 | }; | |
| 334 | } | |
| 335 | switch (file.contents) { | |
| 336 | .bytes => |bytes| { | |
| 337 | cache_dir.writeFile(io, .{ .sub_path = file.sub_path, .data = bytes }) catch |err| { | |
| 338 | return step.fail("unable to write file '{f}{c}{s}': {t}", .{ | |
| 339 | root_path, Dir.path.sep, file.sub_path, err, | |
| 340 | }); | |
| 341 | }; | |
| 342 | }, | |
| 343 | .copy => |file_source| { | |
| 344 | const source_path = file_source.getPath2(b, step); | |
| 345 | const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir, file.sub_path, .{}) catch |err| { | |
| 346 | return step.fail("unable to update file from '{s}' to '{f}{c}{s}': {t}", .{ | |
| 347 | source_path, root_path, Dir.path.sep, file.sub_path, err, | |
| 348 | }); | |
| 349 | }; | |
| 350 | // At this point we already will mark the step as a cache miss. | |
| 351 | // But this is kind of a partial cache hit since individual | |
| 352 | // file copies may be avoided. Oh well, this information is | |
| 353 | // discarded. | |
| 354 | _ = prev_status; | |
| 355 | }, | |
| 356 | } | |
| 357 | } | |
| 358 | ||
| 359 | for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { | |
| 360 | const src_dir_path = dir.source.getPath3(b, step); | |
| 361 | const dest_dirname = dir.sub_path; | |
| 362 | ||
| 363 | if (dest_dirname.len != 0) { | |
| 364 | cache_dir.createDirPath(io, dest_dirname) catch |err| { | |
| 365 | return step.fail("unable to make path '{f}{c}{s}': {t}", .{ | |
| 366 | root_path, Dir.path.sep, dest_dirname, err, | |
| 367 | }); | |
| 368 | }; | |
| 369 | } | |
| 370 | ||
| 371 | var it = try already_open_dir.walk(gpa); | |
| 372 | defer it.deinit(); | |
| 373 | while (try it.next(io)) |entry| { | |
| 374 | if (!dir.options.pathIncluded(entry.path)) continue; | |
| 375 | ||
| 376 | const src_entry_path = try src_dir_path.join(arena, entry.path); | |
| 377 | const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); | |
| 378 | switch (entry.kind) { | |
| 379 | .directory => try cache_dir.createDirPath(io, dest_path), | |
| 380 | .file => { | |
| 381 | const prev_status = Io.Dir.updateFile( | |
| 382 | src_entry_path.root_dir.handle, | |
| 383 | io, | |
| 384 | src_entry_path.sub_path, | |
| 385 | cache_dir, | |
| 386 | dest_path, | |
| 387 | .{}, | |
| 388 | ) catch |err| { | |
| 389 | return step.fail("unable to update file from '{f}' to '{f}{c}{s}': {t}", .{ | |
| 390 | src_entry_path, root_path, Dir.path.sep, dest_path, err, | |
| 391 | }); | |
| 392 | }; | |
| 393 | _ = prev_status; | |
| 394 | }, | |
| 395 | else => continue, | |
| 396 | } | |
| 397 | } | |
| 398 | } | |
| 399 | } |
lib/std/Build/Watch.zig deleted-968
| ... | ... | @@ -1,968 +0,0 @@ |
| 1 | const builtin = @import("builtin"); | |
| 2 | ||
| 3 | const std = @import("../std.zig"); | |
| 4 | const Io = std.Io; | |
| 5 | const Step = std.Build.Step; | |
| 6 | const Allocator = std.mem.Allocator; | |
| 7 | const assert = std.debug.assert; | |
| 8 | const fatal = std.process.fatal; | |
| 9 | const Watch = @This(); | |
| 10 | const FsEvents = @import("Watch/FsEvents.zig"); | |
| 11 | ||
| 12 | os: Os, | |
| 13 | /// The number to show as the number of directories being watched. | |
| 14 | dir_count: usize, | |
| 15 | // These fields are common to most implementations so are kept here for simplicity. | |
| 16 | // They are `undefined` on implementations which do not utilize then. | |
| 17 | dir_table: DirTable, | |
| 18 | generation: Generation, | |
| 19 | ||
| 20 | pub const have_impl = Os != void; | |
| 21 | ||
| 22 | /// Key is the directory to watch which contains one or more files we are | |
| 23 | /// interested in noticing changes to. | |
| 24 | /// | |
| 25 | /// Value is generation. | |
| 26 | const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false); | |
| 27 | ||
| 28 | /// Special key of "." means any changes in this directory trigger the steps. | |
| 29 | const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); | |
| 30 | const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation); | |
| 31 | ||
| 32 | const Generation = u8; | |
| 33 | ||
| 34 | const Hash = std.hash.Wyhash; | |
| 35 | const Cache = std.Build.Cache; | |
| 36 | ||
| 37 | const Os = switch (builtin.os.tag) { | |
| 38 | .linux => struct { | |
| 39 | const posix = std.posix; | |
| 40 | ||
| 41 | /// Keyed differently but indexes correspond 1:1 with `dir_table`. | |
| 42 | handle_table: HandleTable, | |
| 43 | /// fanotify file descriptors are keyed by mount id since marks | |
| 44 | /// are limited to a single filesystem. | |
| 45 | poll_fds: std.AutoArrayHashMapUnmanaged(MountId, posix.pollfd), | |
| 46 | ||
| 47 | const MountId = i32; | |
| 48 | const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, struct { mount_id: MountId, reaction_set: ReactionSet }, FileHandle.Adapter, false); | |
| 49 | ||
| 50 | const fan_mask: std.os.linux.fanotify.MarkMask = .{ | |
| 51 | .CLOSE_WRITE = true, | |
| 52 | .CREATE = true, | |
| 53 | .DELETE = true, | |
| 54 | .DELETE_SELF = true, | |
| 55 | .EVENT_ON_CHILD = true, | |
| 56 | .MOVED_FROM = true, | |
| 57 | .MOVED_TO = true, | |
| 58 | .MOVE_SELF = true, | |
| 59 | .ONDIR = true, | |
| 60 | }; | |
| 61 | ||
| 62 | const FileHandle = struct { | |
| 63 | handle: *align(1) std.os.linux.file_handle, | |
| 64 | ||
| 65 | fn clone(lfh: FileHandle, gpa: Allocator) Allocator.Error!FileHandle { | |
| 66 | const bytes = lfh.slice(); | |
| 67 | const new_ptr = try gpa.alignedAlloc( | |
| 68 | u8, | |
| 69 | .of(std.os.linux.file_handle), | |
| 70 | @sizeOf(std.os.linux.file_handle) + bytes.len, | |
| 71 | ); | |
| 72 | const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); | |
| 73 | new_header.* = lfh.handle.*; | |
| 74 | const new: FileHandle = .{ .handle = new_header }; | |
| 75 | @memcpy(new.slice(), lfh.slice()); | |
| 76 | return new; | |
| 77 | } | |
| 78 | ||
| 79 | fn destroy(lfh: FileHandle, gpa: Allocator) void { | |
| 80 | const ptr: [*]u8 = @ptrCast(lfh.handle); | |
| 81 | const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; | |
| 82 | return gpa.free(allocated_slice); | |
| 83 | } | |
| 84 | ||
| 85 | fn slice(lfh: FileHandle) []u8 { | |
| 86 | const ptr: [*]u8 = &lfh.handle.f_handle; | |
| 87 | return ptr[0..lfh.handle.handle_bytes]; | |
| 88 | } | |
| 89 | ||
| 90 | const Adapter = struct { | |
| 91 | pub fn hash(self: Adapter, a: FileHandle) u32 { | |
| 92 | _ = self; | |
| 93 | const unsigned_type: u32 = @bitCast(a.handle.handle_type); | |
| 94 | return @truncate(Hash.hash(unsigned_type, a.slice())); | |
| 95 | } | |
| 96 | pub fn eql(self: Adapter, a: FileHandle, b: FileHandle, b_index: usize) bool { | |
| 97 | _ = self; | |
| 98 | _ = b_index; | |
| 99 | return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice()); | |
| 100 | } | |
| 101 | }; | |
| 102 | }; | |
| 103 | ||
| 104 | fn init(cwd_path: []const u8) !Watch { | |
| 105 | _ = cwd_path; | |
| 106 | return .{ | |
| 107 | .dir_table = .{}, | |
| 108 | .dir_count = 0, | |
| 109 | .os = switch (builtin.os.tag) { | |
| 110 | .linux => .{ | |
| 111 | .handle_table = .{}, | |
| 112 | .poll_fds = .{}, | |
| 113 | }, | |
| 114 | else => {}, | |
| 115 | }, | |
| 116 | .generation = 0, | |
| 117 | }; | |
| 118 | } | |
| 119 | ||
| 120 | fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path, mount_id: *MountId) !FileHandle { | |
| 121 | var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; | |
| 122 | var buf: [std.fs.max_path_bytes]u8 = undefined; | |
| 123 | const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{ | |
| 124 | path.sub_path, | |
| 125 | }) catch return error.NameTooLong; | |
| 126 | const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); | |
| 127 | stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); | |
| 128 | try posix.name_to_handle_at(path.root_dir.handle.handle, adjusted_path, stack_ptr, mount_id, std.os.linux.AT.HANDLE_FID); | |
| 129 | const stack_lfh: FileHandle = .{ .handle = stack_ptr }; | |
| 130 | return stack_lfh.clone(gpa); | |
| 131 | } | |
| 132 | ||
| 133 | fn markDirtySteps(w: *Watch, gpa: Allocator, fan_fd: posix.fd_t) !bool { | |
| 134 | const fanotify = std.os.linux.fanotify; | |
| 135 | const M = fanotify.event_metadata; | |
| 136 | var events_buf: [256 + 4096]u8 = undefined; | |
| 137 | var any_dirty = false; | |
| 138 | while (true) { | |
| 139 | var len = posix.read(fan_fd, &events_buf) catch |err| switch (err) { | |
| 140 | error.WouldBlock => return any_dirty, | |
| 141 | else => |e| return e, | |
| 142 | }; | |
| 143 | var meta: [*]align(1) M = @ptrCast(&events_buf); | |
| 144 | while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ | |
| 145 | len -= meta[0].event_len; | |
| 146 | meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); | |
| 147 | }) { | |
| 148 | assert(meta[0].vers == M.VERSION); | |
| 149 | if (meta[0].mask.Q_OVERFLOW) { | |
| 150 | any_dirty = true; | |
| 151 | std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); | |
| 152 | markAllFilesDirty(w, gpa); | |
| 153 | return true; | |
| 154 | } | |
| 155 | const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); | |
| 156 | switch (fid.hdr.info_type) { | |
| 157 | .DFID_NAME => { | |
| 158 | const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); | |
| 159 | const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); | |
| 160 | const file_name = std.mem.span(file_name_z); | |
| 161 | const lfh: FileHandle = .{ .handle = file_handle }; | |
| 162 | if (w.os.handle_table.getPtr(lfh)) |value| { | |
| 163 | if (value.reaction_set.getPtr(".")) |glob_set| | |
| 164 | any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); | |
| 165 | if (value.reaction_set.getPtr(file_name)) |step_set| | |
| 166 | any_dirty = markStepSetDirty(gpa, step_set, any_dirty); | |
| 167 | } | |
| 168 | }, | |
| 169 | else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), | |
| 170 | } | |
| 171 | } | |
| 172 | } | |
| 173 | } | |
| 174 | ||
| 175 | fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 176 | // Add missing marks and note persisted ones. | |
| 177 | for (steps) |step| { | |
| 178 | for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { | |
| 179 | const reaction_set = rs: { | |
| 180 | const gop = try w.dir_table.getOrPut(gpa, path); | |
| 181 | if (!gop.found_existing) { | |
| 182 | var mount_id: MountId = undefined; | |
| 183 | const dir_handle = getDirHandle(gpa, path, &mount_id) catch |err| switch (err) { | |
| 184 | error.FileNotFound => { | |
| 185 | std.debug.assert(w.dir_table.swapRemove(path)); | |
| 186 | continue; | |
| 187 | }, | |
| 188 | else => return err, | |
| 189 | }; | |
| 190 | const fan_fd = blk: { | |
| 191 | const fd_gop = try w.os.poll_fds.getOrPut(gpa, mount_id); | |
| 192 | if (!fd_gop.found_existing) { | |
| 193 | const fan_fd = std.posix.fanotify_init(.{ | |
| 194 | .CLASS = .NOTIF, | |
| 195 | .CLOEXEC = true, | |
| 196 | .NONBLOCK = true, | |
| 197 | .REPORT_NAME = true, | |
| 198 | .REPORT_DIR_FID = true, | |
| 199 | .REPORT_FID = true, | |
| 200 | .REPORT_TARGET_FID = true, | |
| 201 | }, 0) catch |err| switch (err) { | |
| 202 | error.UnsupportedFlags => fatal("fanotify_init failed due to old kernel; requires 5.17+", .{}), | |
| 203 | else => |e| return e, | |
| 204 | }; | |
| 205 | fd_gop.value_ptr.* = .{ | |
| 206 | .fd = fan_fd, | |
| 207 | .events = std.posix.POLL.IN, | |
| 208 | .revents = undefined, | |
| 209 | }; | |
| 210 | } | |
| 211 | break :blk fd_gop.value_ptr.*.fd; | |
| 212 | }; | |
| 213 | // `dir_handle` may already be present in the table in | |
| 214 | // the case that we have multiple Cache.Path instances | |
| 215 | // that compare inequal but ultimately point to the same | |
| 216 | // directory on the file system. | |
| 217 | // In such case, we must revert adding this directory, but keep | |
| 218 | // the additions to the step set. | |
| 219 | const dh_gop = try w.os.handle_table.getOrPut(gpa, dir_handle); | |
| 220 | if (dh_gop.found_existing) { | |
| 221 | _ = w.dir_table.pop(); | |
| 222 | } else { | |
| 223 | assert(dh_gop.index == gop.index); | |
| 224 | dh_gop.value_ptr.* = .{ .mount_id = mount_id, .reaction_set = .{} }; | |
| 225 | posix.fanotify_mark(fan_fd, .{ | |
| 226 | .ADD = true, | |
| 227 | .ONLYDIR = true, | |
| 228 | }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| { | |
| 229 | fatal("unable to watch {f}: {s}", .{ path, @errorName(err) }); | |
| 230 | }; | |
| 231 | } | |
| 232 | break :rs &dh_gop.value_ptr.reaction_set; | |
| 233 | } | |
| 234 | break :rs &w.os.handle_table.values()[gop.index].reaction_set; | |
| 235 | }; | |
| 236 | for (files.items) |basename| { | |
| 237 | const gop = try reaction_set.getOrPut(gpa, basename); | |
| 238 | if (!gop.found_existing) gop.value_ptr.* = .{}; | |
| 239 | try gop.value_ptr.put(gpa, step, w.generation); | |
| 240 | } | |
| 241 | } | |
| 242 | } | |
| 243 | ||
| 244 | { | |
| 245 | // Remove marks for files that are no longer inputs. | |
| 246 | var i: usize = 0; | |
| 247 | while (i < w.os.handle_table.entries.len) { | |
| 248 | { | |
| 249 | const reaction_set = &w.os.handle_table.values()[i].reaction_set; | |
| 250 | var step_set_i: usize = 0; | |
| 251 | while (step_set_i < reaction_set.entries.len) { | |
| 252 | const step_set = &reaction_set.values()[step_set_i]; | |
| 253 | var dirent_i: usize = 0; | |
| 254 | while (dirent_i < step_set.entries.len) { | |
| 255 | const generations = step_set.values(); | |
| 256 | if (generations[dirent_i] == w.generation) { | |
| 257 | dirent_i += 1; | |
| 258 | continue; | |
| 259 | } | |
| 260 | step_set.swapRemoveAt(dirent_i); | |
| 261 | } | |
| 262 | if (step_set.entries.len > 0) { | |
| 263 | step_set_i += 1; | |
| 264 | continue; | |
| 265 | } | |
| 266 | reaction_set.swapRemoveAt(step_set_i); | |
| 267 | } | |
| 268 | if (reaction_set.entries.len > 0) { | |
| 269 | i += 1; | |
| 270 | continue; | |
| 271 | } | |
| 272 | } | |
| 273 | ||
| 274 | const path = w.dir_table.keys()[i]; | |
| 275 | ||
| 276 | const mount_id = w.os.handle_table.values()[i].mount_id; | |
| 277 | const fan_fd = w.os.poll_fds.getEntry(mount_id).?.value_ptr.fd; | |
| 278 | posix.fanotify_mark(fan_fd, .{ | |
| 279 | .REMOVE = true, | |
| 280 | .ONLYDIR = true, | |
| 281 | }, fan_mask, path.root_dir.handle.handle, path.subPathOrDot()) catch |err| switch (err) { | |
| 282 | error.FileNotFound => {}, // Expected, harmless. | |
| 283 | else => |e| std.log.warn("unable to unwatch '{f}': {s}", .{ path, @errorName(e) }), | |
| 284 | }; | |
| 285 | ||
| 286 | w.dir_table.swapRemoveAt(i); | |
| 287 | w.os.handle_table.swapRemoveAt(i); | |
| 288 | } | |
| 289 | w.generation +%= 1; | |
| 290 | } | |
| 291 | w.dir_count = w.dir_table.count(); | |
| 292 | } | |
| 293 | ||
| 294 | fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 295 | _ = io; | |
| 296 | const events_len = try std.posix.poll(w.os.poll_fds.values(), timeout.to_i32_ms()); | |
| 297 | if (events_len == 0) | |
| 298 | return .timeout; | |
| 299 | for (w.os.poll_fds.values()) |poll_fd| { | |
| 300 | if (poll_fd.revents & std.posix.POLL.IN == std.posix.POLL.IN and try markDirtySteps(w, gpa, poll_fd.fd)) | |
| 301 | return .dirty; | |
| 302 | } | |
| 303 | return .clean; | |
| 304 | } | |
| 305 | }, | |
| 306 | .windows => struct { | |
| 307 | const windows = std.os.windows; | |
| 308 | ||
| 309 | /// Keyed differently but indexes correspond 1:1 with `dir_table`. | |
| 310 | handle_table: std.ArrayHashMapUnmanaged(*Directory, void, Directory.TableAdapter, false), | |
| 311 | ready_dirs: std.DoublyLinkedList, | |
| 312 | ||
| 313 | const FileId = struct { | |
| 314 | volumeSerialNumber: windows.ULONG, | |
| 315 | indexNumber: windows.LARGE_INTEGER, | |
| 316 | }; | |
| 317 | ||
| 318 | const Directory = struct { | |
| 319 | reaction_set: ReactionSet, | |
| 320 | id: FileId, | |
| 321 | file: Io.File, | |
| 322 | state: enum { idle, listening, ready }, | |
| 323 | iosb: windows.IO_STATUS_BLOCK, | |
| 324 | // 64 KB is the packet size limit when monitoring over a network. | |
| 325 | // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw#remarks | |
| 326 | buffer: [64 * 1024]u8 align(@alignOf(windows.FILE.NOTIFY.INFORMATION)), | |
| 327 | ready_node: std.DoublyLinkedList.Node, | |
| 328 | ||
| 329 | /// Start listening for events, buffer field will be overwritten eventually. | |
| 330 | fn startListening(dir: *Directory, w: *Watch) !void { | |
| 331 | assert(dir.file.flags.nonblocking); | |
| 332 | assert(dir.state == .idle); | |
| 333 | switch (windows.ntdll.NtNotifyChangeDirectoryFileEx( | |
| 334 | dir.file.handle, | |
| 335 | null, | |
| 336 | ¬ifyApc, | |
| 337 | w, | |
| 338 | &dir.iosb, | |
| 339 | &dir.buffer, | |
| 340 | dir.buffer.len, | |
| 341 | .{ | |
| 342 | .FILE_NAME = true, | |
| 343 | .DIR_NAME = true, | |
| 344 | .SIZE = true, | |
| 345 | .LAST_WRITE = true, | |
| 346 | .CREATION = true, | |
| 347 | }, | |
| 348 | .FALSE, | |
| 349 | .Notify, | |
| 350 | )) { | |
| 351 | .SUCCESS, .PENDING => dir.state = .listening, | |
| 352 | .ILLEGAL_FUNCTION => return error.ReadDirectoryChangesUnsupported, | |
| 353 | else => |status| return windows.unexpectedStatus(status), | |
| 354 | } | |
| 355 | } | |
| 356 | ||
| 357 | fn notifyApc(apc_context: ?*anyopaque, iosb: *windows.IO_STATUS_BLOCK, _: windows.ULONG) align(std.Io.Threaded.apc_align) callconv(.winapi) void { | |
| 358 | const w: *Watch = @ptrCast(@alignCast(apc_context)); | |
| 359 | const dir: *Directory = @fieldParentPtr("iosb", iosb); | |
| 360 | assert(iosb.u.Status != .PENDING); | |
| 361 | assert(dir.state == .listening); | |
| 362 | w.os.ready_dirs.append(&dir.ready_node); | |
| 363 | dir.state = .ready; | |
| 364 | } | |
| 365 | ||
| 366 | fn init(gpa: Allocator, path: Cache.Path) !*Directory { | |
| 367 | // The following code is a drawn out NtCreateFile call. (mostly adapted from Io.Dir.makeOpenDirAccessMaskW) | |
| 368 | // It's necessary in order to get the specific flags that are required when calling ReadDirectoryChangesW. | |
| 369 | var dir_handle: windows.HANDLE = undefined; | |
| 370 | const root_fd = path.root_dir.handle.handle; | |
| 371 | const sub_path = path.subPathOrDot(); | |
| 372 | const sub_path_w = try Io.Threaded.sliceToPrefixedFileW(root_fd, sub_path, .{}); // TODO eliminate this call | |
| 373 | var iosb: windows.IO_STATUS_BLOCK = undefined; | |
| 374 | switch (windows.ntdll.NtCreateFile( | |
| 375 | &dir_handle, | |
| 376 | .{ | |
| 377 | .SPECIFIC = .{ .FILE_DIRECTORY = .{ | |
| 378 | .LIST = true, | |
| 379 | } }, | |
| 380 | .STANDARD = .{ .SYNCHRONIZE = true }, | |
| 381 | .GENERIC = .{ .READ = true }, | |
| 382 | }, | |
| 383 | &.{ | |
| 384 | .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w.span())) null else root_fd, | |
| 385 | .ObjectName = @constCast(&sub_path_w.string()), | |
| 386 | }, | |
| 387 | &iosb, | |
| 388 | null, | |
| 389 | .{}, | |
| 390 | .VALID_FLAGS, | |
| 391 | .OPEN, | |
| 392 | .{ | |
| 393 | .DIRECTORY_FILE = true, | |
| 394 | .IO = .ASYNCHRONOUS, | |
| 395 | .OPEN_FOR_BACKUP_INTENT = true, | |
| 396 | }, | |
| 397 | null, | |
| 398 | 0, | |
| 399 | )) { | |
| 400 | .SUCCESS => {}, | |
| 401 | .OBJECT_NAME_INVALID => return error.BadPathName, | |
| 402 | .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, | |
| 403 | .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, | |
| 404 | .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, | |
| 405 | .NOT_A_DIRECTORY => return error.NotDir, | |
| 406 | // This can happen if the directory has 'List folder contents' permission set to 'Deny' | |
| 407 | .ACCESS_DENIED => return error.AccessDenied, | |
| 408 | .INVALID_PARAMETER => unreachable, | |
| 409 | else => |rc| return windows.unexpectedStatus(rc), | |
| 410 | } | |
| 411 | assert(dir_handle != windows.INVALID_HANDLE_VALUE); | |
| 412 | errdefer windows.CloseHandle(dir_handle); | |
| 413 | ||
| 414 | const dir_id = try getFileId(dir_handle); | |
| 415 | ||
| 416 | const dir = try gpa.create(Directory); | |
| 417 | dir.* = .{ | |
| 418 | .reaction_set = .empty, | |
| 419 | .id = dir_id, | |
| 420 | .file = .{ .handle = dir_handle, .flags = .{ .nonblocking = true } }, | |
| 421 | .state = .idle, | |
| 422 | .iosb = undefined, | |
| 423 | .buffer = undefined, | |
| 424 | .ready_node = undefined, | |
| 425 | }; | |
| 426 | return dir; | |
| 427 | } | |
| 428 | ||
| 429 | fn deinit(dir: *Directory, gpa: Allocator, w: *Watch) void { | |
| 430 | state: switch (dir.state) { | |
| 431 | .idle => {}, | |
| 432 | .listening => { | |
| 433 | var cancel_iosb: windows.IO_STATUS_BLOCK = undefined; | |
| 434 | _ = windows.ntdll.NtCancelIoFileEx(dir.file.handle, &dir.iosb, &cancel_iosb); | |
| 435 | while (switch (dir.state) { | |
| 436 | .idle => unreachable, | |
| 437 | .listening => true, | |
| 438 | .ready => false, | |
| 439 | }) Io.Threaded.waitForApcOrAlert(); | |
| 440 | continue :state .ready; | |
| 441 | }, | |
| 442 | .ready => w.os.ready_dirs.remove(&dir.ready_node), | |
| 443 | } | |
| 444 | windows.CloseHandle(dir.file.handle); | |
| 445 | gpa.destroy(dir); | |
| 446 | } | |
| 447 | ||
| 448 | /// Useful to make `*Directory` a key in `std.ArrayHashMap`. | |
| 449 | const TableAdapter = struct { | |
| 450 | pub fn hash(_: TableAdapter, lhs_dir: *Directory) u32 { | |
| 451 | return @truncate(Hash.hash(lhs_dir.id.volumeSerialNumber, @ptrCast(&lhs_dir.id.indexNumber))); | |
| 452 | } | |
| 453 | pub fn eql(_: TableAdapter, lhs_dir: *Directory, rhs_dir: *Directory, rhs_index: usize) bool { | |
| 454 | _ = rhs_index; | |
| 455 | return lhs_dir.id.volumeSerialNumber == rhs_dir.id.volumeSerialNumber and | |
| 456 | lhs_dir.id.indexNumber == rhs_dir.id.indexNumber; | |
| 457 | } | |
| 458 | }; | |
| 459 | }; | |
| 460 | ||
| 461 | fn init(cwd_path: []const u8) !Watch { | |
| 462 | _ = cwd_path; | |
| 463 | return .{ | |
| 464 | .dir_table = .{}, | |
| 465 | .dir_count = 0, | |
| 466 | .os = switch (builtin.os.tag) { | |
| 467 | .windows => .{ | |
| 468 | .handle_table = .empty, | |
| 469 | .ready_dirs = .{}, | |
| 470 | }, | |
| 471 | else => {}, | |
| 472 | }, | |
| 473 | .generation = 0, | |
| 474 | }; | |
| 475 | } | |
| 476 | ||
| 477 | fn getFileId(handle: windows.HANDLE) !FileId { | |
| 478 | var file_id: FileId = undefined; | |
| 479 | var io_status: windows.IO_STATUS_BLOCK = undefined; | |
| 480 | var volume_info: windows.FILE.FS_VOLUME_INFORMATION = undefined; | |
| 481 | switch (windows.ntdll.NtQueryVolumeInformationFile( | |
| 482 | handle, | |
| 483 | &io_status, | |
| 484 | &volume_info, | |
| 485 | @sizeOf(windows.FILE.FS_VOLUME_INFORMATION), | |
| 486 | .Volume, | |
| 487 | )) { | |
| 488 | .SUCCESS => {}, | |
| 489 | // Buffer overflow here indicates that there is more information available than was able to be stored in the buffer | |
| 490 | // size provided. This is treated as success because the type of variable-length information that this would be relevant for | |
| 491 | // (name, volume name, etc) we don't care about. | |
| 492 | .BUFFER_OVERFLOW => {}, | |
| 493 | else => |rc| return windows.unexpectedStatus(rc), | |
| 494 | } | |
| 495 | file_id.volumeSerialNumber = volume_info.VolumeSerialNumber; | |
| 496 | var internal_info: windows.FILE.INTERNAL_INFORMATION = undefined; | |
| 497 | switch (windows.ntdll.NtQueryInformationFile( | |
| 498 | handle, | |
| 499 | &io_status, | |
| 500 | &internal_info, | |
| 501 | @sizeOf(windows.FILE.INTERNAL_INFORMATION), | |
| 502 | .Internal, | |
| 503 | )) { | |
| 504 | .SUCCESS => {}, | |
| 505 | else => |rc| return windows.unexpectedStatus(rc), | |
| 506 | } | |
| 507 | file_id.indexNumber = internal_info.IndexNumber; | |
| 508 | return file_id; | |
| 509 | } | |
| 510 | ||
| 511 | fn markDirtySteps(w: *Watch, gpa: Allocator, dir: *Directory) !bool { | |
| 512 | var any_dirty = false; | |
| 513 | const bytes_returned = dir.iosb.Information; | |
| 514 | if (bytes_returned == 0) { | |
| 515 | std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); | |
| 516 | markAllFilesDirty(w, gpa); | |
| 517 | try dir.startListening(w); | |
| 518 | return true; | |
| 519 | } | |
| 520 | var file_name_buf: [std.fs.max_path_bytes]u8 = undefined; | |
| 521 | var offset: usize = 0; | |
| 522 | while (true) { | |
| 523 | const notify: *windows.FILE.NOTIFY.INFORMATION = @ptrCast(@alignCast(&dir.buffer[offset])); | |
| 524 | const file_name = file_name_buf[0..std.unicode.wtf16LeToWtf8(&file_name_buf, notify.fileName())]; | |
| 525 | if (dir.reaction_set.getPtr(".")) |glob_set| | |
| 526 | any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); | |
| 527 | if (dir.reaction_set.getPtr(file_name)) |step_set| | |
| 528 | any_dirty = markStepSetDirty(gpa, step_set, any_dirty); | |
| 529 | if (notify.NextEntryOffset == 0) | |
| 530 | break; | |
| 531 | ||
| 532 | offset += notify.NextEntryOffset; | |
| 533 | } | |
| 534 | ||
| 535 | // We call this now since at this point we have finished reading dir.buffer. | |
| 536 | try dir.startListening(w); | |
| 537 | return any_dirty; | |
| 538 | } | |
| 539 | ||
| 540 | fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 541 | // Add missing marks and note persisted ones. | |
| 542 | for (steps) |step| { | |
| 543 | for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { | |
| 544 | const dir = dir: { | |
| 545 | const gop = try w.dir_table.getOrPut(gpa, path); | |
| 546 | if (!gop.found_existing) { | |
| 547 | const dir: *Directory = try .init(gpa, path); | |
| 548 | errdefer dir.deinit(gpa, w); | |
| 549 | // `dir.id` may already be present in the table in | |
| 550 | // the case that we have multiple Cache.Path instances | |
| 551 | // that compare inequal but ultimately point to the same | |
| 552 | // directory on the file system. | |
| 553 | // In such case, we must revert adding this directory, but keep | |
| 554 | // the additions to the step set. | |
| 555 | const dh_gop = try w.os.handle_table.getOrPut(gpa, dir); | |
| 556 | if (dh_gop.found_existing) { | |
| 557 | dir.deinit(gpa, w); | |
| 558 | _ = w.dir_table.pop(); | |
| 559 | break :dir w.os.handle_table.keys()[dh_gop.index]; | |
| 560 | } else { | |
| 561 | assert(dh_gop.index == gop.index); | |
| 562 | try dir.startListening(w); | |
| 563 | break :dir dir; | |
| 564 | } | |
| 565 | } | |
| 566 | break :dir w.os.handle_table.keys()[gop.index]; | |
| 567 | }; | |
| 568 | for (files.items) |basename| { | |
| 569 | const gop = try dir.reaction_set.getOrPut(gpa, basename); | |
| 570 | if (!gop.found_existing) gop.value_ptr.* = .{}; | |
| 571 | try gop.value_ptr.put(gpa, step, w.generation); | |
| 572 | } | |
| 573 | } | |
| 574 | } | |
| 575 | ||
| 576 | { | |
| 577 | // Remove marks for files that are no longer inputs. | |
| 578 | var i: usize = 0; | |
| 579 | while (i < w.os.handle_table.entries.len) { | |
| 580 | const dir = w.os.handle_table.keys()[i]; | |
| 581 | { | |
| 582 | var step_set_i: usize = 0; | |
| 583 | while (step_set_i < dir.reaction_set.entries.len) { | |
| 584 | const step_set = &dir.reaction_set.values()[step_set_i]; | |
| 585 | var dirent_i: usize = 0; | |
| 586 | while (dirent_i < step_set.entries.len) { | |
| 587 | const generations = step_set.values(); | |
| 588 | if (generations[dirent_i] == w.generation) { | |
| 589 | dirent_i += 1; | |
| 590 | continue; | |
| 591 | } | |
| 592 | step_set.swapRemoveAt(dirent_i); | |
| 593 | } | |
| 594 | if (step_set.entries.len > 0) { | |
| 595 | step_set_i += 1; | |
| 596 | continue; | |
| 597 | } | |
| 598 | dir.reaction_set.swapRemoveAt(step_set_i); | |
| 599 | } | |
| 600 | if (dir.reaction_set.entries.len > 0) { | |
| 601 | i += 1; | |
| 602 | continue; | |
| 603 | } | |
| 604 | } | |
| 605 | ||
| 606 | w.dir_table.swapRemoveAt(i); | |
| 607 | w.os.handle_table.swapRemoveAt(i); | |
| 608 | dir.deinit(gpa, w); | |
| 609 | } | |
| 610 | w.generation +%= 1; | |
| 611 | } | |
| 612 | w.dir_count = w.dir_table.count(); | |
| 613 | } | |
| 614 | ||
| 615 | fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 616 | for (0..2) |attempt| { | |
| 617 | while (w.os.ready_dirs.popFirst()) |ready_node| { | |
| 618 | const dir: *Directory = @fieldParentPtr("ready_node", ready_node); | |
| 619 | assert(dir.state == .ready); | |
| 620 | dir.state = .idle; | |
| 621 | switch (dir.iosb.u.Status) { | |
| 622 | .SUCCESS => return if (try markDirtySteps(w, gpa, dir)) .dirty else .clean, | |
| 623 | .PENDING => unreachable, | |
| 624 | .CANCELLED => {}, | |
| 625 | else => |status| return windows.unexpectedStatus(status), | |
| 626 | } | |
| 627 | try dir.startListening(w); | |
| 628 | } | |
| 629 | try io.checkCancel(); | |
| 630 | if (attempt == 1) return .timeout; | |
| 631 | const delay_interval: windows.LARGE_INTEGER = switch (timeout) { | |
| 632 | .none => std.math.minInt(windows.LARGE_INTEGER), | |
| 633 | .ms => |ms| -@as(windows.LARGE_INTEGER, ms) * (std.time.ns_per_ms / 100), | |
| 634 | }; | |
| 635 | _ = windows.ntdll.NtDelayExecution(.TRUE, &delay_interval); | |
| 636 | } else unreachable; | |
| 637 | } | |
| 638 | }, | |
| 639 | .dragonfly, .freebsd, .netbsd, .openbsd, .ios, .tvos, .visionos, .watchos => struct { | |
| 640 | const posix = std.posix; | |
| 641 | ||
| 642 | kq_fd: i32, | |
| 643 | /// Indexes correspond 1:1 with `dir_table`. | |
| 644 | handles: std.MultiArrayList(struct { | |
| 645 | rs: ReactionSet, | |
| 646 | /// If the corresponding dir_table Path has sub_path == "", then it | |
| 647 | /// suffices as the open directory handle, and this value will be | |
| 648 | /// -1. Otherwise, it needs to be opened in update(), and will be | |
| 649 | /// stored here. | |
| 650 | dir_fd: i32, | |
| 651 | }), | |
| 652 | ||
| 653 | const dir_open_flags: posix.O = f: { | |
| 654 | var f: posix.O = .{ | |
| 655 | .ACCMODE = .RDONLY, | |
| 656 | .NOFOLLOW = false, | |
| 657 | .DIRECTORY = true, | |
| 658 | .CLOEXEC = true, | |
| 659 | }; | |
| 660 | if (@hasField(posix.O, "EVTONLY")) f.EVTONLY = true; | |
| 661 | if (@hasField(posix.O, "PATH")) f.PATH = true; | |
| 662 | break :f f; | |
| 663 | }; | |
| 664 | ||
| 665 | const EV = std.c.EV; | |
| 666 | const NOTE = std.c.NOTE; | |
| 667 | ||
| 668 | fn init(cwd_path: []const u8) !Watch { | |
| 669 | _ = cwd_path; | |
| 670 | return .{ | |
| 671 | .dir_table = .{}, | |
| 672 | .dir_count = 0, | |
| 673 | .os = .{ | |
| 674 | .kq_fd = try Io.Kqueue.createFileDescriptor(), | |
| 675 | .handles = .empty, | |
| 676 | }, | |
| 677 | .generation = 0, | |
| 678 | }; | |
| 679 | } | |
| 680 | ||
| 681 | fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 682 | const handles = &w.os.handles; | |
| 683 | for (steps) |step| { | |
| 684 | for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { | |
| 685 | const reaction_set = rs: { | |
| 686 | const gop = try w.dir_table.getOrPut(gpa, path); | |
| 687 | if (!gop.found_existing) { | |
| 688 | const skip_open_dir = path.sub_path.len == 0; | |
| 689 | const dir_fd = if (skip_open_dir) | |
| 690 | path.root_dir.handle.handle | |
| 691 | else | |
| 692 | posix.openat(path.root_dir.handle.handle, path.sub_path, dir_open_flags, 0) catch |err| { | |
| 693 | fatal("failed to open directory {f}: {t}", .{ path, err }); | |
| 694 | }; | |
| 695 | // Empirically the dir has to stay open or else no events are triggered. | |
| 696 | errdefer if (!skip_open_dir) std.Io.Threaded.closeFd(dir_fd); | |
| 697 | const changes = [1]posix.Kevent{.{ | |
| 698 | .ident = @bitCast(@as(isize, dir_fd)), | |
| 699 | .filter = std.c.EVFILT.VNODE, | |
| 700 | .flags = EV.ADD | EV.ENABLE | EV.CLEAR, | |
| 701 | .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE, | |
| 702 | .data = 0, | |
| 703 | .udata = gop.index, | |
| 704 | }}; | |
| 705 | _ = try Io.Kqueue.kevent(w.os.kq_fd, &changes, &.{}, null); | |
| 706 | assert(handles.len == gop.index); | |
| 707 | try handles.append(gpa, .{ | |
| 708 | .rs = .{}, | |
| 709 | .dir_fd = if (skip_open_dir) -1 else dir_fd, | |
| 710 | }); | |
| 711 | } | |
| 712 | ||
| 713 | break :rs &handles.items(.rs)[gop.index]; | |
| 714 | }; | |
| 715 | for (files.items) |basename| { | |
| 716 | const gop = try reaction_set.getOrPut(gpa, basename); | |
| 717 | if (!gop.found_existing) gop.value_ptr.* = .{}; | |
| 718 | try gop.value_ptr.put(gpa, step, w.generation); | |
| 719 | } | |
| 720 | } | |
| 721 | } | |
| 722 | ||
| 723 | { | |
| 724 | // Remove marks for files that are no longer inputs. | |
| 725 | var i: usize = 0; | |
| 726 | while (i < handles.len) { | |
| 727 | { | |
| 728 | const reaction_set = &handles.items(.rs)[i]; | |
| 729 | var step_set_i: usize = 0; | |
| 730 | while (step_set_i < reaction_set.entries.len) { | |
| 731 | const step_set = &reaction_set.values()[step_set_i]; | |
| 732 | var dirent_i: usize = 0; | |
| 733 | while (dirent_i < step_set.entries.len) { | |
| 734 | const generations = step_set.values(); | |
| 735 | if (generations[dirent_i] == w.generation) { | |
| 736 | dirent_i += 1; | |
| 737 | continue; | |
| 738 | } | |
| 739 | step_set.swapRemoveAt(dirent_i); | |
| 740 | } | |
| 741 | if (step_set.entries.len > 0) { | |
| 742 | step_set_i += 1; | |
| 743 | continue; | |
| 744 | } | |
| 745 | reaction_set.swapRemoveAt(step_set_i); | |
| 746 | } | |
| 747 | if (reaction_set.entries.len > 0) { | |
| 748 | i += 1; | |
| 749 | continue; | |
| 750 | } | |
| 751 | } | |
| 752 | ||
| 753 | // If the sub_path == "" then this patch has already the | |
| 754 | // dir fd that we need to use as the ident to remove the | |
| 755 | // event. If it was opened above with openat() then we need | |
| 756 | // to access that data via the dir_fd field. | |
| 757 | const path = w.dir_table.keys()[i]; | |
| 758 | const dir_fd = if (path.sub_path.len == 0) | |
| 759 | path.root_dir.handle.handle | |
| 760 | else | |
| 761 | handles.items(.dir_fd)[i]; | |
| 762 | assert(dir_fd != -1); | |
| 763 | ||
| 764 | // The changelist also needs to update the udata field of the last | |
| 765 | // event, since we are doing a swap remove, and we store the dir_table | |
| 766 | // index in the udata field. | |
| 767 | const last_dir_fd = fd: { | |
| 768 | const last_path = w.dir_table.keys()[handles.len - 1]; | |
| 769 | const last_dir_fd = if (last_path.sub_path.len == 0) | |
| 770 | last_path.root_dir.handle.handle | |
| 771 | else | |
| 772 | handles.items(.dir_fd)[handles.len - 1]; | |
| 773 | assert(last_dir_fd != -1); | |
| 774 | break :fd last_dir_fd; | |
| 775 | }; | |
| 776 | const changes = [_]posix.Kevent{ | |
| 777 | .{ | |
| 778 | .ident = @bitCast(@as(isize, dir_fd)), | |
| 779 | .filter = std.c.EVFILT.VNODE, | |
| 780 | .flags = EV.DELETE, | |
| 781 | .fflags = 0, | |
| 782 | .data = 0, | |
| 783 | .udata = i, | |
| 784 | }, | |
| 785 | .{ | |
| 786 | .ident = @bitCast(@as(isize, last_dir_fd)), | |
| 787 | .filter = std.c.EVFILT.VNODE, | |
| 788 | .flags = EV.ADD, | |
| 789 | .fflags = NOTE.DELETE | NOTE.WRITE | NOTE.RENAME | NOTE.REVOKE, | |
| 790 | .data = 0, | |
| 791 | .udata = i, | |
| 792 | }, | |
| 793 | }; | |
| 794 | const filtered_changes = if (i == handles.len - 1) changes[0..1] else &changes; | |
| 795 | _ = try Io.Kqueue.kevent(w.os.kq_fd, filtered_changes, &.{}, null); | |
| 796 | if (path.sub_path.len != 0) std.Io.Threaded.closeFd(dir_fd); | |
| 797 | ||
| 798 | w.dir_table.swapRemoveAt(i); | |
| 799 | handles.swapRemove(i); | |
| 800 | } | |
| 801 | w.generation +%= 1; | |
| 802 | } | |
| 803 | w.dir_count = w.dir_table.count(); | |
| 804 | } | |
| 805 | ||
| 806 | fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 807 | _ = io; | |
| 808 | var timespec_buffer: posix.timespec = undefined; | |
| 809 | var event_buffer: [100]posix.Kevent = undefined; | |
| 810 | var n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, timeout.toTimespec(×pec_buffer)); | |
| 811 | if (n == 0) return .timeout; | |
| 812 | const reaction_sets = w.os.handles.items(.rs); | |
| 813 | var any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], false); | |
| 814 | timespec_buffer = .{ .sec = 0, .nsec = 0 }; | |
| 815 | while (n == event_buffer.len) { | |
| 816 | n = try Io.Kqueue.kevent(w.os.kq_fd, &.{}, &event_buffer, ×pec_buffer); | |
| 817 | if (n == 0) break; | |
| 818 | any_dirty = markDirtySteps(gpa, reaction_sets, event_buffer[0..n], any_dirty); | |
| 819 | } | |
| 820 | return if (any_dirty) .dirty else .clean; | |
| 821 | } | |
| 822 | ||
| 823 | fn markDirtySteps( | |
| 824 | gpa: Allocator, | |
| 825 | reaction_sets: []ReactionSet, | |
| 826 | events: []const std.c.Kevent, | |
| 827 | start_any_dirty: bool, | |
| 828 | ) bool { | |
| 829 | var any_dirty = start_any_dirty; | |
| 830 | for (events) |event| { | |
| 831 | const index: usize = @intCast(event.udata); | |
| 832 | const reaction_set = &reaction_sets[index]; | |
| 833 | // If we knew the basename of the changed file, here we would | |
| 834 | // mark only the step set dirty, and possibly the glob set: | |
| 835 | //if (reaction_set.getPtr(".")) |glob_set| | |
| 836 | // any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); | |
| 837 | //if (reaction_set.getPtr(file_name)) |step_set| | |
| 838 | // any_dirty = markStepSetDirty(gpa, step_set, any_dirty); | |
| 839 | // However we don't know the file name so just mark all the | |
| 840 | // sets dirty for this directory. | |
| 841 | for (reaction_set.values()) |*step_set| { | |
| 842 | any_dirty = markStepSetDirty(gpa, step_set, any_dirty); | |
| 843 | } | |
| 844 | } | |
| 845 | return any_dirty; | |
| 846 | } | |
| 847 | }, | |
| 848 | .macos => struct { | |
| 849 | fse: FsEvents, | |
| 850 | ||
| 851 | fn init(cwd_path: []const u8) !Watch { | |
| 852 | return .{ | |
| 853 | .os = .{ .fse = try .init(cwd_path) }, | |
| 854 | .dir_count = 0, | |
| 855 | .dir_table = undefined, | |
| 856 | .generation = undefined, | |
| 857 | }; | |
| 858 | } | |
| 859 | fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 860 | try w.os.fse.setPaths(gpa, steps); | |
| 861 | w.dir_count = w.os.fse.watch_roots.len; | |
| 862 | } | |
| 863 | fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 864 | _ = io; | |
| 865 | return w.os.fse.wait(gpa, switch (timeout) { | |
| 866 | .none => null, | |
| 867 | .ms => |ms| @as(u64, ms) * std.time.ns_per_ms, | |
| 868 | }); | |
| 869 | } | |
| 870 | }, | |
| 871 | else => void, | |
| 872 | }; | |
| 873 | ||
| 874 | pub fn init(cwd_path: []const u8) !Watch { | |
| 875 | return Os.init(cwd_path); | |
| 876 | } | |
| 877 | ||
| 878 | pub const Match = struct { | |
| 879 | /// Relative to the watched directory, the file path that triggers this | |
| 880 | /// match. | |
| 881 | basename: []const u8, | |
| 882 | /// The step to re-run when file corresponding to `basename` is changed. | |
| 883 | step: *Step, | |
| 884 | ||
| 885 | pub const Context = struct { | |
| 886 | pub fn hash(self: Context, a: Match) u32 { | |
| 887 | _ = self; | |
| 888 | var hasher = Hash.init(0); | |
| 889 | std.hash.autoHash(&hasher, a.step); | |
| 890 | hasher.update(a.basename); | |
| 891 | return @truncate(hasher.final()); | |
| 892 | } | |
| 893 | pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { | |
| 894 | _ = self; | |
| 895 | _ = b_index; | |
| 896 | return a.step == b.step and std.mem.eql(u8, a.basename, b.basename); | |
| 897 | } | |
| 898 | }; | |
| 899 | }; | |
| 900 | ||
| 901 | fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { | |
| 902 | for (switch (builtin.os.tag) { | |
| 903 | .windows => w.os.handle_table.keys(), | |
| 904 | else => w.os.handle_table.values(), | |
| 905 | }) |item| { | |
| 906 | const reaction_set = switch (builtin.os.tag) { | |
| 907 | .linux, .windows => item.reaction_set, | |
| 908 | else => item, | |
| 909 | }; | |
| 910 | for (reaction_set.values()) |step_set| { | |
| 911 | for (step_set.keys()) |step| { | |
| 912 | _ = step.invalidateResult(gpa); | |
| 913 | } | |
| 914 | } | |
| 915 | } | |
| 916 | } | |
| 917 | ||
| 918 | fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool { | |
| 919 | var this_any_dirty = false; | |
| 920 | for (step_set.keys()) |step| { | |
| 921 | if (step.invalidateResult(gpa)) this_any_dirty = true; | |
| 922 | } | |
| 923 | return any_dirty or this_any_dirty; | |
| 924 | } | |
| 925 | ||
| 926 | pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { | |
| 927 | return Os.update(w, gpa, steps); | |
| 928 | } | |
| 929 | ||
| 930 | pub const Timeout = union(enum) { | |
| 931 | none, | |
| 932 | ms: u16, | |
| 933 | ||
| 934 | pub fn to_i32_ms(t: Timeout) i32 { | |
| 935 | return switch (t) { | |
| 936 | .none => -1, | |
| 937 | .ms => |ms| ms, | |
| 938 | }; | |
| 939 | } | |
| 940 | ||
| 941 | pub fn toTimespec(t: Timeout, buf: *std.posix.timespec) ?*std.posix.timespec { | |
| 942 | return switch (t) { | |
| 943 | .none => null, | |
| 944 | .ms => |ms_u16| { | |
| 945 | const ms: isize = ms_u16; | |
| 946 | buf.* = .{ | |
| 947 | .sec = @divTrunc(ms, std.time.ms_per_s), | |
| 948 | .nsec = @rem(ms, std.time.ms_per_s) * std.time.ns_per_ms, | |
| 949 | }; | |
| 950 | return buf; | |
| 951 | }, | |
| 952 | }; | |
| 953 | } | |
| 954 | }; | |
| 955 | ||
| 956 | pub const WaitResult = enum { | |
| 957 | timeout, | |
| 958 | /// File system watching triggered on files that were marked as inputs to at least one Step. | |
| 959 | /// Relevant steps have been marked dirty. | |
| 960 | dirty, | |
| 961 | /// File system watching triggered but none of the events were relevant to | |
| 962 | /// what we are listening to. There is nothing to do. | |
| 963 | clean, | |
| 964 | }; | |
| 965 | ||
| 966 | pub fn wait(w: *Watch, gpa: Allocator, io: Io, timeout: Timeout) !WaitResult { | |
| 967 | return Os.wait(w, gpa, io, timeout); | |
| 968 | } |
lib/std/Build/Watch/FsEvents.zig deleted-479
| ... | ... | @@ -1,479 +0,0 @@ |
| 1 | //! An implementation of file-system watching based on the `FSEventStream` API in macOS. | |
| 2 | //! While macOS supports kqueue, it does not allow detecting changes to files without | |
| 3 | //! placing watches on each individual file, meaning FD limits are reached incredibly | |
| 4 | //! quickly. The File System Events API works differently: it implements *recursive* | |
| 5 | //! directory watches, managed by a system service. Rather than being in libc, the API is | |
| 6 | //! exposed by the CoreServices framework. To avoid a compile dependency on the framework | |
| 7 | //! bundle, we dynamically load CoreServices with `std.DynLib`. | |
| 8 | //! | |
| 9 | //! While the logic in this file *is* specialized to `std.Build.Watch`, efforts have been | |
| 10 | //! made to keep that specialization to a minimum. Other use cases could be served with | |
| 11 | //! relatively minimal modifications to the `watch_paths` field and its usages (in | |
| 12 | //! particular the `setPaths` function). We avoid using the global GCD dispatch queue in | |
| 13 | //! favour of creating our own and synchronizing with an explicit semaphore, meaning this | |
| 14 | //! logic is thread-safe and does not affect process-global state. | |
| 15 | //! | |
| 16 | //! In theory, this API is quite good at avoiding filesystem race conditions. In practice, | |
| 17 | //! the logic that would avoid them is currently disabled, because the build system kind | |
| 18 | //! of relies on them at the time of writing to avoid redundant work -- see the comment at | |
| 19 | //! the top of `wait` for details. | |
| 20 | ||
| 21 | const enable_debug_logs = false; | |
| 22 | ||
| 23 | core_services: std.DynLib, | |
| 24 | resolved_symbols: ResolvedSymbols, | |
| 25 | ||
| 26 | paths_arena: std.heap.ArenaAllocator.State, | |
| 27 | /// The roots of the recursive watches. FSEvents has relatively small limits on the number | |
| 28 | /// of watched paths, so this slice must not be too long. The paths themselves are allocated | |
| 29 | /// into `paths_arena`, but this slice is allocated into the GPA. | |
| 30 | watch_roots: [][:0]const u8, | |
| 31 | /// All of the paths being watched. Value is the set of steps which depend on the file/directory. | |
| 32 | /// Keys and values are in `paths_arena`, but this map is allocated into the GPA. | |
| 33 | watch_paths: std.StringArrayHashMapUnmanaged([]const *std.Build.Step), | |
| 34 | ||
| 35 | /// The semaphore we use to block the thread calling `wait` until the callback determines a relevant | |
| 36 | /// event has occurred. This is retained across `wait` calls for simplicity and efficiency. | |
| 37 | waiting_semaphore: dispatch.semaphore_t, | |
| 38 | /// This dispatch queue is created by us and executes serially. It exists exclusively to trigger the | |
| 39 | /// callbacks of the FSEventStream we create. This is not in use outside of `wait`, but is retained | |
| 40 | /// across `wait` calls for simplicity and efficiency. | |
| 41 | dispatch_queue: dispatch.queue_t, | |
| 42 | /// In theory, this field avoids race conditions. In practice, it is essentially unused at the time | |
| 43 | /// of writing. See the comment at the start of `wait` for details. | |
| 44 | since_event: FSEventStreamEventId, | |
| 45 | ||
| 46 | cwd_path: []const u8, | |
| 47 | ||
| 48 | /// All of the symbols we pull from the `dlopen`ed CoreServices framework. If any of these symbols | |
| 49 | /// is not present, `init` will close the framework and return an error. | |
| 50 | const ResolvedSymbols = struct { | |
| 51 | FSEventStreamCreate: *const fn ( | |
| 52 | allocator: CFAllocatorRef, | |
| 53 | callback: FSEventStreamCallback, | |
| 54 | ctx: ?*const FSEventStreamContext, | |
| 55 | paths_to_watch: CFArrayRef, | |
| 56 | since_when: FSEventStreamEventId, | |
| 57 | latency: CFTimeInterval, | |
| 58 | flags: FSEventStreamCreateFlags, | |
| 59 | ) callconv(.c) FSEventStreamRef, | |
| 60 | FSEventStreamSetDispatchQueue: *const fn (stream: FSEventStreamRef, queue: dispatch.queue_t) callconv(.c) void, | |
| 61 | FSEventStreamStart: *const fn (stream: FSEventStreamRef) callconv(.c) bool, | |
| 62 | FSEventStreamStop: *const fn (stream: FSEventStreamRef) callconv(.c) void, | |
| 63 | FSEventStreamInvalidate: *const fn (stream: FSEventStreamRef) callconv(.c) void, | |
| 64 | FSEventStreamRelease: *const fn (stream: FSEventStreamRef) callconv(.c) void, | |
| 65 | FSEventStreamGetLatestEventId: *const fn (stream: ConstFSEventStreamRef) callconv(.c) FSEventStreamEventId, | |
| 66 | FSEventsGetCurrentEventId: *const fn () callconv(.c) FSEventStreamEventId, | |
| 67 | CFRelease: *const fn (cf: *const anyopaque) callconv(.c) void, | |
| 68 | CFArrayCreate: *const fn ( | |
| 69 | allocator: CFAllocatorRef, | |
| 70 | values: [*]const usize, | |
| 71 | num_values: CFIndex, | |
| 72 | call_backs: ?*const CFArrayCallBacks, | |
| 73 | ) callconv(.c) CFArrayRef, | |
| 74 | CFStringCreateWithCString: *const fn ( | |
| 75 | alloc: CFAllocatorRef, | |
| 76 | c_str: [*:0]const u8, | |
| 77 | encoding: CFStringEncoding, | |
| 78 | ) callconv(.c) CFStringRef, | |
| 79 | CFAllocatorCreate: *const fn (allocator: CFAllocatorRef, context: *const CFAllocatorContext) callconv(.c) CFAllocatorRef, | |
| 80 | kCFAllocatorUseContext: *const CFAllocatorRef, | |
| 81 | }; | |
| 82 | ||
| 83 | pub fn init(cwd_path: []const u8) error{ OpenFrameworkFailed, MissingCoreServicesSymbol, SystemResources }!FsEvents { | |
| 84 | var core_services = std.DynLib.open("/System/Library/Frameworks/CoreServices.framework/CoreServices") catch | |
| 85 | return error.OpenFrameworkFailed; | |
| 86 | errdefer core_services.close(); | |
| 87 | ||
| 88 | var resolved_symbols: ResolvedSymbols = undefined; | |
| 89 | inline for (@typeInfo(ResolvedSymbols).@"struct".fields) |f| { | |
| 90 | @field(resolved_symbols, f.name) = core_services.lookup(f.type, f.name) orelse return error.MissingCoreServicesSymbol; | |
| 91 | } | |
| 92 | ||
| 93 | return .{ | |
| 94 | .core_services = core_services, | |
| 95 | .resolved_symbols = resolved_symbols, | |
| 96 | .paths_arena = .{}, | |
| 97 | .watch_roots = &.{}, | |
| 98 | .watch_paths = .empty, | |
| 99 | .waiting_semaphore = dispatch.semaphore_create(0) orelse return error.SystemResources, | |
| 100 | .dispatch_queue = dispatch.queue_create("zig-watch", .SERIAL()) orelse return error.SystemResources, | |
| 101 | // Not `.since_now`, because this means we can init `FsEvents` *before* we do work in order | |
| 102 | // to notice any changes which happened during said work. | |
| 103 | .since_event = resolved_symbols.FSEventsGetCurrentEventId(), | |
| 104 | .cwd_path = cwd_path, | |
| 105 | }; | |
| 106 | } | |
| 107 | ||
| 108 | pub fn deinit(fse: *FsEvents, gpa: Allocator, io: Io) void { | |
| 109 | fse.waiting_semaphore.as_object().release(); | |
| 110 | fse.dispatch_queue.as_object().release(); | |
| 111 | fse.core_services.close(io); | |
| 112 | ||
| 113 | gpa.free(fse.watch_roots); | |
| 114 | fse.watch_paths.deinit(gpa); | |
| 115 | { | |
| 116 | var paths_arena = fse.paths_arena.promote(gpa); | |
| 117 | paths_arena.deinit(); | |
| 118 | } | |
| 119 | } | |
| 120 | ||
| 121 | pub fn setPaths(fse: *FsEvents, gpa: Allocator, steps: []const *std.Build.Step) !void { | |
| 122 | var paths_arena_instance = fse.paths_arena.promote(gpa); | |
| 123 | defer fse.paths_arena = paths_arena_instance.state; | |
| 124 | const paths_arena = paths_arena_instance.allocator(); | |
| 125 | ||
| 126 | var need_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; | |
| 127 | defer need_dirs.deinit(gpa); | |
| 128 | ||
| 129 | fse.watch_paths.clearRetainingCapacity(); | |
| 130 | ||
| 131 | // We take `step` by pointer for a slight memory optimization in a moment. | |
| 132 | for (steps) |*step| { | |
| 133 | for (step.*.inputs.table.keys(), step.*.inputs.table.values()) |path, *files| { | |
| 134 | const resolved_dir = try std.fs.path.resolvePosix(paths_arena, &.{ | |
| 135 | fse.cwd_path, path.root_dir.path orelse ".", path.sub_path, | |
| 136 | }); | |
| 137 | try need_dirs.put(gpa, resolved_dir, {}); | |
| 138 | for (files.items) |file_name| { | |
| 139 | const watch_path = if (std.mem.eql(u8, file_name, ".")) | |
| 140 | resolved_dir | |
| 141 | else | |
| 142 | try std.fs.path.join(paths_arena, &.{ resolved_dir, file_name }); | |
| 143 | const gop = try fse.watch_paths.getOrPut(gpa, watch_path); | |
| 144 | if (gop.found_existing) { | |
| 145 | const old_steps = gop.value_ptr.*; | |
| 146 | const new_steps = try paths_arena.alloc(*std.Build.Step, old_steps.len + 1); | |
| 147 | @memcpy(new_steps[0..old_steps.len], old_steps); | |
| 148 | new_steps[old_steps.len] = step.*; | |
| 149 | gop.value_ptr.* = new_steps; | |
| 150 | } else { | |
| 151 | // This is why we captured `step` by pointer! We can avoid allocating a slice of one | |
| 152 | // step in the arena in the common case where a file is referenced by only one step. | |
| 153 | gop.value_ptr.* = step[0..1]; | |
| 154 | } | |
| 155 | } | |
| 156 | } | |
| 157 | } | |
| 158 | ||
| 159 | { | |
| 160 | // There's no point looking at directories inside other ones (e.g. "/foo" and "/foo/bar"). | |
| 161 | // To eliminate these, we'll re-add directories in order of path length with a redundancy check. | |
| 162 | const old_dirs = try gpa.dupe([]const u8, need_dirs.keys()); | |
| 163 | defer gpa.free(old_dirs); | |
| 164 | std.mem.sort([]const u8, old_dirs, {}, struct { | |
| 165 | fn lessThan(ctx: void, a: []const u8, b: []const u8) bool { | |
| 166 | ctx; | |
| 167 | return std.mem.lessThan(u8, a, b); | |
| 168 | } | |
| 169 | }.lessThan); | |
| 170 | need_dirs.clearRetainingCapacity(); | |
| 171 | for (old_dirs) |dir_path| { | |
| 172 | var it: std.fs.path.ComponentIterator(.posix, u8) = .init(dir_path); | |
| 173 | while (it.next()) |component| { | |
| 174 | if (need_dirs.contains(component.path)) { | |
| 175 | // this path is '/foo/bar/qux', but '/foo' or '/foo/bar' was already added | |
| 176 | break; | |
| 177 | } | |
| 178 | } else { | |
| 179 | need_dirs.putAssumeCapacityNoClobber(dir_path, {}); | |
| 180 | } | |
| 181 | } | |
| 182 | } | |
| 183 | ||
| 184 | // `need_dirs` is now a set of directories to watch with no redundancy. In practice, this is very | |
| 185 | // likely to have reduced it to a quite small set (e.g. it'll typically coalesce a full `src/` | |
| 186 | // directory into one entry). However, the FSEventStream API has a fairly low undocumented limit | |
| 187 | // on total watches (supposedly 4096), so we should handle the case where we exceed it. To be | |
| 188 | // safe, because this API can be a little unpredictable, we'll cap ourselves a little *below* | |
| 189 | // that known limit. | |
| 190 | if (need_dirs.count() > 2048) { | |
| 191 | // Fallback: watch the whole filesystem. This is excessive, but... it *works* :P | |
| 192 | if (enable_debug_logs) watch_log.debug("too many dirs; recursively watching root", .{}); | |
| 193 | fse.watch_roots = try gpa.realloc(fse.watch_roots, 1); | |
| 194 | fse.watch_roots[0] = "/"; | |
| 195 | } else { | |
| 196 | fse.watch_roots = try gpa.realloc(fse.watch_roots, need_dirs.count()); | |
| 197 | for (fse.watch_roots, need_dirs.keys()) |*out, in| { | |
| 198 | out.* = try paths_arena.dupeSentinel(u8, in, 0); | |
| 199 | } | |
| 200 | } | |
| 201 | if (enable_debug_logs) { | |
| 202 | watch_log.debug("watching {d} paths using {d} recursive watches:", .{ fse.watch_paths.count(), fse.watch_roots.len }); | |
| 203 | for (fse.watch_roots) |dir_path| { | |
| 204 | watch_log.debug("- '{s}'", .{dir_path}); | |
| 205 | } | |
| 206 | } | |
| 207 | } | |
| 208 | ||
| 209 | pub fn wait(fse: *FsEvents, gpa: Allocator, timeout_ns: ?u64) error{ OutOfMemory, StartFailed }!std.Build.Watch.WaitResult { | |
| 210 | if (fse.watch_roots.len == 0) @panic("nothing to watch"); | |
| 211 | ||
| 212 | const rs = fse.resolved_symbols; | |
| 213 | ||
| 214 | // At the time of writing, using `since_event` in the obvious way causes redundant rebuilds | |
| 215 | // to occur, because one step modifies a file which is an input to another step. The solution | |
| 216 | // to this problem will probably be either: | |
| 217 | // | |
| 218 | // a) Don't include the output of one step as a watch input of another; only mark external | |
| 219 | // files as watch inputs. Or... | |
| 220 | // | |
| 221 | // b) Note the current event ID when a step begins, and disregard events preceding that ID | |
| 222 | // when considering whether to dirty that step in `eventCallback`. | |
| 223 | // | |
| 224 | // For now, to avoid the redundant rebuilds, we bypass this `since_event` mechanism. This does | |
| 225 | // introduce race conditions, but the other `std.Build.Watch` implementations suffer from those | |
| 226 | // too at the time of writing, so this is kind of expected. | |
| 227 | fse.since_event = .since_now; | |
| 228 | ||
| 229 | const cf_allocator = rs.CFAllocatorCreate(rs.kCFAllocatorUseContext.*, &.{ | |
| 230 | .version = 0, | |
| 231 | .info = @constCast(&gpa), | |
| 232 | .retain = null, | |
| 233 | .release = null, | |
| 234 | .copy_description = null, | |
| 235 | .allocate = &cf_alloc_callbacks.allocate, | |
| 236 | .reallocate = &cf_alloc_callbacks.reallocate, | |
| 237 | .deallocate = &cf_alloc_callbacks.deallocate, | |
| 238 | .preferred_size = null, | |
| 239 | }) orelse return error.OutOfMemory; | |
| 240 | defer rs.CFRelease(cf_allocator); | |
| 241 | ||
| 242 | const cf_paths = try gpa.alloc(?CFStringRef, fse.watch_roots.len); | |
| 243 | @memset(cf_paths, null); | |
| 244 | defer { | |
| 245 | for (cf_paths) |o| if (o) |p| rs.CFRelease(p); | |
| 246 | gpa.free(cf_paths); | |
| 247 | } | |
| 248 | for (fse.watch_roots, cf_paths) |raw_path, *cf_path| { | |
| 249 | cf_path.* = rs.CFStringCreateWithCString(cf_allocator, raw_path, .utf8); | |
| 250 | } | |
| 251 | const cf_paths_array = rs.CFArrayCreate(cf_allocator, @ptrCast(cf_paths), @intCast(cf_paths.len), null); | |
| 252 | defer rs.CFRelease(cf_paths_array); | |
| 253 | ||
| 254 | const callback_ctx: EventCallbackCtx = .{ | |
| 255 | .fse = fse, | |
| 256 | .gpa = gpa, | |
| 257 | }; | |
| 258 | const event_stream = rs.FSEventStreamCreate( | |
| 259 | null, | |
| 260 | &eventCallback, | |
| 261 | &.{ | |
| 262 | .version = 0, | |
| 263 | .info = @constCast(&callback_ctx), | |
| 264 | .retain = null, | |
| 265 | .release = null, | |
| 266 | .copy_description = null, | |
| 267 | }, | |
| 268 | cf_paths_array, | |
| 269 | fse.since_event, | |
| 270 | 0.05, // 0.05s latency; higher values increase efficiency by coalescing more events | |
| 271 | .{ .watch_root = true, .file_events = true }, | |
| 272 | ); | |
| 273 | defer rs.FSEventStreamRelease(event_stream); | |
| 274 | rs.FSEventStreamSetDispatchQueue(event_stream, fse.dispatch_queue); | |
| 275 | defer rs.FSEventStreamInvalidate(event_stream); | |
| 276 | if (!rs.FSEventStreamStart(event_stream)) return error.StartFailed; | |
| 277 | defer rs.FSEventStreamStop(event_stream); | |
| 278 | const result = fse.waiting_semaphore.wait(timeout: { | |
| 279 | const ns = timeout_ns orelse break :timeout .FOREVER; | |
| 280 | break :timeout .time(.NOW, @intCast(ns)); | |
| 281 | }); | |
| 282 | return switch (result) { | |
| 283 | 0 => .dirty, | |
| 284 | else => .timeout, | |
| 285 | }; | |
| 286 | } | |
| 287 | ||
| 288 | const cf_alloc_callbacks = struct { | |
| 289 | const log = std.log.scoped(.cf_alloc); | |
| 290 | fn allocate(size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque { | |
| 291 | if (enable_debug_logs) log.debug("allocate {d}", .{size}); | |
| 292 | _ = hint; | |
| 293 | const gpa: *const Allocator = @ptrCast(@alignCast(info)); | |
| 294 | const mem = gpa.alignedAlloc(u8, .of(usize), @intCast(size + @sizeOf(usize))) catch return null; | |
| 295 | const metadata: *usize = @ptrCast(mem); | |
| 296 | metadata.* = @intCast(size); | |
| 297 | return mem[@sizeOf(usize)..].ptr; | |
| 298 | } | |
| 299 | fn reallocate(ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque { | |
| 300 | if (enable_debug_logs) log.debug("reallocate @{*} {d}", .{ ptr, new_size }); | |
| 301 | _ = hint; | |
| 302 | if (ptr == null or new_size == 0) return null; // not a bug: documentation explicitly states that realloc on NULL should return NULL | |
| 303 | const gpa: *const Allocator = @ptrCast(@alignCast(info)); | |
| 304 | const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize)); | |
| 305 | const old_size = @as(*const usize, @ptrCast(old_base)).*; | |
| 306 | const old_mem = old_base[0 .. old_size + @sizeOf(usize)]; | |
| 307 | const new_mem = gpa.realloc(old_mem, @intCast(new_size + @sizeOf(usize))) catch return null; | |
| 308 | const metadata: *usize = @ptrCast(new_mem); | |
| 309 | metadata.* = @intCast(new_size); | |
| 310 | return new_mem[@sizeOf(usize)..].ptr; | |
| 311 | } | |
| 312 | fn deallocate(ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void { | |
| 313 | if (enable_debug_logs) log.debug("deallocate @{*}", .{ptr}); | |
| 314 | const gpa: *const Allocator = @ptrCast(@alignCast(info)); | |
| 315 | const old_base: [*]align(@alignOf(usize)) u8 = @alignCast(@as([*]u8, @ptrCast(ptr)) - @sizeOf(usize)); | |
| 316 | const old_size = @as(*const usize, @ptrCast(old_base)).*; | |
| 317 | const old_mem = old_base[0 .. old_size + @sizeOf(usize)]; | |
| 318 | gpa.free(old_mem); | |
| 319 | } | |
| 320 | }; | |
| 321 | ||
| 322 | const EventCallbackCtx = struct { | |
| 323 | fse: *FsEvents, | |
| 324 | gpa: Allocator, | |
| 325 | }; | |
| 326 | ||
| 327 | fn eventCallback( | |
| 328 | stream: ConstFSEventStreamRef, | |
| 329 | client_callback_info: ?*anyopaque, | |
| 330 | num_events: usize, | |
| 331 | events_paths_ptr: *anyopaque, | |
| 332 | events_flags_ptr: [*]const FSEventStreamEventFlags, | |
| 333 | events_ids_ptr: [*]const FSEventStreamEventId, | |
| 334 | ) callconv(.c) void { | |
| 335 | const ctx: *const EventCallbackCtx = @ptrCast(@alignCast(client_callback_info)); | |
| 336 | const fse = ctx.fse; | |
| 337 | const gpa = ctx.gpa; | |
| 338 | const rs = fse.resolved_symbols; | |
| 339 | const events_paths_ptr_casted: [*]const [*:0]const u8 = @ptrCast(@alignCast(events_paths_ptr)); | |
| 340 | const events_paths = events_paths_ptr_casted[0..num_events]; | |
| 341 | const events_ids = events_ids_ptr[0..num_events]; | |
| 342 | const events_flags = events_flags_ptr[0..num_events]; | |
| 343 | var any_dirty = false; | |
| 344 | for (events_paths, events_ids, events_flags) |event_path_nts, event_id, event_flags| { | |
| 345 | _ = event_id; | |
| 346 | if (event_flags.history_done) continue; // sentinel | |
| 347 | const event_path = std.mem.span(event_path_nts); | |
| 348 | switch (event_flags.must_scan_sub_dirs) { | |
| 349 | false => { | |
| 350 | if (fse.watch_paths.get(event_path)) |steps| { | |
| 351 | assert(steps.len > 0); | |
| 352 | for (steps) |s| { | |
| 353 | if (s.invalidateResult(gpa)) any_dirty = true; | |
| 354 | } | |
| 355 | } | |
| 356 | if (std.fs.path.dirname(event_path)) |event_dirname| { | |
| 357 | // Modifying '/foo/bar' triggers the watch on '/foo'. | |
| 358 | if (fse.watch_paths.get(event_dirname)) |steps| { | |
| 359 | assert(steps.len > 0); | |
| 360 | for (steps) |s| { | |
| 361 | if (s.invalidateResult(gpa)) any_dirty = true; | |
| 362 | } | |
| 363 | } | |
| 364 | } | |
| 365 | }, | |
| 366 | true => { | |
| 367 | // This is unlikely, but can occasionally happen when bottlenecked: events have been | |
| 368 | // coalesced into one. We want to see if any of these events are actually relevant | |
| 369 | // to us. The only way we can reasonably do that in this rare edge case is iterate | |
| 370 | // the watch paths and see if any is under this directory. That's acceptable because | |
| 371 | // we would otherwise kick off a rebuild which would be clearing those paths anyway. | |
| 372 | const changed_path = std.fs.path.dirname(event_path) orelse event_path; | |
| 373 | for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| { | |
| 374 | if (dirStartsWith(watching_path, changed_path)) { | |
| 375 | for (steps) |s| { | |
| 376 | if (s.invalidateResult(gpa)) any_dirty = true; | |
| 377 | } | |
| 378 | } | |
| 379 | } | |
| 380 | }, | |
| 381 | } | |
| 382 | } | |
| 383 | if (any_dirty) { | |
| 384 | fse.since_event = rs.FSEventStreamGetLatestEventId(stream); | |
| 385 | _ = fse.waiting_semaphore.signal(); | |
| 386 | } | |
| 387 | } | |
| 388 | fn dirStartsWith(path: []const u8, prefix: []const u8) bool { | |
| 389 | if (std.mem.eql(u8, path, prefix)) return true; | |
| 390 | if (!std.mem.startsWith(u8, path, prefix)) return false; | |
| 391 | if (path[prefix.len] != '/') return false; // `path` is `/foo/barx`, `prefix` is `/foo/bar` | |
| 392 | return true; // `path` is `/foo/bar/...`, `prefix` is `/foo/bar` | |
| 393 | } | |
| 394 | ||
| 395 | const CFAllocatorRef = ?*const opaque {}; | |
| 396 | const CFArrayRef = *const opaque {}; | |
| 397 | const CFStringRef = *const opaque {}; | |
| 398 | const CFTimeInterval = f64; | |
| 399 | const CFIndex = i32; | |
| 400 | const CFOptionFlags = enum(u32) { _ }; | |
| 401 | const CFAllocatorRetainCallBack = *const fn (info: ?*const anyopaque) callconv(.c) *const anyopaque; | |
| 402 | const CFAllocatorReleaseCallBack = *const fn (info: ?*const anyopaque) callconv(.c) void; | |
| 403 | const CFAllocatorCopyDescriptionCallBack = *const fn (info: ?*const anyopaque) callconv(.c) CFStringRef; | |
| 404 | const CFAllocatorAllocateCallBack = *const fn (alloc_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque; | |
| 405 | const CFAllocatorReallocateCallBack = *const fn (ptr: ?*anyopaque, new_size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) ?*const anyopaque; | |
| 406 | const CFAllocatorDeallocateCallBack = *const fn (ptr: *anyopaque, info: ?*const anyopaque) callconv(.c) void; | |
| 407 | const CFAllocatorPreferredSizeCallBack = *const fn (size: CFIndex, hint: CFOptionFlags, info: ?*const anyopaque) callconv(.c) CFIndex; | |
| 408 | const CFAllocatorContext = extern struct { | |
| 409 | version: CFIndex, | |
| 410 | info: ?*anyopaque, | |
| 411 | retain: ?CFAllocatorRetainCallBack, | |
| 412 | release: ?CFAllocatorReleaseCallBack, | |
| 413 | copy_description: ?CFAllocatorCopyDescriptionCallBack, | |
| 414 | allocate: CFAllocatorAllocateCallBack, | |
| 415 | reallocate: ?CFAllocatorReallocateCallBack, | |
| 416 | deallocate: ?CFAllocatorDeallocateCallBack, | |
| 417 | preferred_size: ?CFAllocatorPreferredSizeCallBack, | |
| 418 | }; | |
| 419 | const CFArrayCallBacks = opaque {}; | |
| 420 | const CFStringEncoding = enum(u32) { | |
| 421 | invalid_id = std.math.maxInt(u32), | |
| 422 | mac_roman = 0, | |
| 423 | windows_latin_1 = 0x500, | |
| 424 | iso_latin_1 = 0x201, | |
| 425 | next_step_latin = 0xB01, | |
| 426 | ascii = 0x600, | |
| 427 | unicode = 0x100, | |
| 428 | utf8 = 0x8000100, | |
| 429 | non_lossy_ascii = 0xBFF, | |
| 430 | }; | |
| 431 | ||
| 432 | const FSEventStreamRef = *opaque {}; | |
| 433 | const ConstFSEventStreamRef = *const @typeInfo(FSEventStreamRef).pointer.child; | |
| 434 | const FSEventStreamCallback = *const fn ( | |
| 435 | stream: ConstFSEventStreamRef, | |
| 436 | client_callback_info: ?*anyopaque, | |
| 437 | num_events: usize, | |
| 438 | event_paths: *anyopaque, | |
| 439 | event_flags: [*]const FSEventStreamEventFlags, | |
| 440 | event_ids: [*]const FSEventStreamEventId, | |
| 441 | ) callconv(.c) void; | |
| 442 | const FSEventStreamContext = extern struct { | |
| 443 | version: CFIndex, | |
| 444 | info: ?*anyopaque, | |
| 445 | retain: ?CFAllocatorRetainCallBack, | |
| 446 | release: ?CFAllocatorReleaseCallBack, | |
| 447 | copy_description: ?CFAllocatorCopyDescriptionCallBack, | |
| 448 | }; | |
| 449 | const FSEventStreamEventId = enum(u64) { | |
| 450 | since_now = std.math.maxInt(u64), | |
| 451 | _, | |
| 452 | }; | |
| 453 | const FSEventStreamCreateFlags = packed struct(u32) { | |
| 454 | use_cf_types: bool = false, | |
| 455 | no_defer: bool = false, | |
| 456 | watch_root: bool = false, | |
| 457 | ignore_self: bool = false, | |
| 458 | file_events: bool = false, | |
| 459 | _: u27 = 0, | |
| 460 | }; | |
| 461 | const FSEventStreamEventFlags = packed struct(u32) { | |
| 462 | must_scan_sub_dirs: bool, | |
| 463 | user_dropped: bool, | |
| 464 | kernel_dropped: bool, | |
| 465 | event_ids_wrapped: bool, | |
| 466 | history_done: bool, | |
| 467 | root_changed: bool, | |
| 468 | mount: bool, | |
| 469 | unmount: bool, | |
| 470 | _: u24 = 0, | |
| 471 | }; | |
| 472 | ||
| 473 | const dispatch = std.c.dispatch; | |
| 474 | const std = @import("std"); | |
| 475 | const Io = std.Io; | |
| 476 | const assert = std.debug.assert; | |
| 477 | const Allocator = std.mem.Allocator; | |
| 478 | const watch_log = std.log.scoped(.watch); | |
| 479 | const FsEvents = @This(); |
lib/std/Build/WebServer.zig deleted-926
| ... | ... | @@ -1,926 +0,0 @@ |
| 1 | gpa: Allocator, | |
| 2 | graph: *const Build.Graph, | |
| 3 | all_steps: []const *Build.Step, | |
| 4 | listen_address: net.IpAddress, | |
| 5 | root_prog_node: std.Progress.Node, | |
| 6 | watch: bool, | |
| 7 | ||
| 8 | tcp_server: ?net.Server, | |
| 9 | serve_task: ?Io.Future(Io.Cancelable!void), | |
| 10 | ||
| 11 | /// Uses `Io.Clock.awake`. | |
| 12 | base_timestamp: Io.Timestamp, | |
| 13 | /// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`. | |
| 14 | step_names_trailing: []u8, | |
| 15 | ||
| 16 | /// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps. | |
| 17 | /// Accessed atomically. | |
| 18 | step_status_bits: []u8, | |
| 19 | ||
| 20 | fuzz: ?Fuzz, | |
| 21 | time_report_mutex: Io.Mutex, | |
| 22 | time_report_msgs: [][]u8, | |
| 23 | time_report_update_times: []i64, | |
| 24 | ||
| 25 | build_status: std.atomic.Value(abi.BuildStatus), | |
| 26 | /// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate` | |
| 27 | /// to increment this value. Each client thread waits for this increment with `Io.futexWaitTimeout`, so | |
| 28 | /// `notifyUpdate` will wake those threads. Updates are sent on a short interval regardless, so it | |
| 29 | /// is recommended to only use `notifyUpdate` for changes which the user should see immediately. For | |
| 30 | /// instance, we do not call `notifyUpdate` when the number of "unique runs" in the fuzzer changes, | |
| 31 | /// because this value changes quickly so this would result in constantly spamming all clients with | |
| 32 | /// an unreasonable number of packets. | |
| 33 | update_id: std.atomic.Value(u32), | |
| 34 | ||
| 35 | runner_request_mutex: Io.Mutex, | |
| 36 | runner_request_ready_cond: Io.Condition, | |
| 37 | runner_request_empty_cond: Io.Condition, | |
| 38 | runner_request: ?RunnerRequest, | |
| 39 | ||
| 40 | /// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates | |
| 41 | /// on a fixed interval of this many milliseconds. | |
| 42 | const default_update_interval_ms = 500; | |
| 43 | ||
| 44 | pub const base_clock: Io.Clock = .awake; | |
| 45 | ||
| 46 | /// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`. | |
| 47 | pub fn notifyUpdate(ws: *WebServer) void { | |
| 48 | _ = ws.update_id.rmw(.Add, 1, .release); | |
| 49 | ws.graph.io.futexWake(u32, &ws.update_id.raw, 16); | |
| 50 | } | |
| 51 | ||
| 52 | pub const Options = struct { | |
| 53 | gpa: Allocator, | |
| 54 | graph: *const std.Build.Graph, | |
| 55 | all_steps: []const *Build.Step, | |
| 56 | root_prog_node: std.Progress.Node, | |
| 57 | watch: bool, | |
| 58 | listen_address: net.IpAddress, | |
| 59 | base_timestamp: Io.Clock.Timestamp, | |
| 60 | }; | |
| 61 | pub fn init(opts: Options) WebServer { | |
| 62 | // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` | |
| 63 | // instead of threads, so that the web server can function in single-threaded builds. | |
| 64 | comptime assert(!builtin.single_threaded); | |
| 65 | assert(opts.base_timestamp.clock == base_clock); | |
| 66 | ||
| 67 | const all_steps = opts.all_steps; | |
| 68 | ||
| 69 | const step_names_trailing = opts.gpa.alloc(u8, len: { | |
| 70 | var name_bytes: usize = 0; | |
| 71 | for (all_steps) |step| name_bytes += step.name.len; | |
| 72 | break :len name_bytes + all_steps.len * 4; | |
| 73 | }) catch @panic("out of memory"); | |
| 74 | { | |
| 75 | const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]); | |
| 76 | var idx: usize = all_steps.len * 4; | |
| 77 | for (all_steps, step_name_lens) |step, *name_len| { | |
| 78 | name_len.* = @intCast(step.name.len); | |
| 79 | @memcpy(step_names_trailing[idx..][0..step.name.len], step.name); | |
| 80 | idx += step.name.len; | |
| 81 | } | |
| 82 | assert(idx == step_names_trailing.len); | |
| 83 | } | |
| 84 | ||
| 85 | const step_status_bits = opts.gpa.alloc( | |
| 86 | u8, | |
| 87 | std.math.divCeil(usize, all_steps.len, 4) catch unreachable, | |
| 88 | ) catch @panic("out of memory"); | |
| 89 | @memset(step_status_bits, 0); | |
| 90 | ||
| 91 | const time_reports_len: usize = if (opts.graph.time_report) all_steps.len else 0; | |
| 92 | const time_report_msgs = opts.gpa.alloc([]u8, time_reports_len) catch @panic("out of memory"); | |
| 93 | const time_report_update_times = opts.gpa.alloc(i64, time_reports_len) catch @panic("out of memory"); | |
| 94 | @memset(time_report_msgs, &.{}); | |
| 95 | @memset(time_report_update_times, std.math.minInt(i64)); | |
| 96 | ||
| 97 | return .{ | |
| 98 | .gpa = opts.gpa, | |
| 99 | .graph = opts.graph, | |
| 100 | .all_steps = all_steps, | |
| 101 | .listen_address = opts.listen_address, | |
| 102 | .root_prog_node = opts.root_prog_node, | |
| 103 | .watch = opts.watch, | |
| 104 | ||
| 105 | .tcp_server = null, | |
| 106 | .serve_task = null, | |
| 107 | ||
| 108 | .base_timestamp = opts.base_timestamp.raw, | |
| 109 | .step_names_trailing = step_names_trailing, | |
| 110 | ||
| 111 | .step_status_bits = step_status_bits, | |
| 112 | ||
| 113 | .fuzz = null, | |
| 114 | .time_report_mutex = .init, | |
| 115 | .time_report_msgs = time_report_msgs, | |
| 116 | .time_report_update_times = time_report_update_times, | |
| 117 | ||
| 118 | .build_status = .init(.idle), | |
| 119 | .update_id = .init(0), | |
| 120 | ||
| 121 | .runner_request_mutex = .init, | |
| 122 | .runner_request_ready_cond = .init, | |
| 123 | .runner_request_empty_cond = .init, | |
| 124 | .runner_request = null, | |
| 125 | }; | |
| 126 | } | |
| 127 | pub fn deinit(ws: *WebServer) void { | |
| 128 | const gpa = ws.gpa; | |
| 129 | const io = ws.graph.io; | |
| 130 | ||
| 131 | gpa.free(ws.step_names_trailing); | |
| 132 | gpa.free(ws.step_status_bits); | |
| 133 | ||
| 134 | if (ws.fuzz) |*f| f.deinit(); | |
| 135 | for (ws.time_report_msgs) |msg| gpa.free(msg); | |
| 136 | gpa.free(ws.time_report_msgs); | |
| 137 | gpa.free(ws.time_report_update_times); | |
| 138 | ||
| 139 | if (ws.serve_task) |t| { | |
| 140 | if (ws.tcp_server) |*s| s.stream.close(io); | |
| 141 | t.await(); | |
| 142 | } | |
| 143 | if (ws.tcp_server) |*s| s.deinit(); | |
| 144 | ||
| 145 | gpa.free(ws.step_names_trailing); | |
| 146 | } | |
| 147 | pub fn start(ws: *WebServer) error{AlreadyReported}!void { | |
| 148 | assert(ws.tcp_server == null); | |
| 149 | assert(ws.serve_task == null); | |
| 150 | const io = ws.graph.io; | |
| 151 | ||
| 152 | ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| { | |
| 153 | log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err }); | |
| 154 | return error.AlreadyReported; | |
| 155 | }; | |
| 156 | ws.serve_task = io.concurrent(serve, .{ws}) catch |err| { | |
| 157 | log.err("unable to spawn web server thread: {t}", .{err}); | |
| 158 | ws.tcp_server.?.deinit(io); | |
| 159 | ws.tcp_server = null; | |
| 160 | return error.AlreadyReported; | |
| 161 | }; | |
| 162 | ||
| 163 | log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.socket.address}); | |
| 164 | if (ws.listen_address.getPort() == 0) { | |
| 165 | log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.socket.address}); | |
| 166 | } | |
| 167 | } | |
| 168 | fn serve(ws: *WebServer) Io.Cancelable!void { | |
| 169 | const io = ws.graph.io; | |
| 170 | var group: Io.Group = .init; | |
| 171 | defer group.cancel(io); | |
| 172 | while (true) { | |
| 173 | var stream = ws.tcp_server.?.accept(io) catch |err| switch (err) { | |
| 174 | error.Canceled => |e| return e, | |
| 175 | else => |e| { | |
| 176 | log.err("failed to accept connection: {t}", .{e}); | |
| 177 | return; | |
| 178 | }, | |
| 179 | }; | |
| 180 | group.concurrent(io, accept, .{ ws, stream }) catch |err| { | |
| 181 | log.err("unable to spawn connection thread: {t}", .{err}); | |
| 182 | stream.close(io); | |
| 183 | continue; | |
| 184 | }; | |
| 185 | } | |
| 186 | } | |
| 187 | ||
| 188 | pub fn startBuild(ws: *WebServer) void { | |
| 189 | if (ws.fuzz) |*fuzz| { | |
| 190 | fuzz.deinit(); | |
| 191 | ws.fuzz = null; | |
| 192 | } | |
| 193 | for (ws.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic); | |
| 194 | ws.build_status.store(.running, .monotonic); | |
| 195 | ws.notifyUpdate(); | |
| 196 | } | |
| 197 | ||
| 198 | pub fn updateStepStatus(ws: *WebServer, step: *Build.Step, new_status: abi.StepUpdate.Status) void { | |
| 199 | const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { | |
| 200 | if (s == step) break @intCast(i); | |
| 201 | } else unreachable; | |
| 202 | const ptr = &ws.step_status_bits[step_idx / 4]; | |
| 203 | const bit_offset: u3 = @intCast((step_idx % 4) * 2); | |
| 204 | const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset); | |
| 205 | const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset; | |
| 206 | _ = @atomicRmw(u8, ptr, .Xor, mask, .monotonic); | |
| 207 | ws.notifyUpdate(); | |
| 208 | } | |
| 209 | ||
| 210 | pub fn finishBuild(ws: *WebServer, opts: struct { | |
| 211 | fuzz: bool, | |
| 212 | }) void { | |
| 213 | if (opts.fuzz) { | |
| 214 | switch (builtin.os.tag) { | |
| 215 | // Current implementation depends on two things that need to be ported to Windows: | |
| 216 | // * Memory-mapping to share data between the fuzzer and build runner. | |
| 217 | // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving | |
| 218 | // many addresses to source locations). | |
| 219 | .windows => std.process.fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), | |
| 220 | else => {}, | |
| 221 | } | |
| 222 | if (@bitSizeOf(usize) != 64) { | |
| 223 | // Current implementation depends on posix.mmap()'s second | |
| 224 | // parameter, `length: usize`, being compatible with file system's | |
| 225 | // u64 return value. This is not the case on 32-bit platforms. | |
| 226 | // Affects or affected by issues #5185, #22523, and #22464. | |
| 227 | std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); | |
| 228 | } | |
| 229 | ||
| 230 | assert(ws.fuzz == null); | |
| 231 | ||
| 232 | ws.build_status.store(.fuzz_init, .monotonic); | |
| 233 | ws.notifyUpdate(); | |
| 234 | ||
| 235 | ws.fuzz = Fuzz.init( | |
| 236 | ws.gpa, | |
| 237 | ws.graph.io, | |
| 238 | ws.all_steps, | |
| 239 | ws.root_prog_node, | |
| 240 | .{ .forever = .{ .ws = ws } }, | |
| 241 | ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)}); | |
| 242 | ws.fuzz.?.start(); | |
| 243 | } | |
| 244 | ||
| 245 | ws.build_status.store(if (ws.watch) .watching else .idle, .monotonic); | |
| 246 | ws.notifyUpdate(); | |
| 247 | } | |
| 248 | ||
| 249 | pub fn now(s: *const WebServer) i64 { | |
| 250 | const io = s.graph.io; | |
| 251 | const ts = base_clock.now(io); | |
| 252 | return @intCast(s.base_timestamp.durationTo(ts).toNanoseconds()); | |
| 253 | } | |
| 254 | ||
| 255 | fn accept(ws: *WebServer, stream: net.Stream) void { | |
| 256 | const io = ws.graph.io; | |
| 257 | defer { | |
| 258 | // `net.Stream.close` wants to helpfully overwrite `stream` with | |
| 259 | // `undefined`, but it cannot do so since it is an immutable parameter. | |
| 260 | var copy = stream; | |
| 261 | copy.close(io); | |
| 262 | } | |
| 263 | var send_buffer: [4096]u8 = undefined; | |
| 264 | var recv_buffer: [4096]u8 = undefined; | |
| 265 | var connection_reader = stream.reader(io, &recv_buffer); | |
| 266 | var connection_writer = stream.writer(io, &send_buffer); | |
| 267 | var server: http.Server = .init(&connection_reader.interface, &connection_writer.interface); | |
| 268 | ||
| 269 | while (true) { | |
| 270 | var request = server.receiveHead() catch |err| switch (err) { | |
| 271 | error.HttpConnectionClosing => return, | |
| 272 | else => return log.err("failed to receive http request: {t}", .{err}), | |
| 273 | }; | |
| 274 | switch (request.upgradeRequested()) { | |
| 275 | .websocket => |opt_key| { | |
| 276 | const key = opt_key orelse return log.err("missing websocket key", .{}); | |
| 277 | var web_socket = request.respondWebSocket(.{ .key = key }) catch { | |
| 278 | return log.err("failed to respond web socket: {t}", .{connection_writer.err.?}); | |
| 279 | }; | |
| 280 | ws.serveWebSocket(&web_socket) catch |err| { | |
| 281 | log.err("failed to serve websocket: {t}", .{err}); | |
| 282 | return; | |
| 283 | }; | |
| 284 | comptime unreachable; | |
| 285 | }, | |
| 286 | .other => |name| return log.err("unknown upgrade request: {s}", .{name}), | |
| 287 | .none => { | |
| 288 | ws.serveRequest(&request) catch |err| switch (err) { | |
| 289 | error.AlreadyReported => return, | |
| 290 | else => { | |
| 291 | log.err("failed to serve '{s}': {t}", .{ request.head.target, err }); | |
| 292 | return; | |
| 293 | }, | |
| 294 | }; | |
| 295 | }, | |
| 296 | } | |
| 297 | } | |
| 298 | } | |
| 299 | ||
| 300 | fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { | |
| 301 | const io = ws.graph.io; | |
| 302 | ||
| 303 | var prev_build_status = ws.build_status.load(.monotonic); | |
| 304 | ||
| 305 | const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len); | |
| 306 | defer ws.gpa.free(prev_step_status_bits); | |
| 307 | for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| { | |
| 308 | copy.* = @atomicLoad(u8, shared, .monotonic); | |
| 309 | } | |
| 310 | ||
| 311 | var recv_thread = try io.concurrent(recvWebSocketMessages, .{ ws, sock }); | |
| 312 | defer recv_thread.cancel(io); | |
| 313 | ||
| 314 | { | |
| 315 | const hello_header: abi.Hello = .{ | |
| 316 | .status = prev_build_status, | |
| 317 | .flags = .{ | |
| 318 | .time_report = ws.graph.time_report, | |
| 319 | }, | |
| 320 | .timestamp = ws.now(), | |
| 321 | .steps_len = @intCast(ws.all_steps.len), | |
| 322 | }; | |
| 323 | var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits }; | |
| 324 | try sock.writeMessageVec(&bufs, .binary); | |
| 325 | } | |
| 326 | ||
| 327 | var prev_fuzz: Fuzz.Previous = .init; | |
| 328 | var prev_time: i64 = std.math.minInt(i64); | |
| 329 | while (true) { | |
| 330 | const start_time = ws.now(); | |
| 331 | const start_update_id = ws.update_id.load(.acquire); | |
| 332 | ||
| 333 | if (ws.fuzz) |*fuzz| { | |
| 334 | try fuzz.sendUpdate(sock, &prev_fuzz); | |
| 335 | } | |
| 336 | ||
| 337 | { | |
| 338 | try ws.time_report_mutex.lock(io); | |
| 339 | defer ws.time_report_mutex.unlock(io); | |
| 340 | for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| { | |
| 341 | if (update_time <= prev_time) continue; | |
| 342 | // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so | |
| 343 | // that we don't hold up the build system on the client accepting this packet. | |
| 344 | const owned_msg = try ws.gpa.dupe(u8, msg); | |
| 345 | defer ws.gpa.free(owned_msg); | |
| 346 | // Temporarily unlock, then re-lock after the message is sent. | |
| 347 | ws.time_report_mutex.unlock(io); | |
| 348 | defer ws.time_report_mutex.lockUncancelable(io); | |
| 349 | try sock.writeMessage(owned_msg, .binary); | |
| 350 | } | |
| 351 | } | |
| 352 | ||
| 353 | { | |
| 354 | const build_status = ws.build_status.load(.monotonic); | |
| 355 | if (build_status != prev_build_status) { | |
| 356 | prev_build_status = build_status; | |
| 357 | const msg: abi.StatusUpdate = .{ .new = build_status }; | |
| 358 | try sock.writeMessage(@ptrCast(&msg), .binary); | |
| 359 | } | |
| 360 | } | |
| 361 | ||
| 362 | for (prev_step_status_bits, ws.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| { | |
| 363 | const cur_byte = @atomicLoad(u8, shared, .monotonic); | |
| 364 | if (prev_byte.* == cur_byte) continue; | |
| 365 | const cur: [4]abi.StepUpdate.Status = .{ | |
| 366 | @enumFromInt(@as(u2, @truncate(cur_byte >> 0))), | |
| 367 | @enumFromInt(@as(u2, @truncate(cur_byte >> 2))), | |
| 368 | @enumFromInt(@as(u2, @truncate(cur_byte >> 4))), | |
| 369 | @enumFromInt(@as(u2, @truncate(cur_byte >> 6))), | |
| 370 | }; | |
| 371 | const prev: [4]abi.StepUpdate.Status = .{ | |
| 372 | @enumFromInt(@as(u2, @truncate(prev_byte.* >> 0))), | |
| 373 | @enumFromInt(@as(u2, @truncate(prev_byte.* >> 2))), | |
| 374 | @enumFromInt(@as(u2, @truncate(prev_byte.* >> 4))), | |
| 375 | @enumFromInt(@as(u2, @truncate(prev_byte.* >> 6))), | |
| 376 | }; | |
| 377 | for (cur, prev, byte_idx * 4..) |cur_status, prev_status, step_idx| { | |
| 378 | const msg: abi.StepUpdate = .{ .step_idx = @intCast(step_idx), .bits = .{ .status = cur_status } }; | |
| 379 | if (cur_status != prev_status) try sock.writeMessage(@ptrCast(&msg), .binary); | |
| 380 | } | |
| 381 | prev_byte.* = cur_byte; | |
| 382 | } | |
| 383 | ||
| 384 | prev_time = start_time; | |
| 385 | ||
| 386 | const old_cp = io.swapCancelProtection(.blocked); | |
| 387 | defer _ = io.swapCancelProtection(old_cp); | |
| 388 | io.futexWaitTimeout( | |
| 389 | u32, | |
| 390 | &ws.update_id.raw, | |
| 391 | start_update_id, | |
| 392 | .{ .duration = .{ | |
| 393 | .clock = .awake, | |
| 394 | .raw = .fromMilliseconds(default_update_interval_ms), | |
| 395 | } }, | |
| 396 | ) catch |err| switch (err) { | |
| 397 | error.Canceled => unreachable, | |
| 398 | }; | |
| 399 | } | |
| 400 | } | |
| 401 | fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void { | |
| 402 | const io = ws.graph.io; | |
| 403 | ||
| 404 | while (true) { | |
| 405 | const msg = sock.readSmallMessage() catch return; | |
| 406 | if (msg.opcode != .binary) continue; | |
| 407 | if (msg.data.len == 0) continue; | |
| 408 | const tag: abi.ToServerTag = @enumFromInt(msg.data[0]); | |
| 409 | switch (tag) { | |
| 410 | _ => continue, | |
| 411 | .rebuild => while (true) { | |
| 412 | ws.runner_request_mutex.lock(io) catch |err| switch (err) { | |
| 413 | error.Canceled => return, | |
| 414 | }; | |
| 415 | defer ws.runner_request_mutex.unlock(io); | |
| 416 | if (ws.runner_request == null) { | |
| 417 | ws.runner_request = .rebuild; | |
| 418 | ws.runner_request_ready_cond.signal(io); | |
| 419 | break; | |
| 420 | } | |
| 421 | ws.runner_request_empty_cond.wait(io, &ws.runner_request_mutex) catch return; | |
| 422 | }, | |
| 423 | } | |
| 424 | } | |
| 425 | } | |
| 426 | ||
| 427 | fn serveRequest(ws: *WebServer, req: *http.Server.Request) !void { | |
| 428 | // Strip an optional leading '/debug' component from the request. | |
| 429 | const target: []const u8, const debug: bool = target: { | |
| 430 | if (mem.eql(u8, req.head.target, "/debug")) break :target .{ "/", true }; | |
| 431 | if (mem.eql(u8, req.head.target, "/debug/")) break :target .{ "/", true }; | |
| 432 | if (mem.startsWith(u8, req.head.target, "/debug/")) break :target .{ req.head.target["/debug".len..], true }; | |
| 433 | break :target .{ req.head.target, false }; | |
| 434 | }; | |
| 435 | ||
| 436 | if (mem.eql(u8, target, "/")) return serveLibFile(ws, req, "build-web/index.html", "text/html"); | |
| 437 | if (mem.eql(u8, target, "/main.js")) return serveLibFile(ws, req, "build-web/main.js", "application/javascript"); | |
| 438 | if (mem.eql(u8, target, "/style.css")) return serveLibFile(ws, req, "build-web/style.css", "text/css"); | |
| 439 | if (mem.eql(u8, target, "/time_report.css")) return serveLibFile(ws, req, "build-web/time_report.css", "text/css"); | |
| 440 | if (mem.eql(u8, target, "/main.wasm")) return serveClientWasm(ws, req, if (debug) .Debug else .ReleaseFast); | |
| 441 | ||
| 442 | if (ws.fuzz) |*fuzz| { | |
| 443 | if (mem.eql(u8, target, "/sources.tar")) return fuzz.serveSourcesTar(req); | |
| 444 | } | |
| 445 | ||
| 446 | try req.respond("not found", .{ | |
| 447 | .status = .not_found, | |
| 448 | .extra_headers = &.{ | |
| 449 | .{ .name = "Content-Type", .value = "text/plain" }, | |
| 450 | }, | |
| 451 | }); | |
| 452 | } | |
| 453 | ||
| 454 | fn serveLibFile( | |
| 455 | ws: *WebServer, | |
| 456 | request: *http.Server.Request, | |
| 457 | sub_path: []const u8, | |
| 458 | content_type: []const u8, | |
| 459 | ) !void { | |
| 460 | return serveFile(ws, request, .{ | |
| 461 | .root_dir = ws.graph.zig_lib_directory, | |
| 462 | .sub_path = sub_path, | |
| 463 | }, content_type); | |
| 464 | } | |
| 465 | fn serveClientWasm( | |
| 466 | ws: *WebServer, | |
| 467 | req: *http.Server.Request, | |
| 468 | optimize_mode: std.builtin.OptimizeMode, | |
| 469 | ) !void { | |
| 470 | var arena_state: std.heap.ArenaAllocator = .init(ws.gpa); | |
| 471 | defer arena_state.deinit(); | |
| 472 | const arena = arena_state.allocator(); | |
| 473 | ||
| 474 | // We always rebuild the wasm on-the-fly, so that if it is edited the user can just refresh the page. | |
| 475 | const bin_path = try buildClientWasm(ws, arena, optimize_mode); | |
| 476 | return serveFile(ws, req, bin_path, "application/wasm"); | |
| 477 | } | |
| 478 | ||
| 479 | pub fn serveFile( | |
| 480 | ws: *WebServer, | |
| 481 | request: *http.Server.Request, | |
| 482 | path: Cache.Path, | |
| 483 | content_type: []const u8, | |
| 484 | ) !void { | |
| 485 | const gpa = ws.gpa; | |
| 486 | const io = ws.graph.io; | |
| 487 | // The desired API is actually sendfile, which will require enhancing http.Server. | |
| 488 | // We load the file with every request so that the user can make changes to the file | |
| 489 | // and refresh the HTML page without restarting this server. | |
| 490 | const file_contents = path.root_dir.handle.readFileAlloc(io, path.sub_path, gpa, .limited(10 * 1024 * 1024)) catch |err| { | |
| 491 | log.err("failed to read '{f}': {t}", .{ path, err }); | |
| 492 | return error.AlreadyReported; | |
| 493 | }; | |
| 494 | defer gpa.free(file_contents); | |
| 495 | try request.respond(file_contents, .{ | |
| 496 | .extra_headers = &.{ | |
| 497 | .{ .name = "Content-Type", .value = content_type }, | |
| 498 | cache_control_header, | |
| 499 | }, | |
| 500 | }); | |
| 501 | } | |
| 502 | pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void { | |
| 503 | const graph = ws.graph; | |
| 504 | const io = graph.io; | |
| 505 | ||
| 506 | var send_buffer: [0x4000]u8 = undefined; | |
| 507 | var response = try request.respondStreaming(&send_buffer, .{ | |
| 508 | .respond_options = .{ | |
| 509 | .extra_headers = &.{ | |
| 510 | .{ .name = "Content-Type", .value = "application/x-tar" }, | |
| 511 | cache_control_header, | |
| 512 | }, | |
| 513 | }, | |
| 514 | }); | |
| 515 | ||
| 516 | var archiver: std.tar.Writer = .{ .underlying_writer = &response.writer }; | |
| 517 | ||
| 518 | for (paths) |path| { | |
| 519 | var file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| { | |
| 520 | log.err("failed to open '{f}': {s}", .{ path, @errorName(err) }); | |
| 521 | continue; | |
| 522 | }; | |
| 523 | defer file.close(io); | |
| 524 | const stat = try file.stat(io); | |
| 525 | var read_buffer: [1024]u8 = undefined; | |
| 526 | var file_reader: Io.File.Reader = .initSize(file, io, &read_buffer, stat.size); | |
| 527 | ||
| 528 | // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can | |
| 529 | // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI: | |
| 530 | // it turns out the WASM treats the first path component as the module name, typically | |
| 531 | // resulting in modules named "" and "src". The compiler needs to tell the build system | |
| 532 | // about the module graph so that the build system can correctly encode this information in | |
| 533 | // the tar file. | |
| 534 | // | |
| 535 | // Additionally, this needs to ensure that all path separators for both prefix and | |
| 536 | // sub_path are using the POSIX-style `/` on platforms that don't use it as their native | |
| 537 | // path separator. | |
| 538 | archiver.prefix = path.root_dir.path orelse graph.cache.cwd; | |
| 539 | try archiver.writeFile(path.sub_path, &file_reader, @intCast(stat.mtime.toSeconds())); | |
| 540 | } | |
| 541 | ||
| 542 | // intentionally not calling `archiver.finishPedantically` | |
| 543 | try response.end(); | |
| 544 | } | |
| 545 | ||
| 546 | fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.OptimizeMode) !Cache.Path { | |
| 547 | const root_name = "build-web"; | |
| 548 | const arch_os_abi = "wasm32-freestanding"; | |
| 549 | const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; | |
| 550 | ||
| 551 | const gpa = ws.gpa; | |
| 552 | const graph = ws.graph; | |
| 553 | const io = graph.io; | |
| 554 | ||
| 555 | const main_src_path: Cache.Path = .{ | |
| 556 | .root_dir = graph.zig_lib_directory, | |
| 557 | .sub_path = "build-web/main.zig", | |
| 558 | }; | |
| 559 | const walk_src_path: Cache.Path = .{ | |
| 560 | .root_dir = graph.zig_lib_directory, | |
| 561 | .sub_path = "docs/wasm/Walk.zig", | |
| 562 | }; | |
| 563 | const html_render_src_path: Cache.Path = .{ | |
| 564 | .root_dir = graph.zig_lib_directory, | |
| 565 | .sub_path = "docs/wasm/html_render.zig", | |
| 566 | }; | |
| 567 | ||
| 568 | var argv: std.ArrayList([]const u8) = .empty; | |
| 569 | ||
| 570 | try argv.appendSlice(arena, &.{ | |
| 571 | graph.zig_exe, "build-exe", // | |
| 572 | "-fno-entry", // | |
| 573 | "-O", @tagName(optimize), // | |
| 574 | "-target", arch_os_abi, // | |
| 575 | "-mcpu", cpu_features, // | |
| 576 | "--cache-dir", graph.global_cache_root.path orelse ".", // | |
| 577 | "--global-cache-dir", graph.global_cache_root.path orelse ".", // | |
| 578 | "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // | |
| 579 | "--name", root_name, // | |
| 580 | "-rdynamic", // | |
| 581 | "-fsingle-threaded", // | |
| 582 | "--dep", "Walk", // | |
| 583 | "--dep", "html_render", // | |
| 584 | try std.fmt.allocPrint(arena, "-Mroot={f}", .{main_src_path}), // | |
| 585 | try std.fmt.allocPrint(arena, "-MWalk={f}", .{walk_src_path}), // | |
| 586 | "--dep", "Walk", // | |
| 587 | try std.fmt.allocPrint(arena, "-Mhtml_render={f}", .{html_render_src_path}), // | |
| 588 | "--listen=-", | |
| 589 | }); | |
| 590 | ||
| 591 | var child = try std.process.spawn(io, .{ | |
| 592 | .argv = argv.items, | |
| 593 | .environ_map = &graph.environ_map, | |
| 594 | .stdin = .pipe, | |
| 595 | .stdout = .pipe, | |
| 596 | .stderr = .pipe, | |
| 597 | }); | |
| 598 | defer child.kill(io); | |
| 599 | ||
| 600 | var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited }); | |
| 601 | defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {}; | |
| 602 | ||
| 603 | var stdout_buffer: [512]u8 = undefined; | |
| 604 | var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer); | |
| 605 | const stdout = &stdout_reader.interface; | |
| 606 | ||
| 607 | { | |
| 608 | var w = child.stdin.?.writer(io, &.{}); | |
| 609 | w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch |err| switch (err) { | |
| 610 | error.WriteFailed => return w.err.?, | |
| 611 | }; | |
| 612 | w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch |err| switch (err) { | |
| 613 | error.WriteFailed => return w.err.?, | |
| 614 | }; | |
| 615 | } | |
| 616 | ||
| 617 | const Header = std.zig.Server.Message.Header; | |
| 618 | ||
| 619 | var result: ?Cache.Path = null; | |
| 620 | var result_error_bundle = std.zig.ErrorBundle.empty; | |
| 621 | var body_buffer: std.ArrayList(u8) = .empty; | |
| 622 | defer body_buffer.deinit(gpa); | |
| 623 | ||
| 624 | while (true) { | |
| 625 | const header = stdout.takeStruct(Header, .little) catch |err| switch (err) { | |
| 626 | error.ReadFailed => |e| return e, | |
| 627 | error.EndOfStream => break, | |
| 628 | }; | |
| 629 | body_buffer.clearRetainingCapacity(); | |
| 630 | try stdout.appendExact(gpa, &body_buffer, header.bytes_len); | |
| 631 | const body = body_buffer.items; | |
| 632 | ||
| 633 | switch (header.tag) { | |
| 634 | .zig_version => { | |
| 635 | if (!std.mem.eql(u8, builtin.zig_version_string, body)) { | |
| 636 | return error.ZigProtocolVersionMismatch; | |
| 637 | } | |
| 638 | }, | |
| 639 | .error_bundle => { | |
| 640 | result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body); | |
| 641 | }, | |
| 642 | .emit_digest => { | |
| 643 | const EmitDigest = std.zig.Server.Message.EmitDigest; | |
| 644 | const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body); | |
| 645 | if (!ebp_hdr.flags.cache_hit) { | |
| 646 | log.info("source changes detected; rebuilt wasm component", .{}); | |
| 647 | } | |
| 648 | const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; | |
| 649 | result = .{ | |
| 650 | .root_dir = graph.global_cache_root, | |
| 651 | .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), | |
| 652 | }; | |
| 653 | }, | |
| 654 | else => {}, // ignore other messages | |
| 655 | } | |
| 656 | } | |
| 657 | ||
| 658 | const stderr_contents = try stderr_task.await(io); | |
| 659 | if (stderr_contents.len > 0) { | |
| 660 | std.debug.print("{s}", .{stderr_contents}); | |
| 661 | } | |
| 662 | ||
| 663 | // Send EOF to stdin. | |
| 664 | child.stdin.?.close(io); | |
| 665 | child.stdin = null; | |
| 666 | ||
| 667 | switch (try child.wait(io)) { | |
| 668 | .exited => |code| { | |
| 669 | if (code != 0) { | |
| 670 | log.err( | |
| 671 | "the following command exited with error code {d}:\n{s}", | |
| 672 | .{ code, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, | |
| 673 | ); | |
| 674 | return error.WasmCompilationFailed; | |
| 675 | } | |
| 676 | }, | |
| 677 | .signal => |sig| { | |
| 678 | log.err( | |
| 679 | "the following command terminated with signal {t}:\n{s}", | |
| 680 | .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, | |
| 681 | ); | |
| 682 | return error.WasmCompilationFailed; | |
| 683 | }, | |
| 684 | .stopped => |sig| { | |
| 685 | log.err( | |
| 686 | "the following command stopped unexpectedly with signal {t}:\n{s}", | |
| 687 | .{ sig, try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items) }, | |
| 688 | ); | |
| 689 | return error.WasmCompilationFailed; | |
| 690 | }, | |
| 691 | .unknown => { | |
| 692 | log.err( | |
| 693 | "the following command terminated unexpectedly:\n{s}", | |
| 694 | .{try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items)}, | |
| 695 | ); | |
| 696 | return error.WasmCompilationFailed; | |
| 697 | }, | |
| 698 | } | |
| 699 | ||
| 700 | if (result_error_bundle.errorMessageCount() > 0) { | |
| 701 | try result_error_bundle.renderToStderr(io, .{}, .auto); | |
| 702 | log.err("the following command failed with {d} compilation errors:\n{s}", .{ | |
| 703 | result_error_bundle.errorMessageCount(), | |
| 704 | try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), | |
| 705 | }); | |
| 706 | return error.WasmCompilationFailed; | |
| 707 | } | |
| 708 | ||
| 709 | const base_path = result orelse { | |
| 710 | log.err("child process failed to report result\n{s}", .{ | |
| 711 | try Build.Step.allocPrintCmd(arena, .inherit, null, argv.items), | |
| 712 | }); | |
| 713 | return error.WasmCompilationFailed; | |
| 714 | }; | |
| 715 | const bin_name = try std.zig.binNameAlloc(arena, .{ | |
| 716 | .root_name = root_name, | |
| 717 | .target = &(std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ | |
| 718 | .arch_os_abi = arch_os_abi, | |
| 719 | .cpu_features = cpu_features, | |
| 720 | }) catch unreachable) catch unreachable), | |
| 721 | .output_mode = .Exe, | |
| 722 | }); | |
| 723 | return base_path.join(arena, bin_name); | |
| 724 | } | |
| 725 | ||
| 726 | fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 { | |
| 727 | var file_reader: Io.File.Reader = .initStreaming(file, io, &.{}); | |
| 728 | return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { | |
| 729 | error.ReadFailed => return file_reader.err.?, | |
| 730 | else => |e| return e, | |
| 731 | }; | |
| 732 | } | |
| 733 | ||
| 734 | pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { | |
| 735 | compile: *Build.Step.Compile, | |
| 736 | ||
| 737 | use_llvm: bool, | |
| 738 | stats: abi.time_report.CompileResult.Stats, | |
| 739 | ns_total: u64, | |
| 740 | ||
| 741 | llvm_pass_timings_len: u32, | |
| 742 | files_len: u32, | |
| 743 | decls_len: u32, | |
| 744 | ||
| 745 | /// The trailing data of `abi.time_report.CompileResult`, except the step name. | |
| 746 | trailing: []const u8, | |
| 747 | }) void { | |
| 748 | const gpa = ws.gpa; | |
| 749 | const io = ws.graph.io; | |
| 750 | ||
| 751 | const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { | |
| 752 | if (s == &opts.compile.step) break @intCast(i); | |
| 753 | } else unreachable; | |
| 754 | ||
| 755 | const old_buf = old: { | |
| 756 | ws.time_report_mutex.lock(io) catch return; | |
| 757 | defer ws.time_report_mutex.unlock(io); | |
| 758 | const old = ws.time_report_msgs[step_idx]; | |
| 759 | ws.time_report_msgs[step_idx] = &.{}; | |
| 760 | break :old old; | |
| 761 | }; | |
| 762 | const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory"); | |
| 763 | ||
| 764 | const out_header: *align(1) abi.time_report.CompileResult = @ptrCast(buf[0..@sizeOf(abi.time_report.CompileResult)]); | |
| 765 | out_header.* = .{ | |
| 766 | .step_idx = step_idx, | |
| 767 | .flags = .{ | |
| 768 | .use_llvm = opts.use_llvm, | |
| 769 | }, | |
| 770 | .stats = opts.stats, | |
| 771 | .ns_total = opts.ns_total, | |
| 772 | .llvm_pass_timings_len = opts.llvm_pass_timings_len, | |
| 773 | .files_len = opts.files_len, | |
| 774 | .decls_len = opts.decls_len, | |
| 775 | }; | |
| 776 | @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing); | |
| 777 | ||
| 778 | { | |
| 779 | ws.time_report_mutex.lock(io) catch return; | |
| 780 | defer ws.time_report_mutex.unlock(io); | |
| 781 | assert(ws.time_report_msgs[step_idx].len == 0); | |
| 782 | ws.time_report_msgs[step_idx] = buf; | |
| 783 | ws.time_report_update_times[step_idx] = ws.now(); | |
| 784 | } | |
| 785 | ws.notifyUpdate(); | |
| 786 | } | |
| 787 | ||
| 788 | pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, duration: Io.Duration) void { | |
| 789 | const gpa = ws.gpa; | |
| 790 | const io = ws.graph.io; | |
| 791 | ||
| 792 | const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { | |
| 793 | if (s == step) break @intCast(i); | |
| 794 | } else unreachable; | |
| 795 | ||
| 796 | const old_buf = old: { | |
| 797 | ws.time_report_mutex.lock(io) catch return; | |
| 798 | defer ws.time_report_mutex.unlock(io); | |
| 799 | const old = ws.time_report_msgs[step_idx]; | |
| 800 | ws.time_report_msgs[step_idx] = &.{}; | |
| 801 | break :old old; | |
| 802 | }; | |
| 803 | const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory"); | |
| 804 | const out: *align(1) abi.time_report.GenericResult = @ptrCast(buf); | |
| 805 | out.* = .{ | |
| 806 | .step_idx = step_idx, | |
| 807 | .ns_total = @intCast(duration.toNanoseconds()), | |
| 808 | }; | |
| 809 | { | |
| 810 | ws.time_report_mutex.lock(io) catch return; | |
| 811 | defer ws.time_report_mutex.unlock(io); | |
| 812 | assert(ws.time_report_msgs[step_idx].len == 0); | |
| 813 | ws.time_report_msgs[step_idx] = buf; | |
| 814 | ws.time_report_update_times[step_idx] = ws.now(); | |
| 815 | } | |
| 816 | ws.notifyUpdate(); | |
| 817 | } | |
| 818 | ||
| 819 | pub fn updateTimeReportRunTest( | |
| 820 | ws: *WebServer, | |
| 821 | run: *Build.Step.Run, | |
| 822 | tests: *const Build.Step.Run.CachedTestMetadata, | |
| 823 | ns_per_test: []const u64, | |
| 824 | ) void { | |
| 825 | const gpa = ws.gpa; | |
| 826 | const io = ws.graph.io; | |
| 827 | ||
| 828 | const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { | |
| 829 | if (s == &run.step) break @intCast(i); | |
| 830 | } else unreachable; | |
| 831 | ||
| 832 | assert(tests.names.len == ns_per_test.len); | |
| 833 | const tests_len: u32 = @intCast(tests.names.len); | |
| 834 | ||
| 835 | const new_len: u64 = len: { | |
| 836 | var names_len: u64 = 0; | |
| 837 | for (0..tests_len) |i| { | |
| 838 | names_len += tests.testName(@intCast(i)).len + 1; | |
| 839 | } | |
| 840 | break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len; | |
| 841 | }; | |
| 842 | const old_buf = old: { | |
| 843 | ws.time_report_mutex.lock(io) catch return; | |
| 844 | defer ws.time_report_mutex.unlock(io); | |
| 845 | const old = ws.time_report_msgs[step_idx]; | |
| 846 | ws.time_report_msgs[step_idx] = &.{}; | |
| 847 | break :old old; | |
| 848 | }; | |
| 849 | const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory"); | |
| 850 | ||
| 851 | const out_header: *align(1) abi.time_report.RunTestResult = @ptrCast(buf[0..@sizeOf(abi.time_report.RunTestResult)]); | |
| 852 | out_header.* = .{ | |
| 853 | .step_idx = step_idx, | |
| 854 | .tests_len = tests_len, | |
| 855 | }; | |
| 856 | var offset: usize = @sizeOf(abi.time_report.RunTestResult); | |
| 857 | const ns_per_test_out: []align(1) u64 = @ptrCast(buf[offset..][0 .. tests_len * 8]); | |
| 858 | @memcpy(ns_per_test_out, ns_per_test); | |
| 859 | offset += tests_len * 8; | |
| 860 | for (0..tests_len) |i| { | |
| 861 | const name = tests.testName(@intCast(i)); | |
| 862 | @memcpy(buf[offset..][0..name.len], name); | |
| 863 | buf[offset..][name.len] = 0; | |
| 864 | offset += name.len + 1; | |
| 865 | } | |
| 866 | assert(offset == buf.len); | |
| 867 | ||
| 868 | { | |
| 869 | ws.time_report_mutex.lock(io) catch return; | |
| 870 | defer ws.time_report_mutex.unlock(io); | |
| 871 | assert(ws.time_report_msgs[step_idx].len == 0); | |
| 872 | ws.time_report_msgs[step_idx] = buf; | |
| 873 | ws.time_report_update_times[step_idx] = ws.now(); | |
| 874 | } | |
| 875 | ws.notifyUpdate(); | |
| 876 | } | |
| 877 | ||
| 878 | const RunnerRequest = union(enum) { | |
| 879 | rebuild, | |
| 880 | }; | |
| 881 | pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { | |
| 882 | const io = ws.graph.io; | |
| 883 | ws.runner_request_mutex.lock(io) catch return; | |
| 884 | defer ws.runner_request_mutex.unlock(io); | |
| 885 | if (ws.runner_request) |req| { | |
| 886 | ws.runner_request = null; | |
| 887 | ws.runner_request_empty_cond.signal(); | |
| 888 | return req; | |
| 889 | } | |
| 890 | return null; | |
| 891 | } | |
| 892 | pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest { | |
| 893 | const io = ws.graph.io; | |
| 894 | try ws.runner_request_mutex.lock(io); | |
| 895 | defer ws.runner_request_mutex.unlock(io); | |
| 896 | while (true) { | |
| 897 | if (ws.runner_request) |req| { | |
| 898 | ws.runner_request = null; | |
| 899 | ws.runner_request_empty_cond.signal(io); | |
| 900 | return req; | |
| 901 | } | |
| 902 | try ws.runner_request_ready_cond.wait(io, &ws.runner_request_mutex); | |
| 903 | } | |
| 904 | } | |
| 905 | ||
| 906 | const cache_control_header: http.Header = .{ | |
| 907 | .name = "Cache-Control", | |
| 908 | .value = "max-age=0, must-revalidate", | |
| 909 | }; | |
| 910 | ||
| 911 | const builtin = @import("builtin"); | |
| 912 | ||
| 913 | const std = @import("std"); | |
| 914 | const Io = std.Io; | |
| 915 | const net = std.Io.net; | |
| 916 | const assert = std.debug.assert; | |
| 917 | const mem = std.mem; | |
| 918 | const log = std.log.scoped(.web_server); | |
| 919 | const Allocator = std.mem.Allocator; | |
| 920 | const Build = std.Build; | |
| 921 | const Cache = Build.Cache; | |
| 922 | const Fuzz = Build.Fuzz; | |
| 923 | const abi = Build.abi; | |
| 924 | const http = std.http; | |
| 925 | ||
| 926 | const WebServer = @This(); |
lib/std/zig.zig-2
| ... | ... | @@ -11,8 +11,6 @@ const Writer = std.Io.Writer; |
| 11 | 11 | |
| 12 | 12 | const tokenizer = @import("zig/tokenizer.zig"); |
| 13 | 13 | |
| 14 | /// The serialized output of configure phase ingested by make phase. | |
| 15 | pub const Configuration = @import("zig/Configuration.zig"); | |
| 16 | 14 | pub const ErrorBundle = @import("zig/ErrorBundle.zig"); |
| 17 | 15 | pub const Server = @import("zig/Server.zig"); |
| 18 | 16 | pub const Client = @import("zig/Client.zig"); |
lib/std/zig/Configuration.zig+230-10
| ... | ... | @@ -3,22 +3,240 @@ const Configuration = @This(); |
| 3 | 3 | const std = @import("../std.zig"); |
| 4 | 4 | const Io = std.Io; |
| 5 | 5 | const Allocator = std.mem.Allocator; |
| 6 | const assert = std.debug.assert; | |
| 6 | 7 | |
| 7 | 8 | string_bytes: []u8, |
| 8 | 9 | steps: []Step, |
| 9 | 10 | path_deps_base: []Path.Base, |
| 10 | 11 | path_deps_sub: []String, |
| 11 | 12 | unlazy_deps: []String, |
| 13 | extra: []u32, | |
| 12 | 14 | |
| 15 | /// The field order here matches `Configuration` which documents the order in | |
| 16 | /// the serialized format. | |
| 13 | 17 | pub const Header = extern struct { |
| 14 | 18 | string_bytes_len: u32, |
| 15 | 19 | steps_len: u32, |
| 16 | 20 | path_deps_len: u32, |
| 17 | 21 | unlazy_deps_len: u32, |
| 22 | extra_len: u32, | |
| 23 | ||
| 24 | /// Index into `steps`. | |
| 25 | default_step: u32, | |
| 26 | }; | |
| 27 | ||
| 28 | pub const Wip = struct { | |
| 29 | gpa: Allocator, | |
| 30 | string_table: StringTable = .empty, | |
| 31 | deps_table: DepsTable = .empty, | |
| 32 | ||
| 33 | string_bytes: std.ArrayList(u8) = .empty, | |
| 34 | unlazy_deps: std.ArrayList(String) = .empty, | |
| 35 | steps: std.ArrayList(Step) = .empty, | |
| 36 | path_deps: std.MultiArrayList(Path) = .empty, | |
| 37 | extra: std.ArrayList(u32) = .empty, | |
| 38 | ||
| 39 | const DepsTable = std.HashMapUnmanaged(Deps, void, DepsTableContext, std.hash_map.default_max_load_percentage); | |
| 40 | ||
| 41 | const DepsTableContext = struct { | |
| 42 | extra: []const u32, | |
| 43 | ||
| 44 | pub fn eql(ctx: @This(), a: Deps, b: Deps) bool { | |
| 45 | const len_a = ctx.extra[@intFromEnum(a)]; | |
| 46 | const len_b = ctx.extra[@intFromEnum(b)]; | |
| 47 | const slice_a = ctx.extra[@intFromEnum(a) + 1 ..][0..len_a]; | |
| 48 | const slice_b = ctx.extra[@intFromEnum(b) + 1 ..][0..len_b]; | |
| 49 | return std.mem.eql(u32, slice_a, slice_b); | |
| 50 | } | |
| 51 | ||
| 52 | pub fn hash(ctx: @This(), key: Deps) u64 { | |
| 53 | const len = ctx.extra[@intFromEnum(key)]; | |
| 54 | const slice = ctx.extra[@intFromEnum(key) + 1 ..][0..len]; | |
| 55 | return std.hash_map.hashString(@ptrCast(slice)); | |
| 56 | } | |
| 57 | }; | |
| 58 | ||
| 59 | const StringTable = std.HashMapUnmanaged(String, void, StringTableContext, std.hash_map.default_max_load_percentage); | |
| 60 | const StringTableContext = struct { | |
| 61 | bytes: []const u8, | |
| 62 | ||
| 63 | pub fn eql(_: @This(), a: String, b: String) bool { | |
| 64 | return a == b; | |
| 65 | } | |
| 66 | ||
| 67 | pub fn hash(ctx: @This(), key: String) u64 { | |
| 68 | return std.hash_map.hashString(std.mem.sliceTo(ctx.bytes[@intFromEnum(key)..], 0)); | |
| 69 | } | |
| 70 | }; | |
| 71 | ||
| 72 | const StringTableIndexAdapter = struct { | |
| 73 | bytes: []const u8, | |
| 74 | ||
| 75 | pub fn eql(ctx: @This(), a: []const u8, b: String) bool { | |
| 76 | return std.mem.eql(u8, a, std.mem.sliceTo(ctx.bytes[@intFromEnum(b)..], 0)); | |
| 77 | } | |
| 78 | ||
| 79 | pub fn hash(_: @This(), adapted_key: []const u8) u64 { | |
| 80 | assert(std.mem.indexOfScalar(u8, adapted_key, 0) == null); | |
| 81 | return std.hash_map.hashString(adapted_key); | |
| 82 | } | |
| 83 | }; | |
| 84 | ||
| 85 | pub fn init(gpa: Allocator) Wip { | |
| 86 | return .{ .gpa = gpa }; | |
| 87 | } | |
| 88 | ||
| 89 | pub fn deinit(wip: *Wip) void { | |
| 90 | const gpa = wip.gpa; | |
| 91 | wip.string_bytes.deinit(gpa); | |
| 92 | wip.unlazy_deps.deinit(gpa); | |
| 93 | wip.steps.deinit(gpa); | |
| 94 | wip.path_deps.deinit(gpa); | |
| 95 | wip.extra.deinit(gpa); | |
| 96 | wip.* = undefined; | |
| 97 | } | |
| 98 | ||
| 99 | pub const Static = struct { | |
| 100 | default_step: u32, | |
| 101 | }; | |
| 102 | ||
| 103 | pub fn write(wip: *Wip, w: *Io.Writer, static: Static) Io.Writer.Error!void { | |
| 104 | const header: Header = .{ | |
| 105 | .string_bytes_len = @intCast(wip.string_bytes.items.len), | |
| 106 | .steps_len = @intCast(wip.steps.items.len), | |
| 107 | .path_deps_len = @intCast(wip.path_deps.len), | |
| 108 | .unlazy_deps_len = @intCast(wip.unlazy_deps.items.len), | |
| 109 | .extra_len = @intCast(wip.extra.items.len), | |
| 110 | ||
| 111 | .default_step = static.default_step, | |
| 112 | }; | |
| 113 | var buffers = [_][]const u8{ | |
| 114 | @ptrCast(&header), | |
| 115 | wip.string_bytes.items, | |
| 116 | @ptrCast(wip.steps.items), | |
| 117 | @ptrCast(wip.path_deps.items(.base)), | |
| 118 | @ptrCast(wip.path_deps.items(.sub)), | |
| 119 | @ptrCast(wip.unlazy_deps.items), | |
| 120 | @ptrCast(wip.extra.items), | |
| 121 | }; | |
| 122 | try w.writeVecAll(&buffers); | |
| 123 | } | |
| 124 | ||
| 125 | pub fn addString(wip: *Wip, bytes: []const u8) Allocator.Error!String { | |
| 126 | const gpa = wip.gpa; | |
| 127 | assert(std.mem.indexOfScalar(u8, bytes, 0) == null); | |
| 128 | const gop = try wip.string_table.getOrPutContextAdapted( | |
| 129 | gpa, | |
| 130 | @as([]const u8, bytes), | |
| 131 | @as(StringTableIndexAdapter, .{ .bytes = wip.string_bytes.items }), | |
| 132 | @as(StringTableContext, .{ .bytes = wip.string_bytes.items }), | |
| 133 | ); | |
| 134 | if (gop.found_existing) return gop.key_ptr.*; | |
| 135 | ||
| 136 | try wip.string_bytes.ensureUnusedCapacity(gpa, bytes.len + 1); | |
| 137 | const new_off: String = @enumFromInt(wip.string_bytes.items.len); | |
| 138 | ||
| 139 | wip.string_bytes.appendSliceAssumeCapacity(bytes); | |
| 140 | wip.string_bytes.appendAssumeCapacity(0); | |
| 141 | ||
| 142 | gop.key_ptr.* = new_off; | |
| 143 | ||
| 144 | return new_off; | |
| 145 | } | |
| 146 | ||
| 147 | pub fn prepareDeps(wip: *Wip, n: usize) Allocator.Error![]u32 { | |
| 148 | const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1); | |
| 149 | slice[0] = @intCast(n); | |
| 150 | return slice[1..]; | |
| 151 | } | |
| 152 | ||
| 153 | pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps { | |
| 154 | const gpa = wip.gpa; | |
| 155 | const gop = try wip.deps_table.getOrPutContext(gpa, deps, @as(DepsTableContext, .{ | |
| 156 | .extra = wip.extra.items, | |
| 157 | })); | |
| 158 | if (gop.found_existing) { | |
| 159 | wip.extra.items.len = @intFromEnum(deps); | |
| 160 | return gop.key_ptr.*; | |
| 161 | } else { | |
| 162 | return deps; | |
| 163 | } | |
| 164 | } | |
| 165 | ||
| 166 | pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { | |
| 167 | const gpa = wip.gpa; | |
| 168 | const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; | |
| 169 | try wip.extra.ensureUnusedCapacity(gpa, fields.len); | |
| 170 | return addExtraAssumeCapacity(wip, extra); | |
| 171 | } | |
| 172 | ||
| 173 | pub fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 { | |
| 174 | const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; | |
| 175 | const result: u32 = @intCast(wip.extra.items.len); | |
| 176 | wip.extra.items.len += fields.len; | |
| 177 | setExtra(wip, result, extra); | |
| 178 | return result; | |
| 179 | } | |
| 180 | ||
| 181 | fn setExtra(wip: *Wip, index: usize, extra: anytype) void { | |
| 182 | const fields = @typeInfo(@TypeOf(extra)).@"struct".fields; | |
| 183 | var i = index; | |
| 184 | inline for (fields) |field| { | |
| 185 | wip.extra.items[i] = switch (field.type) { | |
| 186 | u32 => @field(extra, field.name), | |
| 187 | String, Deps => @intFromEnum(@field(extra, field.name)), | |
| 188 | else => @compileError("bad field type"), | |
| 189 | }; | |
| 190 | i += 1; | |
| 191 | } | |
| 192 | } | |
| 18 | 193 | }; |
| 19 | 194 | |
| 20 | 195 | pub const Step = extern struct { |
| 21 | 196 | name: String, |
| 197 | flags: Flags, | |
| 198 | deps: Deps, | |
| 199 | /// Points into `extra` for step-specific data. | |
| 200 | extra_index: u32, | |
| 201 | ||
| 202 | pub const Flags = packed struct(u32) { | |
| 203 | tag: Tag, | |
| 204 | _: u24 = 0, | |
| 205 | }; | |
| 206 | ||
| 207 | pub const Index = enum(u32) { | |
| 208 | _, | |
| 209 | }; | |
| 210 | ||
| 211 | pub const Tag = enum(u8) { | |
| 212 | top_level, | |
| 213 | compile, | |
| 214 | install_artifact, | |
| 215 | install_file, | |
| 216 | install_dir, | |
| 217 | remove_dir, | |
| 218 | fail, | |
| 219 | fmt, | |
| 220 | translate_c, | |
| 221 | write_file, | |
| 222 | update_source_files, | |
| 223 | run, | |
| 224 | check_file, | |
| 225 | check_object, | |
| 226 | config_header, | |
| 227 | objcopy, | |
| 228 | options, | |
| 229 | }; | |
| 230 | ||
| 231 | pub const TopLevel = struct { | |
| 232 | description: String, | |
| 233 | }; | |
| 234 | }; | |
| 235 | ||
| 236 | /// Points into `extra`, where the first element is number of deps, | |
| 237 | /// following elements is `Step.Index` per dep. | |
| 238 | pub const Deps = enum(u32) { | |
| 239 | _, | |
| 22 | 240 | }; |
| 23 | 241 | |
| 24 | 242 | pub const Path = extern struct { |
| ... | ... | @@ -27,8 +245,8 @@ pub const Path = extern struct { |
| 27 | 245 | |
| 28 | 246 | pub const Base = enum(u8) { |
| 29 | 247 | cwd, |
| 30 | global_cache, | |
| 31 | 248 | local_cache, |
| 249 | global_cache, | |
| 32 | 250 | build_root, |
| 33 | 251 | }; |
| 34 | 252 | |
| ... | ... | @@ -40,6 +258,7 @@ pub const Path = extern struct { |
| 40 | 258 | } |
| 41 | 259 | }; |
| 42 | 260 | |
| 261 | /// Points into `string_bytes`, null-terminated. | |
| 43 | 262 | pub const String = enum(u32) { |
| 44 | 263 | _, |
| 45 | 264 | |
| ... | ... | @@ -49,24 +268,29 @@ pub const String = enum(u32) { |
| 49 | 268 | } |
| 50 | 269 | }; |
| 51 | 270 | |
| 52 | pub const LoadError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; | |
| 271 | pub const LoadFileError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; | |
| 53 | 272 | |
| 54 | pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration { | |
| 273 | pub fn loadFile(arena: Allocator, io: Io, file: Io.File) LoadFileError!Configuration { | |
| 55 | 274 | var buffer: [2000]u8 = undefined; |
| 56 | 275 | var fr = file.reader(io, &buffer); |
| 57 | const header = fr.interface.takeStruct(Header, .little) catch |err| switch (err) { | |
| 276 | return load(arena, &fr.interface) catch |err| switch (err) { | |
| 58 | 277 | error.ReadFailed => return fr.err.?, |
| 59 | 278 | else => |e| return e, |
| 60 | 279 | }; |
| 280 | } | |
| 281 | ||
| 282 | pub const LoadError = Io.Reader.Error || Allocator.Error; | |
| 61 | 283 | |
| 284 | pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { | |
| 285 | const header = try reader.takeStruct(Header, .little); | |
| 62 | 286 | var result: Configuration = .{ |
| 63 | 287 | .string_bytes = try arena.alloc(u8, header.string_bytes_len), |
| 64 | 288 | .steps = try arena.alloc(Step, header.steps_len), |
| 65 | 289 | .path_deps_sub = try arena.alloc(String, header.path_deps_len), |
| 66 | 290 | .path_deps_base = try arena.alloc(Path.Base, header.path_deps_len), |
| 67 | 291 | .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), |
| 292 | .extra = try arena.alloc(u32, header.extra_len), | |
| 68 | 293 | }; |
| 69 | ||
| 70 | 294 | var vecs = [_][]u8{ |
| 71 | 295 | result.string_bytes, |
| 72 | 296 | @ptrCast(result.steps), |
| ... | ... | @@ -74,10 +298,6 @@ pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration { |
| 74 | 298 | @ptrCast(result.path_deps_sub), |
| 75 | 299 | @ptrCast(result.unlazy_deps), |
| 76 | 300 | }; |
| 77 | fr.interface.readVecAll(&vecs) catch |err| switch (err) { | |
| 78 | error.ReadFailed => return fr.err.?, | |
| 79 | else => |e| return e, | |
| 80 | }; | |
| 81 | ||
| 301 | try reader.readVecAll(&vecs); | |
| 82 | 302 | return result; |
| 83 | 303 | } |
src/main.zig+117-38
| ... | ... | @@ -298,7 +298,11 @@ fn mainArgs( |
| 298 | 298 | return process.exit(try llvmArMain(arena, args)); |
| 299 | 299 | } else if (mem.eql(u8, cmd, "build")) { |
| 300 | 300 | dev.check(.build_command); |
| 301 | return cmdBuild(gpa, arena, io, cmd_args, environ_map); | |
| 301 | var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ | |
| 302 | .child_allocator = arena, | |
| 303 | .io = io, | |
| 304 | }; | |
| 305 | return cmdBuild(gpa, thread_safe_arena.allocator(), io, cmd_args, environ_map); | |
| 302 | 306 | } else if (mem.eql(u8, cmd, "clang") or |
| 303 | 307 | mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) |
| 304 | 308 | { |
| ... | ... | @@ -4941,6 +4945,7 @@ test sanitizeExampleName { |
| 4941 | 4945 | |
| 4942 | 4946 | fn cmdBuild( |
| 4943 | 4947 | gpa: Allocator, |
| 4948 | /// Needs a thread-safe arena. | |
| 4944 | 4949 | arena: Allocator, |
| 4945 | 4950 | io: Io, |
| 4946 | 4951 | args: []const []const u8, |
| ... | ... | @@ -4973,28 +4978,34 @@ fn cmdBuild( |
| 4973 | 4978 | var debug_target: ?[]const u8 = null; |
| 4974 | 4979 | var debug_libc_paths_file: ?[]const u8 = null; |
| 4975 | 4980 | |
| 4981 | const self_exe_path = try process.executablePathAlloc(io, arena); | |
| 4982 | const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}); | |
| 4983 | ||
| 4984 | try configure_argv.ensureUnusedCapacity(arena, 16); | |
| 4985 | ||
| 4976 | 4986 | const argv_index_exe = configure_argv.items.len; |
| 4977 | _ = try configure_argv.addOne(arena); | |
| 4987 | _ = configure_argv.addOneAssumeCapacity(); | |
| 4978 | 4988 | |
| 4979 | const self_exe_path = try process.executablePathAlloc(io, arena); | |
| 4980 | try configure_argv.append(arena, self_exe_path); | |
| 4989 | configure_argv.appendAssumeCapacity("--zig"); | |
| 4990 | configure_argv.appendAssumeCapacity(self_exe_path); | |
| 4981 | 4991 | |
| 4992 | configure_argv.appendAssumeCapacity("--zig-lib-dir"); | |
| 4982 | 4993 | const argv_index_zig_lib_dir = configure_argv.items.len; |
| 4983 | _ = try configure_argv.addOne(arena); | |
| 4994 | _ = configure_argv.addOneAssumeCapacity(); | |
| 4984 | 4995 | |
| 4996 | configure_argv.appendAssumeCapacity("--build-root"); | |
| 4985 | 4997 | const argv_index_build_file = configure_argv.items.len; |
| 4986 | _ = try configure_argv.addOne(arena); | |
| 4998 | _ = configure_argv.addOneAssumeCapacity(); | |
| 4987 | 4999 | |
| 5000 | configure_argv.appendAssumeCapacity("--local-cache"); | |
| 4988 | 5001 | const argv_index_cache_dir = configure_argv.items.len; |
| 4989 | _ = try configure_argv.addOne(arena); | |
| 5002 | _ = configure_argv.addOneAssumeCapacity(); | |
| 4990 | 5003 | |
| 5004 | configure_argv.appendAssumeCapacity("--global-cache"); | |
| 4991 | 5005 | const argv_index_global_cache_dir = configure_argv.items.len; |
| 4992 | _ = try configure_argv.addOne(arena); | |
| 5006 | _ = configure_argv.addOneAssumeCapacity(); | |
| 4993 | 5007 | |
| 4994 | try configure_argv.appendSlice(arena, &.{ | |
| 4995 | "--seed", | |
| 4996 | try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}), | |
| 4997 | }); | |
| 5008 | configure_argv.appendSliceAssumeCapacity(&.{ "--seed", default_seed }); | |
| 4998 | 5009 | const argv_index_seed = configure_argv.items.len - 1; |
| 4999 | 5010 | |
| 5000 | 5011 | const argv_index_configuration_file = make_argv.items.len; |
| ... | ... | @@ -5192,14 +5203,6 @@ fn cmdBuild( |
| 5192 | 5203 | ); |
| 5193 | 5204 | try setThreadLimit(arena, thread_limit); |
| 5194 | 5205 | |
| 5195 | // Kick off an optimized compilation of the make runner. | |
| 5196 | var make_runner_task = io.async(compileMakeRunner, .{ io, .{ | |
| 5197 | .dirs = &dirs, | |
| 5198 | .optimize = .ReleaseSafe, | |
| 5199 | .parent_prog_node = root_prog_node, | |
| 5200 | } }); | |
| 5201 | defer if (make_runner_task.cancel(io)) |mr| mr.deinit(io) else |_| {}; | |
| 5202 | ||
| 5203 | 5206 | // Cache lookup for configure options. If we get a match, we can skip |
| 5204 | 5207 | // execution of the configure script. If not, we get the file path to pass |
| 5205 | 5208 | // to the configure process. |
| ... | ... | @@ -5255,6 +5258,19 @@ fn cmdBuild( |
| 5255 | 5258 | break :lci lci; |
| 5256 | 5259 | }; |
| 5257 | 5260 | |
| 5261 | // Kick off an optimized compilation of the make runner. | |
| 5262 | var make_runner_task = io.async(compileMakeRunner, .{ gpa, arena, io, .{ | |
| 5263 | .dirs = &dirs, | |
| 5264 | .environ_map = environ_map, | |
| 5265 | .parent_prog_node = root_prog_node, | |
| 5266 | .resolved_target = resolved_target, | |
| 5267 | .libc_installation = libc_installation, | |
| 5268 | .thread_limit = thread_limit, | |
| 5269 | .self_exe_path = self_exe_path, | |
| 5270 | .color = color, | |
| 5271 | } }); | |
| 5272 | defer _ = make_runner_task.cancel(io) catch {}; | |
| 5273 | ||
| 5258 | 5274 | configure_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; |
| 5259 | 5275 | configure_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; |
| 5260 | 5276 | configure_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; |
| ... | ... | @@ -5305,7 +5321,7 @@ fn cmdBuild( |
| 5305 | 5321 | .root_src_path = fs.path.basename(runner), |
| 5306 | 5322 | } else .{ |
| 5307 | 5323 | .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), |
| 5308 | .root_src_path = "build_runner.zig", | |
| 5324 | .root_src_path = "configure_runner.zig", | |
| 5309 | 5325 | }; |
| 5310 | 5326 | |
| 5311 | 5327 | const config = try Compilation.Config.resolve(.{ |
| ... | ... | @@ -5533,7 +5549,7 @@ fn cmdBuild( |
| 5533 | 5549 | const comp = Compilation.create(gpa, arena, io, &create_diag, .{ |
| 5534 | 5550 | .libc_installation = libc_installation, |
| 5535 | 5551 | .dirs = dirs, |
| 5536 | .root_name = "build", | |
| 5552 | .root_name = "configure", | |
| 5537 | 5553 | .config = config, |
| 5538 | 5554 | .root_mod = root_mod, |
| 5539 | 5555 | .main_mod = build_mod, |
| ... | ... | @@ -5554,7 +5570,7 @@ fn cmdBuild( |
| 5554 | 5570 | .environ_map = environ_map, |
| 5555 | 5571 | }) catch |err| switch (err) { |
| 5556 | 5572 | error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), |
| 5557 | else => fatal("failed to create compilation: {t}", .{err}), | |
| 5573 | else => |e| fatal("failed to create compilation: {t}", .{e}), | |
| 5558 | 5574 | }; |
| 5559 | 5575 | defer comp.destroy(); |
| 5560 | 5576 | |
| ... | ... | @@ -5625,7 +5641,7 @@ fn cmdBuild( |
| 5625 | 5641 | // add them to `config_man` before obtaining the final digest. |
| 5626 | 5642 | // * If it contains a set of lazy packages that need to be |
| 5627 | 5643 | // fetched, we need to fetch those now and re-run configure. |
| 5628 | var configuration = std.zig.Configuration.load(arena, io, config_tmp_file) catch |err| | |
| 5644 | var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| | |
| 5629 | 5645 | fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); |
| 5630 | 5646 | |
| 5631 | 5647 | if (configuration.unlazy_deps.len != 0) { |
| ... | ... | @@ -5661,7 +5677,7 @@ fn cmdBuild( |
| 5661 | 5677 | } |
| 5662 | 5678 | |
| 5663 | 5679 | for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { |
| 5664 | const conf_path: std.zig.Configuration.Path = .{ .base = base, .sub = sub }; | |
| 5680 | const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; | |
| 5665 | 5681 | try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); |
| 5666 | 5682 | } |
| 5667 | 5683 | |
| ... | ... | @@ -5707,7 +5723,6 @@ fn cmdBuild( |
| 5707 | 5723 | |
| 5708 | 5724 | const make_runner = make_runner_task.await(io) catch |err| |
| 5709 | 5725 | fatal("failed to compile maker: {t}", .{err}); |
| 5710 | defer make_runner.deinit(io); | |
| 5711 | 5726 | |
| 5712 | 5727 | make_argv.items[0] = try make_runner.exe_path.toString(arena); |
| 5713 | 5728 | make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); |
| ... | ... | @@ -5748,22 +5763,86 @@ const MakeRunner = struct { |
| 5748 | 5763 | exe_path: Path, |
| 5749 | 5764 | |
| 5750 | 5765 | const Options = struct { |
| 5766 | environ_map: *const process.Environ.Map, | |
| 5751 | 5767 | dirs: *Compilation.Directories, |
| 5752 | optimize: std.builtin.OptimizeMode, | |
| 5753 | 5768 | parent_prog_node: std.Progress.Node, |
| 5769 | resolved_target: Package.Module.ResolvedTarget, | |
| 5770 | libc_installation: ?*const LibCInstallation, | |
| 5771 | self_exe_path: []const u8, | |
| 5772 | thread_limit: usize, | |
| 5773 | color: Color, | |
| 5754 | 5774 | }; |
| 5755 | ||
| 5756 | fn deinit(mr: MakeRunner, io: Io) void { | |
| 5757 | _ = mr; | |
| 5758 | _ = io; | |
| 5759 | @panic("TODO"); | |
| 5760 | } | |
| 5761 | 5775 | }; |
| 5762 | 5776 | |
| 5763 | fn compileMakeRunner(io: Io, options: MakeRunner.Options) !MakeRunner { | |
| 5764 | _ = io; | |
| 5765 | _ = options; | |
| 5766 | @panic("TODO"); | |
| 5777 | fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner { | |
| 5778 | const compile_prog_node = options.parent_prog_node.start("Compile Maker", 0); | |
| 5779 | defer compile_prog_node.end(); | |
| 5780 | ||
| 5781 | const optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(options.environ_map)) | |
| 5782 | .Debug | |
| 5783 | else | |
| 5784 | .ReleaseSafe; | |
| 5785 | const strip = optimize_mode != .Debug; | |
| 5786 | ||
| 5787 | const main_mod_paths: Package.Module.CreateOptions.Paths = .{ | |
| 5788 | .root = try .fromRoot(arena, options.dirs.*, .zig_lib, "compiler"), | |
| 5789 | .root_src_path = "maker.zig", | |
| 5790 | }; | |
| 5791 | ||
| 5792 | const config = try Compilation.Config.resolve(.{ | |
| 5793 | .output_mode = .Exe, | |
| 5794 | .root_strip = strip, | |
| 5795 | .root_optimize_mode = optimize_mode, | |
| 5796 | .resolved_target = options.resolved_target, | |
| 5797 | .have_zcu = true, | |
| 5798 | .emit_bin = true, | |
| 5799 | .is_test = false, | |
| 5800 | }); | |
| 5801 | ||
| 5802 | const root_mod = try Package.Module.create(arena, .{ | |
| 5803 | .paths = main_mod_paths, | |
| 5804 | .fully_qualified_name = "root", | |
| 5805 | .cc_argv = &.{}, | |
| 5806 | .inherited = .{ | |
| 5807 | .resolved_target = options.resolved_target, | |
| 5808 | .optimize_mode = optimize_mode, | |
| 5809 | .strip = strip, | |
| 5810 | }, | |
| 5811 | .global = config, | |
| 5812 | .parent = null, | |
| 5813 | }); | |
| 5814 | ||
| 5815 | var create_diag: Compilation.CreateDiagnostic = undefined; | |
| 5816 | const comp = Compilation.create(gpa, arena, io, &create_diag, .{ | |
| 5817 | .dirs = options.dirs.*, | |
| 5818 | .root_name = "maker", | |
| 5819 | .config = config, | |
| 5820 | .root_mod = root_mod, | |
| 5821 | .main_mod = root_mod, | |
| 5822 | .emit_bin = .yes_cache, | |
| 5823 | .self_exe_path = options.self_exe_path, | |
| 5824 | .thread_limit = options.thread_limit, | |
| 5825 | .cache_mode = .whole, | |
| 5826 | .environ_map = options.environ_map, | |
| 5827 | }) catch |err| switch (err) { | |
| 5828 | error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), | |
| 5829 | error.Canceled => |e| return e, | |
| 5830 | else => |e| fatal("failed to create compilation: {t}", .{e}), | |
| 5831 | }; | |
| 5832 | defer comp.destroy(); | |
| 5833 | ||
| 5834 | try updateModule(comp, options.color, compile_prog_node); | |
| 5835 | ||
| 5836 | const exe_path: Path = .{ | |
| 5837 | .root_dir = options.dirs.global_cache, | |
| 5838 | .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ | |
| 5839 | &Cache.binToHex(comp.digest.?), comp.emit_bin.?, | |
| 5840 | }), | |
| 5841 | }; | |
| 5842 | ||
| 5843 | return .{ | |
| 5844 | .exe_path = exe_path, | |
| 5845 | }; | |
| 5767 | 5846 | } |
| 5768 | 5847 | |
| 5769 | 5848 | const Fork = struct { |
| ... | ... | @@ -5972,7 +6051,7 @@ fn jitCmdInner( |
| 5972 | 6051 | .environ_map = environ_map, |
| 5973 | 6052 | }) catch |err| switch (err) { |
| 5974 | 6053 | error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), |
| 5975 | else => fatal("failed to create compilation: {s}", .{@errorName(err)}), | |
| 6054 | else => fatal("failed to create compilation: {t}", .{err}), | |
| 5976 | 6055 | }; |
| 5977 | 6056 | defer comp.destroy(); |
| 5978 | 6057 |