authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-02 17:15:24-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-02 20:43:01-07:00
logd753c3a1216e11ec35b243e9147038c5274cc913
treebbbbe3e5316cc85f4c5b23a5104f2a623a490d4b
parent252f4ab2a509c5b8b2e570f4a0de6cb706dd9588

CLI: implement 'zig build' lazy dependency fetching

This makes `zig build` notice when lazy dependencies were missing, fetch them, and then rebuild the build runner and run it again.

2 files changed, 495 insertions(+), 459 deletions(-)

src/Package/Fetch.zig+5-1
...@@ -106,8 +106,12 @@ pub const JobQueue = struct {...@@ -106,8 +106,12 @@ pub const JobQueue = struct {
106 /// If this is true, `recursive` must be false.106 /// If this is true, `recursive` must be false.
107 debug_hash: bool,107 debug_hash: bool,
108 work_around_btrfs_bug: bool,108 work_around_btrfs_bug: bool,
109 /// Set of hashes that will be additionally fetched even if they are marked
110 /// as lazy.
111 unlazy_set: UnlazySet = .{},
109112
110 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);113 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
114 pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, void);
111115
112 pub fn deinit(jq: *JobQueue) void {116 pub fn deinit(jq: *JobQueue) void {
113 if (jq.all_fetches.items.len == 0) return;117 if (jq.all_fetches.items.len == 0) return;
...@@ -375,7 +379,7 @@ pub fn run(f: *Fetch) RunError!void {...@@ -375,7 +379,7 @@ pub fn run(f: *Fetch) RunError!void {
375 error.FileNotFound => {379 error.FileNotFound => {
376 switch (f.lazy_status) {380 switch (f.lazy_status) {
377 .eager => {},381 .eager => {},
378 .available => {382 .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) {
379 f.lazy_status = .unavailable;383 f.lazy_status = .unavailable;
380 return;384 return;
381 },385 },
src/main.zig+490-458
...@@ -5150,512 +5150,544 @@ pub const usage_build =...@@ -5150,512 +5150,544 @@ pub const usage_build =
5150;5150;
51515151
5152pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {5152pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5153 const work_around_btrfs_bug = builtin.os.tag == .linux and5153 var progress: std.Progress = .{ .dont_print_on_dumb = true };
5154 EnvVar.ZIG_BTRFS_WORKAROUND.isSet();
5155 const color: Color = .auto;
51565154
5157 // We want to release all the locks before executing the child process, so we make a nice5155 var build_file: ?[]const u8 = null;
5158 // big block here to ensure the cleanup gets run when we extract out our argv.5156 var override_lib_dir: ?[]const u8 = try EnvVar.ZIG_LIB_DIR.get(arena);
5159 const child_argv = argv: {5157 var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena);
5160 const self_exe_path = try introspect.findZigExePath(arena);5158 var override_local_cache_dir: ?[]const u8 = try EnvVar.ZIG_LOCAL_CACHE_DIR.get(arena);
5159 var override_build_runner: ?[]const u8 = try EnvVar.ZIG_BUILD_RUNNER.get(arena);
5160 var child_argv = std.ArrayList([]const u8).init(arena);
5161 var reference_trace: ?u32 = null;
5162 var debug_compile_errors = false;
5163 var verbose_link = (builtin.os.tag != .wasi or builtin.link_libc) and
5164 EnvVar.ZIG_VERBOSE_LINK.isSet();
5165 var verbose_cc = (builtin.os.tag != .wasi or builtin.link_libc) and
5166 EnvVar.ZIG_VERBOSE_CC.isSet();
5167 var verbose_air = false;
5168 var verbose_intern_pool = false;
5169 var verbose_generic_instances = false;
5170 var verbose_llvm_ir: ?[]const u8 = null;
5171 var verbose_llvm_bc: ?[]const u8 = null;
5172 var verbose_cimport = false;
5173 var verbose_llvm_cpu_features = false;
5174 var fetch_only = false;
5175 var system_pkg_dir_path: ?[]const u8 = null;
51615176
5162 var build_file: ?[]const u8 = null;5177 const argv_index_exe = child_argv.items.len;
5163 var override_lib_dir: ?[]const u8 = try EnvVar.ZIG_LIB_DIR.get(arena);5178 _ = try child_argv.addOne();
5164 var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena);
5165 var override_local_cache_dir: ?[]const u8 = try EnvVar.ZIG_LOCAL_CACHE_DIR.get(arena);
5166 var override_build_runner: ?[]const u8 = try EnvVar.ZIG_BUILD_RUNNER.get(arena);
5167 var child_argv = std.ArrayList([]const u8).init(arena);
5168 var reference_trace: ?u32 = null;
5169 var debug_compile_errors = false;
5170 var verbose_link = (builtin.os.tag != .wasi or builtin.link_libc) and
5171 EnvVar.ZIG_VERBOSE_LINK.isSet();
5172 var verbose_cc = (builtin.os.tag != .wasi or builtin.link_libc) and
5173 EnvVar.ZIG_VERBOSE_CC.isSet();
5174 var verbose_air = false;
5175 var verbose_intern_pool = false;
5176 var verbose_generic_instances = false;
5177 var verbose_llvm_ir: ?[]const u8 = null;
5178 var verbose_llvm_bc: ?[]const u8 = null;
5179 var verbose_cimport = false;
5180 var verbose_llvm_cpu_features = false;
5181 var fetch_only = false;
5182 var system_pkg_dir_path: ?[]const u8 = null;
5183
5184 const argv_index_exe = child_argv.items.len;
5185 _ = try child_argv.addOne();
5186
5187 try child_argv.append(self_exe_path);
5188
5189 const argv_index_build_file = child_argv.items.len;
5190 _ = try child_argv.addOne();
5191
5192 const argv_index_cache_dir = child_argv.items.len;
5193 _ = try child_argv.addOne();
5194
5195 const argv_index_global_cache_dir = child_argv.items.len;
5196 _ = try child_argv.addOne();
5197
5198 try child_argv.appendSlice(&.{
5199 "--seed",
5200 try std.fmt.allocPrint(arena, "0x{x}", .{std.crypto.random.int(u32)}),
5201 });
5202 const argv_index_seed = child_argv.items.len - 1;
52035179
5204 {5180 const self_exe_path = try introspect.findZigExePath(arena);
5205 var i: usize = 0;5181 try child_argv.append(self_exe_path);
5206 while (i < args.len) : (i += 1) {5182
5207 const arg = args[i];5183 const argv_index_build_file = child_argv.items.len;
5208 if (mem.startsWith(u8, arg, "-")) {5184 _ = try child_argv.addOne();
5209 if (mem.eql(u8, arg, "--build-file")) {5185
5210 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});5186 const argv_index_cache_dir = child_argv.items.len;
5211 i += 1;5187 _ = try child_argv.addOne();
5212 build_file = args[i];5188
5213 continue;5189 const argv_index_global_cache_dir = child_argv.items.len;
5214 } else if (mem.eql(u8, arg, "--zig-lib-dir")) {5190 _ = try child_argv.addOne();
5215 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});5191
5216 i += 1;5192 try child_argv.appendSlice(&.{
5217 override_lib_dir = args[i];5193 "--seed",
5218 try child_argv.appendSlice(&.{ arg, args[i] });5194 try std.fmt.allocPrint(arena, "0x{x}", .{std.crypto.random.int(u32)}),
5219 continue;5195 });
5220 } else if (mem.eql(u8, arg, "--build-runner")) {5196 const argv_index_seed = child_argv.items.len - 1;
5221 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});5197
5222 i += 1;5198 {
5223 override_build_runner = args[i];5199 var i: usize = 0;
5224 continue;5200 while (i < args.len) : (i += 1) {
5225 } else if (mem.eql(u8, arg, "--cache-dir")) {5201 const arg = args[i];
5226 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});5202 if (mem.startsWith(u8, arg, "-")) {
5227 i += 1;5203 if (mem.eql(u8, arg, "--build-file")) {
5228 override_local_cache_dir = args[i];5204 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5229 continue;5205 i += 1;
5230 } else if (mem.eql(u8, arg, "--global-cache-dir")) {5206 build_file = args[i];
5231 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});5207 continue;
5232 i += 1;5208 } else if (mem.eql(u8, arg, "--zig-lib-dir")) {
5233 override_global_cache_dir = args[i];5209 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5234 continue;5210 i += 1;
5235 } else if (mem.eql(u8, arg, "-freference-trace")) {5211 override_lib_dir = args[i];
5236 reference_trace = 256;5212 try child_argv.appendSlice(&.{ arg, args[i] });
5237 } else if (mem.eql(u8, arg, "--fetch")) {5213 continue;
5238 fetch_only = true;5214 } else if (mem.eql(u8, arg, "--build-runner")) {
5239 } else if (mem.eql(u8, arg, "--system")) {5215 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5240 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});5216 i += 1;
5241 i += 1;5217 override_build_runner = args[i];
5242 system_pkg_dir_path = args[i];5218 continue;
5243 try child_argv.append("--system");5219 } else if (mem.eql(u8, arg, "--cache-dir")) {
5244 continue;5220 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5245 } else if (mem.startsWith(u8, arg, "-freference-trace=")) {5221 i += 1;
5246 const num = arg["-freference-trace=".len..];5222 override_local_cache_dir = args[i];
5247 reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| {5223 continue;
5248 fatal("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) });5224 } else if (mem.eql(u8, arg, "--global-cache-dir")) {
5249 };5225 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5250 } else if (mem.eql(u8, arg, "-fno-reference-trace")) {5226 i += 1;
5251 reference_trace = null;5227 override_global_cache_dir = args[i];
5252 } else if (mem.eql(u8, arg, "--debug-log")) {5228 continue;
5253 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});5229 } else if (mem.eql(u8, arg, "-freference-trace")) {
5254 try child_argv.appendSlice(args[i .. i + 2]);5230 reference_trace = 256;
5255 i += 1;5231 } else if (mem.eql(u8, arg, "--fetch")) {
5256 if (!build_options.enable_logging) {5232 fetch_only = true;
5257 warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{});5233 } else if (mem.eql(u8, arg, "--system")) {
5258 } else {5234 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5259 try log_scopes.append(arena, args[i]);5235 i += 1;
5260 }5236 system_pkg_dir_path = args[i];
5261 continue;5237 try child_argv.append("--system");
5262 } else if (mem.eql(u8, arg, "--debug-compile-errors")) {5238 continue;
5263 if (!crash_report.is_enabled) {5239 } else if (mem.startsWith(u8, arg, "-freference-trace=")) {
5264 warn("Zig was compiled in a release mode. --debug-compile-errors has no effect.", .{});5240 const num = arg["-freference-trace=".len..];
5265 } else {5241 reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| {
5266 debug_compile_errors = true;5242 fatal("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) });
5267 }5243 };
5268 } else if (mem.eql(u8, arg, "--verbose-link")) {5244 } else if (mem.eql(u8, arg, "-fno-reference-trace")) {
5269 verbose_link = true;5245 reference_trace = null;
5270 } else if (mem.eql(u8, arg, "--verbose-cc")) {5246 } else if (mem.eql(u8, arg, "--debug-log")) {
5271 verbose_cc = true;5247 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5272 } else if (mem.eql(u8, arg, "--verbose-air")) {5248 try child_argv.appendSlice(args[i .. i + 2]);
5273 verbose_air = true;5249 i += 1;
5274 } else if (mem.eql(u8, arg, "--verbose-intern-pool")) {5250 if (!build_options.enable_logging) {
5275 verbose_intern_pool = true;5251 warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{});
5276 } else if (mem.eql(u8, arg, "--verbose-generic-instances")) {5252 } else {
5277 verbose_generic_instances = true;5253 try log_scopes.append(arena, args[i]);
5278 } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
5279 verbose_llvm_ir = "-";
5280 } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) {
5281 verbose_llvm_ir = arg["--verbose-llvm-ir=".len..];
5282 } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) {
5283 verbose_llvm_bc = arg["--verbose-llvm-bc=".len..];
5284 } else if (mem.eql(u8, arg, "--verbose-cimport")) {
5285 verbose_cimport = true;
5286 } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) {
5287 verbose_llvm_cpu_features = true;
5288 } else if (mem.eql(u8, arg, "--seed")) {
5289 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5290 i += 1;
5291 child_argv.items[argv_index_seed] = args[i];
5292 continue;
5293 }5254 }
5255 continue;
5256 } else if (mem.eql(u8, arg, "--debug-compile-errors")) {
5257 if (!crash_report.is_enabled) {
5258 warn("Zig was compiled in a release mode. --debug-compile-errors has no effect.", .{});
5259 } else {
5260 debug_compile_errors = true;
5261 }
5262 } else if (mem.eql(u8, arg, "--verbose-link")) {
5263 verbose_link = true;
5264 } else if (mem.eql(u8, arg, "--verbose-cc")) {
5265 verbose_cc = true;
5266 } else if (mem.eql(u8, arg, "--verbose-air")) {
5267 verbose_air = true;
5268 } else if (mem.eql(u8, arg, "--verbose-intern-pool")) {
5269 verbose_intern_pool = true;
5270 } else if (mem.eql(u8, arg, "--verbose-generic-instances")) {
5271 verbose_generic_instances = true;
5272 } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
5273 verbose_llvm_ir = "-";
5274 } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) {
5275 verbose_llvm_ir = arg["--verbose-llvm-ir=".len..];
5276 } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) {
5277 verbose_llvm_bc = arg["--verbose-llvm-bc=".len..];
5278 } else if (mem.eql(u8, arg, "--verbose-cimport")) {
5279 verbose_cimport = true;
5280 } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) {
5281 verbose_llvm_cpu_features = true;
5282 } else if (mem.eql(u8, arg, "--seed")) {
5283 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
5284 i += 1;
5285 child_argv.items[argv_index_seed] = args[i];
5286 continue;
5294 }5287 }
5295 try child_argv.append(arg);
5296 }5288 }
5289 try child_argv.append(arg);
5297 }5290 }
5291 }
52985292
5299 var zig_lib_directory: Compilation.Directory = if (override_lib_dir) |lib_dir| .{5293 const work_around_btrfs_bug = builtin.os.tag == .linux and
5300 .path = lib_dir,5294 EnvVar.ZIG_BTRFS_WORKAROUND.isSet();
5301 .handle = fs.cwd().openDir(lib_dir, .{}) catch |err| {5295 const color: Color = .auto;
5302 fatal("unable to open zig lib directory from 'zig-lib-dir' argument: '{s}': {s}", .{ lib_dir, @errorName(err) });
5303 },
5304 } else introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| {
5305 fatal("unable to find zig installation directory '{s}': {s}", .{ self_exe_path, @errorName(err) });
5306 };
5307 defer zig_lib_directory.handle.close();
53085296
5309 var cleanup_build_dir: ?fs.Dir = null;5297 const target_query: std.Target.Query = .{};
5310 defer if (cleanup_build_dir) |*dir| dir.close();5298 const resolved_target: Package.Module.ResolvedTarget = .{
5299 .result = resolveTargetQueryOrFatal(target_query),
5300 .is_native_os = true,
5301 .is_native_abi = true,
5302 };
53115303
5312 const cwd_path = try process.getCwdAlloc(arena);5304 const exe_basename = try std.zig.binNameAlloc(arena, .{
5313 const build_root = try findBuildRoot(arena, .{5305 .root_name = "build",
5314 .cwd_path = cwd_path,5306 .target = resolved_target.result,
5315 .build_file = build_file,5307 .output_mode = .Exe,
5316 });5308 });
5317 child_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path;5309 const emit_bin: Compilation.EmitLoc = .{
5310 .directory = null, // Use the local zig-cache.
5311 .basename = exe_basename,
5312 };
53185313
5319 var global_cache_directory: Compilation.Directory = l: {5314 var thread_pool: ThreadPool = undefined;
5320 const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);5315 try thread_pool.init(.{ .allocator = gpa });
5321 break :l .{5316 defer thread_pool.deinit();
5322 .handle = try fs.cwd().makeOpenPath(p, .{}),5317
5323 .path = p,5318 var http_client: std.http.Client = .{ .allocator = gpa };
5319 defer http_client.deinit();
5320
5321 gimmeMoreOfThoseSweetSweetFileDescriptors();
5322
5323 var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
5324
5325 // This loop is re-evaluated when the build script exits with an indication that it
5326 // could not continue due to missing lazy dependencies.
5327 while (true) {
5328 // We want to release all the locks before executing the child process, so we make a nice
5329 // big block here to ensure the cleanup gets run when we extract out our argv.
5330 {
5331 var zig_lib_directory: Compilation.Directory = if (override_lib_dir) |lib_dir| .{
5332 .path = lib_dir,
5333 .handle = fs.cwd().openDir(lib_dir, .{}) catch |err| {
5334 fatal("unable to open zig lib directory from 'zig-lib-dir' argument: '{s}': {s}", .{ lib_dir, @errorName(err) });
5335 },
5336 } else introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| {
5337 fatal("unable to find zig installation directory '{s}': {s}", .{ self_exe_path, @errorName(err) });
5324 };5338 };
5325 };5339 defer zig_lib_directory.handle.close();
5326 defer global_cache_directory.handle.close();
53275340
5328 child_argv.items[argv_index_global_cache_dir] = global_cache_directory.path orelse cwd_path;5341 const cwd_path = try process.getCwdAlloc(arena);
5342 const build_root = try findBuildRoot(arena, .{
5343 .cwd_path = cwd_path,
5344 .build_file = build_file,
5345 });
5346 child_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path;
53295347
5330 var local_cache_directory: Compilation.Directory = l: {5348 var global_cache_directory: Compilation.Directory = l: {
5331 if (override_local_cache_dir) |local_cache_dir_path| {5349 const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
5332 break :l .{5350 break :l .{
5333 .handle = try fs.cwd().makeOpenPath(local_cache_dir_path, .{}),5351 .handle = try fs.cwd().makeOpenPath(p, .{}),
5334 .path = local_cache_dir_path,5352 .path = p,
5335 };5353 };
5336 }
5337 const cache_dir_path = try build_root.directory.join(arena, &[_][]const u8{"zig-cache"});
5338 break :l .{
5339 .handle = try build_root.directory.handle.makeOpenPath("zig-cache", .{}),
5340 .path = cache_dir_path,
5341 };5354 };
5342 };5355 defer global_cache_directory.handle.close();
5343 defer local_cache_directory.handle.close();
5344
5345 child_argv.items[argv_index_cache_dir] = local_cache_directory.path orelse cwd_path;
53465356
5347 gimmeMoreOfThoseSweetSweetFileDescriptors();5357 child_argv.items[argv_index_global_cache_dir] = global_cache_directory.path orelse cwd_path;
53485358
5349 const target_query: std.Target.Query = .{};5359 var local_cache_directory: Compilation.Directory = l: {
5350 const resolved_target: Package.Module.ResolvedTarget = .{5360 if (override_local_cache_dir) |local_cache_dir_path| {
5351 .result = resolveTargetQueryOrFatal(target_query),5361 break :l .{
5352 .is_native_os = true,5362 .handle = try fs.cwd().makeOpenPath(local_cache_dir_path, .{}),
5353 .is_native_abi = true,5363 .path = local_cache_dir_path,
5354 };5364 };
53555365 }
5356 const exe_basename = try std.zig.binNameAlloc(arena, .{5366 const cache_dir_path = try build_root.directory.join(arena, &[_][]const u8{"zig-cache"});
5357 .root_name = "build",5367 break :l .{
5358 .target = resolved_target.result,5368 .handle = try build_root.directory.handle.makeOpenPath("zig-cache", .{}),
5359 .output_mode = .Exe,5369 .path = cache_dir_path,
5360 });5370 };
5361 const emit_bin: Compilation.EmitLoc = .{5371 };
5362 .directory = null, // Use the local zig-cache.5372 defer local_cache_directory.handle.close();
5363 .basename = exe_basename,
5364 };
5365 var thread_pool: ThreadPool = undefined;
5366 try thread_pool.init(.{ .allocator = gpa });
5367 defer thread_pool.deinit();
53685373
5369 const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_build_runner) |runner| .{5374 child_argv.items[argv_index_cache_dir] = local_cache_directory.path orelse cwd_path;
5370 .root = .{
5371 .root_dir = Cache.Directory.cwd(),
5372 .sub_path = fs.path.dirname(runner) orelse "",
5373 },
5374 .root_src_path = fs.path.basename(runner),
5375 } else .{
5376 .root = .{ .root_dir = zig_lib_directory },
5377 .root_src_path = "build_runner.zig",
5378 };
53795375
5380 const config = try Compilation.Config.resolve(.{5376 const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_build_runner) |runner| .{
5381 .output_mode = .Exe,5377 .root = .{
5382 .resolved_target = resolved_target,5378 .root_dir = Cache.Directory.cwd(),
5383 .have_zcu = true,5379 .sub_path = fs.path.dirname(runner) orelse "",
5384 .emit_bin = true,5380 },
5385 .is_test = false,5381 .root_src_path = fs.path.basename(runner),
5386 });5382 } else .{
5383 .root = .{ .root_dir = zig_lib_directory },
5384 .root_src_path = "build_runner.zig",
5385 };
53875386
5388 const root_mod = try Package.Module.create(arena, .{5387 const config = try Compilation.Config.resolve(.{
5389 .global_cache_directory = global_cache_directory,5388 .output_mode = .Exe,
5390 .paths = main_mod_paths,
5391 .fully_qualified_name = "root",
5392 .cc_argv = &.{},
5393 .inherited = .{
5394 .resolved_target = resolved_target,5389 .resolved_target = resolved_target,
5395 },5390 .have_zcu = true,
5396 .global = config,5391 .emit_bin = true,
5397 .parent = null,5392 .is_test = false,
5398 .builtin_mod = null,5393 });
5399 });
5400
5401 const builtin_mod = root_mod.getBuiltinDependency();
54025394
5403 const build_mod = try Package.Module.create(arena, .{5395 const root_mod = try Package.Module.create(arena, .{
5404 .global_cache_directory = global_cache_directory,5396 .global_cache_directory = global_cache_directory,
5405 .paths = .{5397 .paths = main_mod_paths,
5406 .root = .{ .root_dir = build_root.directory },5398 .fully_qualified_name = "root",
5407 .root_src_path = build_root.build_zig_basename,5399 .cc_argv = &.{},
5408 },5400 .inherited = .{
5409 .fully_qualified_name = "root.@build",5401 .resolved_target = resolved_target,
5410 .cc_argv = &.{},5402 },
5411 .inherited = .{},5403 .global = config,
5412 .global = config,5404 .parent = null,
5413 .parent = root_mod,5405 .builtin_mod = null,
5414 .builtin_mod = builtin_mod,5406 });
5415 });
5416 if (build_options.only_core_functionality) {
5417 try createEmptyDependenciesModule(
5418 arena,
5419 root_mod,
5420 global_cache_directory,
5421 local_cache_directory,
5422 builtin_mod,
5423 config,
5424 );
5425 } else {
5426 var http_client: std.http.Client = .{ .allocator = gpa };
5427 defer http_client.deinit();
54285407
5429 var progress: std.Progress = .{ .dont_print_on_dumb = true };5408 const builtin_mod = root_mod.getBuiltinDependency();
5430 const root_prog_node = progress.start("Fetch Packages", 0);
5431 defer root_prog_node.end();
54325409
5433 var job_queue: Package.Fetch.JobQueue = .{5410 const build_mod = try Package.Module.create(arena, .{
5434 .http_client = &http_client,5411 .global_cache_directory = global_cache_directory,
5435 .thread_pool = &thread_pool,5412 .paths = .{
5436 .global_cache = global_cache_directory,5413 .root = .{ .root_dir = build_root.directory },
5437 .read_only = false,5414 .root_src_path = build_root.build_zig_basename,
5438 .recursive = true,5415 },
5439 .debug_hash = false,5416 .fully_qualified_name = "root.@build",
5440 .work_around_btrfs_bug = work_around_btrfs_bug,5417 .cc_argv = &.{},
5441 };5418 .inherited = .{},
5442 defer job_queue.deinit();5419 .global = config,
5420 .parent = root_mod,
5421 .builtin_mod = builtin_mod,
5422 });
54435423
5444 if (system_pkg_dir_path) |p| {5424 var cleanup_build_dir: ?fs.Dir = null;
5445 job_queue.global_cache = .{5425 defer if (cleanup_build_dir) |*dir| dir.close();
5446 .path = p,5426
5447 .handle = fs.cwd().openDir(p, .{}) catch |err| {5427 if (build_options.only_core_functionality) {
5448 fatal("unable to open system package directory '{s}': {s}", .{5428 try createEmptyDependenciesModule(
5449 p, @errorName(err),5429 arena,
5450 });5430 root_mod,
5451 },5431 global_cache_directory,
5452 };5432 local_cache_directory,
5453 job_queue.read_only = true;5433 builtin_mod,
5454 cleanup_build_dir = job_queue.global_cache.handle;5434 config,
5435 );
5455 } else {5436 } else {
5456 try http_client.loadDefaultProxies();5437 const root_prog_node = progress.start("Fetch Packages", 0);
5457 }5438 defer root_prog_node.end();
5439
5440 var job_queue: Package.Fetch.JobQueue = .{
5441 .http_client = &http_client,
5442 .thread_pool = &thread_pool,
5443 .global_cache = global_cache_directory,
5444 .read_only = false,
5445 .recursive = true,
5446 .debug_hash = false,
5447 .work_around_btrfs_bug = work_around_btrfs_bug,
5448 .unlazy_set = unlazy_set,
5449 };
5450 defer job_queue.deinit();
5451
5452 if (system_pkg_dir_path) |p| {
5453 job_queue.global_cache = .{
5454 .path = p,
5455 .handle = fs.cwd().openDir(p, .{}) catch |err| {
5456 fatal("unable to open system package directory '{s}': {s}", .{
5457 p, @errorName(err),
5458 });
5459 },
5460 };
5461 job_queue.read_only = true;
5462 cleanup_build_dir = job_queue.global_cache.handle;
5463 } else {
5464 try http_client.loadDefaultProxies();
5465 }
54585466
5459 try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);5467 try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
5460 try job_queue.table.ensureUnusedCapacity(gpa, 1);5468 try job_queue.table.ensureUnusedCapacity(gpa, 1);
54615469
5462 var fetch: Package.Fetch = .{5470 var fetch: Package.Fetch = .{
5463 .arena = std.heap.ArenaAllocator.init(gpa),5471 .arena = std.heap.ArenaAllocator.init(gpa),
5464 .location = .{ .relative_path = build_mod.root },5472 .location = .{ .relative_path = build_mod.root },
5465 .location_tok = 0,5473 .location_tok = 0,
5466 .hash_tok = 0,5474 .hash_tok = 0,
5467 .name_tok = 0,5475 .name_tok = 0,
5468 .lazy_status = .eager,5476 .lazy_status = .eager,
5469 .parent_package_root = build_mod.root,5477 .parent_package_root = build_mod.root,
5470 .parent_manifest_ast = null,5478 .parent_manifest_ast = null,
5471 .prog_node = root_prog_node,5479 .prog_node = root_prog_node,
5472 .job_queue = &job_queue,5480 .job_queue = &job_queue,
5473 .omit_missing_hash_error = true,5481 .omit_missing_hash_error = true,
5474 .allow_missing_paths_field = false,5482 .allow_missing_paths_field = false,
54755483
5476 .package_root = undefined,5484 .package_root = undefined,
5477 .error_bundle = undefined,5485 .error_bundle = undefined,
5478 .manifest = null,5486 .manifest = null,
5479 .manifest_ast = undefined,5487 .manifest_ast = undefined,
5480 .actual_hash = undefined,5488 .actual_hash = undefined,
5481 .has_build_zig = true,5489 .has_build_zig = true,
5482 .oom_flag = false,5490 .oom_flag = false,
54835491
5484 .module = build_mod,5492 .module = build_mod,
5485 };5493 };
5486 job_queue.all_fetches.appendAssumeCapacity(&fetch);5494 job_queue.all_fetches.appendAssumeCapacity(&fetch);
54875495
5488 job_queue.table.putAssumeCapacityNoClobber(5496 job_queue.table.putAssumeCapacityNoClobber(
5489 Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory),5497 Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory),
5490 &fetch,5498 &fetch,
5491 );5499 );
54925500
5493 job_queue.wait_group.start();5501 job_queue.wait_group.start();
5494 try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{ &fetch, "root" });5502 try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{ &fetch, "root" });
5495 job_queue.wait_group.wait();5503 job_queue.wait_group.wait();
54965504
5497 try job_queue.consolidateErrors();5505 try job_queue.consolidateErrors();
54985506
5499 if (fetch.error_bundle.root_list.items.len > 0) {5507 if (fetch.error_bundle.root_list.items.len > 0) {
5500 var errors = try fetch.error_bundle.toOwnedBundle("");5508 var errors = try fetch.error_bundle.toOwnedBundle("");
5501 errors.renderToStdErr(renderOptions(color));5509 errors.renderToStdErr(renderOptions(color));
5502 process.exit(1);5510 process.exit(1);
5503 }5511 }
55045512
5505 if (fetch_only) return cleanExit();5513 if (fetch_only) return cleanExit();
55065514
5507 var source_buf = std.ArrayList(u8).init(gpa);5515 var source_buf = std.ArrayList(u8).init(gpa);
5508 defer source_buf.deinit();5516 defer source_buf.deinit();
5509 try job_queue.createDependenciesSource(&source_buf);5517 try job_queue.createDependenciesSource(&source_buf);
5510 const deps_mod = try createDependenciesModule(5518 const deps_mod = try createDependenciesModule(
5511 arena,5519 arena,
5512 source_buf.items,5520 source_buf.items,
5513 root_mod,5521 root_mod,
5514 global_cache_directory,5522 global_cache_directory,
5515 local_cache_directory,5523 local_cache_directory,
5516 builtin_mod,5524 builtin_mod,
5517 config,5525 config,
5518 );5526 );
55195527
5520 {5528 {
5521 // We need a Module for each package's build.zig.5529 // We need a Module for each package's build.zig.
5522 const hashes = job_queue.table.keys();5530 const hashes = job_queue.table.keys();
5523 const fetches = job_queue.table.values();5531 const fetches = job_queue.table.values();
5524 try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));5532 try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
5525 for (hashes, fetches) |hash, f| {5533 for (hashes, fetches) |hash, f| {
5526 if (f == &fetch) {5534 if (f == &fetch) {
5527 // The first one is a dummy package for the current project.5535 // The first one is a dummy package for the current project.
5528 continue;5536 continue;
5537 }
5538 if (!f.has_build_zig)
5539 continue;
5540 const m = try Package.Module.create(arena, .{
5541 .global_cache_directory = global_cache_directory,
5542 .paths = .{
5543 .root = try f.package_root.clone(arena),
5544 .root_src_path = Package.build_zig_basename,
5545 },
5546 .fully_qualified_name = try std.fmt.allocPrint(
5547 arena,
5548 "root.@dependencies.{s}",
5549 .{&hash},
5550 ),
5551 .cc_argv = &.{},
5552 .inherited = .{},
5553 .global = config,
5554 .parent = root_mod,
5555 .builtin_mod = builtin_mod,
5556 });
5557 const hash_cloned = try arena.dupe(u8, &hash);
5558 deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
5559 f.module = m;
5529 }5560 }
5530 if (!f.has_build_zig)
5531 continue;
5532 const m = try Package.Module.create(arena, .{
5533 .global_cache_directory = global_cache_directory,
5534 .paths = .{
5535 .root = try f.package_root.clone(arena),
5536 .root_src_path = Package.build_zig_basename,
5537 },
5538 .fully_qualified_name = try std.fmt.allocPrint(
5539 arena,
5540 "root.@dependencies.{s}",
5541 .{&hash},
5542 ),
5543 .cc_argv = &.{},
5544 .inherited = .{},
5545 .global = config,
5546 .parent = root_mod,
5547 .builtin_mod = builtin_mod,
5548 });
5549 const hash_cloned = try arena.dupe(u8, &hash);
5550 deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
5551 f.module = m;
5552 }
55535561
5554 // Each build.zig module needs access to each of its5562 // Each build.zig module needs access to each of its
5555 // dependencies' build.zig modules by name.5563 // dependencies' build.zig modules by name.
5556 for (fetches) |f| {5564 for (fetches) |f| {
5557 const mod = f.module orelse continue;5565 const mod = f.module orelse continue;
5558 const man = f.manifest orelse continue;5566 const man = f.manifest orelse continue;
5559 const dep_names = man.dependencies.keys();5567 const dep_names = man.dependencies.keys();
5560 try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));5568 try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
5561 for (dep_names, man.dependencies.values()) |name, dep| {5569 for (dep_names, man.dependencies.values()) |name, dep| {
5562 const dep_digest = Package.Fetch.depDigest(5570 const dep_digest = Package.Fetch.depDigest(
5563 f.package_root,5571 f.package_root,
5564 global_cache_directory,5572 global_cache_directory,
5565 dep,5573 dep,
5566 ) orelse continue;5574 ) orelse continue;
5567 const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue;5575 const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue;
5568 const name_cloned = try arena.dupe(u8, name);5576 const name_cloned = try arena.dupe(u8, name);
5569 mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod);5577 mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod);
5578 }
5570 }5579 }
5571 }5580 }
5572 }5581 }
5573 }
5574
5575 try root_mod.deps.put(arena, "@build", build_mod);
55765582
5577 const comp = Compilation.create(gpa, arena, .{5583 try root_mod.deps.put(arena, "@build", build_mod);
5578 .zig_lib_directory = zig_lib_directory,
5579 .local_cache_directory = local_cache_directory,
5580 .global_cache_directory = global_cache_directory,
5581 .root_name = "build",
5582 .config = config,
5583 .root_mod = root_mod,
5584 .main_mod = build_mod,
5585 .emit_bin = emit_bin,
5586 .emit_h = null,
5587 .self_exe_path = self_exe_path,
5588 .thread_pool = &thread_pool,
5589 .verbose_cc = verbose_cc,
5590 .verbose_link = verbose_link,
5591 .verbose_air = verbose_air,
5592 .verbose_intern_pool = verbose_intern_pool,
5593 .verbose_generic_instances = verbose_generic_instances,
5594 .verbose_llvm_ir = verbose_llvm_ir,
5595 .verbose_llvm_bc = verbose_llvm_bc,
5596 .verbose_cimport = verbose_cimport,
5597 .verbose_llvm_cpu_features = verbose_llvm_cpu_features,
5598 .cache_mode = .whole,
5599 .reference_trace = reference_trace,
5600 .debug_compile_errors = debug_compile_errors,
5601 }) catch |err| {
5602 fatal("unable to create compilation: {s}", .{@errorName(err)});
5603 };
5604 defer comp.destroy();
56055584
5606 updateModule(comp, color) catch |err| switch (err) {5585 const comp = Compilation.create(gpa, arena, .{
5607 error.SemanticAnalyzeFail => process.exit(2),5586 .zig_lib_directory = zig_lib_directory,
5608 else => |e| return e,5587 .local_cache_directory = local_cache_directory,
5609 };5588 .global_cache_directory = global_cache_directory,
5589 .root_name = "build",
5590 .config = config,
5591 .root_mod = root_mod,
5592 .main_mod = build_mod,
5593 .emit_bin = emit_bin,
5594 .emit_h = null,
5595 .self_exe_path = self_exe_path,
5596 .thread_pool = &thread_pool,
5597 .verbose_cc = verbose_cc,
5598 .verbose_link = verbose_link,
5599 .verbose_air = verbose_air,
5600 .verbose_intern_pool = verbose_intern_pool,
5601 .verbose_generic_instances = verbose_generic_instances,
5602 .verbose_llvm_ir = verbose_llvm_ir,
5603 .verbose_llvm_bc = verbose_llvm_bc,
5604 .verbose_cimport = verbose_cimport,
5605 .verbose_llvm_cpu_features = verbose_llvm_cpu_features,
5606 .cache_mode = .whole,
5607 .reference_trace = reference_trace,
5608 .debug_compile_errors = debug_compile_errors,
5609 }) catch |err| {
5610 fatal("unable to create compilation: {s}", .{@errorName(err)});
5611 };
5612 defer comp.destroy();
56105613
5611 // Since incremental compilation isn't done yet, we use cache_mode = whole5614 updateModule(comp, color) catch |err| switch (err) {
5612 // above, and thus the output file is already closed.5615 error.SemanticAnalyzeFail => process.exit(2),
5613 //try comp.makeBinFileExecutable();5616 else => |e| return e,
5614 child_argv.items[argv_index_exe] =5617 };
5615 try local_cache_directory.join(arena, &.{comp.cache_use.whole.bin_sub_path.?});
56165618
5617 break :argv child_argv.items;5619 // Since incremental compilation isn't done yet, we use cache_mode = whole
5618 };5620 // above, and thus the output file is already closed.
5621 //try comp.makeBinFileExecutable();
5622 child_argv.items[argv_index_exe] =
5623 try local_cache_directory.join(arena, &.{comp.cache_use.whole.bin_sub_path.?});
5624 }
56195625
5620 if (process.can_spawn) {5626 if (process.can_spawn) {
5621 var child = std.ChildProcess.init(child_argv, gpa);5627 var child = std.ChildProcess.init(child_argv.items, gpa);
5622 child.stdin_behavior = .Inherit;5628 child.stdin_behavior = .Inherit;
5623 child.stdout_behavior = .Pipe;5629 child.stdout_behavior = .Pipe;
5624 child.stderr_behavior = .Inherit;5630 child.stderr_behavior = .Inherit;
56255631
5626 try child.spawn();5632 try child.spawn();
5627 // Since only one output stream is piped, we can simply do a blocking5633 // Since only one output stream is piped, we can simply do a blocking
5628 // read until the stream is finished.5634 // read until the stream is finished.
5629 const stdout = try child.stdout.?.readToEndAlloc(arena, 50 * 1024 * 1024);5635 const stdout = try child.stdout.?.readToEndAlloc(arena, 50 * 1024 * 1024);
5630 const term = try child.wait();5636 const term = try child.wait();
5631 switch (term) {5637 switch (term) {
5632 .Exited => |code| {5638 .Exited => |code| {
5633 if (code == 0) return cleanExit();5639 if (code == 0) return cleanExit();
5634 // Indicates that the build runner has reported compile errors5640 // Indicates that the build runner has reported compile errors
5635 // and this parent process does not need to report any further5641 // and this parent process does not need to report any further
5636 // diagnostics.5642 // diagnostics.
5637 if (code == 2) process.exit(2);5643 if (code == 2) process.exit(2);
56385644
5639 if (code == 3) {5645 if (code == 3) {
5640 // Indicates the configure phase failed due to missing lazy5646 // Indicates the configure phase failed due to missing lazy
5641 // dependencies and stdout contains the hashes of the ones5647 // dependencies and stdout contains the hashes of the ones
5642 // that are missing.5648 // that are missing.
5643 std.debug.print("missing lazy dependencies: '{s}'\n", .{stdout});5649 var it = mem.splitScalar(u8, stdout, '\n');
5644 std.debug.print("TODO: fetch them and rebuild the build script\n", .{});5650 var any_errors = false;
5645 process.exit(1);5651 while (it.next()) |hash| {
5646 }5652 if (hash.len == 0) continue;
5653 const digest_len = @typeInfo(Package.Manifest.MultiHashHexDigest).Array.len;
5654 if (hash.len != digest_len) {
5655 std.log.err("invalid digest (length {d} instead of {d}): '{s}'", .{
5656 hash.len, digest_len, hash,
5657 });
5658 any_errors = true;
5659 continue;
5660 }
5661 try unlazy_set.put(arena, hash[0..digest_len].*, {});
5662 }
5663 if (any_errors) process.exit(3);
5664 if (system_pkg_dir_path) |p| {
5665 // In this mode, the system needs to provide these packages; they
5666 // cannot be fetched by Zig.
5667 for (unlazy_set.keys()) |hash| {
5668 std.log.err("lazy dependency package not found: {s}{c}{s}", .{
5669 p, fs.path.sep, hash,
5670 });
5671 }
5672 std.log.info("remote package fetching disabled due to --system mode", .{});
5673 std.log.info("dependencies might be avoidable depending on build configuration", .{});
5674 process.exit(3);
5675 }
5676 continue;
5677 }
56475678
5648 const cmd = try std.mem.join(arena, " ", child_argv);5679 const cmd = try std.mem.join(arena, " ", child_argv.items);
5649 fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });5680 fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
5650 },5681 },
5651 else => {5682 else => {
5652 const cmd = try std.mem.join(arena, " ", child_argv);5683 const cmd = try std.mem.join(arena, " ", child_argv.items);
5653 fatal("the following build command crashed:\n{s}", .{cmd});5684 fatal("the following build command crashed:\n{s}", .{cmd});
5654 },5685 },
5686 }
5687 } else {
5688 const cmd = try std.mem.join(arena, " ", child_argv.items);
5689 fatal("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd });
5655 }5690 }
5656 } else {
5657 const cmd = try std.mem.join(arena, " ", child_argv);
5658 fatal("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd });
5659 }5691 }
5660}5692}
56615693