authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2023-09-13 10:46:25+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-15 14:04:23-07:00
log94529ffb621fa633437ac48d8f90003e26e8ce5b
tree9454170d72caac01866caef3c25a0affa1dcb6de
parent1a0e6bcdb140c844384d62b78a7f4247753f9ffd

package manager: write deps in a flat format, eliminating the FQN concept

The new `@depedencies` module contains generated code like the following (where strings like "abc123" represent hashes): ```zig pub const root_deps = [_]struct { []const u8, []const u8 }{ .{ "foo", "abc123" }, }; pub const packages = struct { pub const abc123 = struct { pub const build_root = "/home/mlugg/.cache/zig/blah/abc123"; pub const build_zig = @import("abc123"); pub const deps = [_]struct { []const u8, []const u8 }{ .{ "bar", "abc123" }, .{ "name", "ghi789" }, }; }; }; ``` Each package contains a build root string, the build.zig import, and a mapping from dependency names to package hashes. There is also such a mapping for the root package dependencies. In theory, we could now remove the `dep_prefix` field from `std.Build`, since its main purpose is now handled differently. I believe this is a desirable goal, as it doesn't really make sense to assign a single FQN to any package (because it may appear in many different places in the package hierarchy). This commit does not remove that field, as it's used non-trivially in a few places in the build runner and compiler tests: this will be a future enhancement. Resolves: #16354 Resolves: #17135

5 files changed, 115 insertions(+), 57 deletions(-)

lib/build_runner.zig+1
......@@ -81,6 +81,7 @@ pub fn main() !void {
8181 global_cache_directory,
8282 host,
8383 &cache,
84 dependencies.root_deps,
8485 );
8586 defer builder.destroy();
8687
lib/std/Build.zig+26-15
......@@ -132,6 +132,10 @@ modules: std.StringArrayHashMap(*Module),
132132/// A map from build root dirs to the corresponding `*Dependency`. This is shared with all child
133133/// `Build`s.
134134initialized_deps: *InitializedDepMap,
135/// A mapping from dependency names to package hashes.
136available_deps: AvailableDeps,
137
138const AvailableDeps = []const struct { []const u8, []const u8 };
135139
136140const InitializedDepMap = std.HashMap(InitializedDepKey, *Dependency, InitializedDepContext, std.hash_map.default_max_load_percentage);
137141const InitializedDepKey = struct {
......@@ -248,6 +252,7 @@ pub fn create(
248252 global_cache_root: Cache.Directory,
249253 host: NativeTargetInfo,
250254 cache: *Cache,
255 available_deps: AvailableDeps,
251256) !*Build {
252257 const env_map = try allocator.create(EnvMap);
253258 env_map.* = try process.getEnvMap(allocator);
......@@ -308,6 +313,7 @@ pub fn create(
308313 .host = host,
309314 .modules = std.StringArrayHashMap(*Module).init(allocator),
310315 .initialized_deps = initialized_deps,
316 .available_deps = available_deps,
311317 };
312318 try self.top_level_steps.put(allocator, self.install_tls.step.name, &self.install_tls);
313319 try self.top_level_steps.put(allocator, self.uninstall_tls.step.name, &self.uninstall_tls);
......@@ -319,14 +325,15 @@ fn createChild(
319325 parent: *Build,
320326 dep_name: []const u8,
321327 build_root: Cache.Directory,
328 pkg_deps: AvailableDeps,
322329 user_input_options: UserInputOptionsMap,
323330) !*Build {
324 const child = try createChildOnly(parent, dep_name, build_root, user_input_options);
331 const child = try createChildOnly(parent, dep_name, build_root, pkg_deps, user_input_options);
325332 try determineAndApplyInstallPrefix(child);
326333 return child;
327334}
328335
329fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, user_input_options: UserInputOptionsMap) !*Build {
336fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, pkg_deps: AvailableDeps, user_input_options: UserInputOptionsMap) !*Build {
330337 const allocator = parent.allocator;
331338 const child = try allocator.create(Build);
332339 child.* = .{
......@@ -393,6 +400,7 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc
393400 .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }),
394401 .modules = std.StringArrayHashMap(*Module).init(allocator),
395402 .initialized_deps = parent.initialized_deps,
403 .available_deps = pkg_deps,
396404 };
397405 try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls);
398406 try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls);
......@@ -1705,20 +1713,22 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
17051713 const build_runner = @import("root");
17061714 const deps = build_runner.dependencies;
17071715
1708 inline for (@typeInfo(deps.imports).Struct.decls) |decl| {
1709 if (mem.startsWith(u8, decl.name, b.dep_prefix) and
1710 mem.endsWith(u8, decl.name, name) and
1711 decl.name.len == b.dep_prefix.len + name.len)
1712 {
1713 const build_zig = @field(deps.imports, decl.name);
1714 const build_root = @field(deps.build_root, decl.name);
1715 return dependencyInner(b, name, build_root, build_zig, args);
1716 const pkg_hash = for (b.available_deps) |dep| {
1717 if (mem.eql(u8, dep[0], name)) break dep[1];
1718 } else {
1719 const full_path = b.pathFromRoot("build.zig.zon");
1720 std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });
1721 process.exit(1);
1722 };
1723
1724 inline for (@typeInfo(deps.packages).Struct.decls) |decl| {
1725 if (mem.eql(u8, decl.name, pkg_hash)) {
1726 const pkg = @field(deps.packages, decl.name);
1727 return dependencyInner(b, name, pkg.build_root, pkg.build_zig, pkg.deps, args);
17161728 }
17171729 }
17181730
1719 const full_path = b.pathFromRoot("build.zig.zon");
1720 std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });
1721 process.exit(1);
1731 unreachable; // Bad @dependencies source
17221732}
17231733
17241734pub fn anonymousDependency(
......@@ -1737,7 +1747,7 @@ pub fn anonymousDependency(
17371747 '/', '\\' => byte.* = '.',
17381748 else => continue,
17391749 };
1740 return dependencyInner(b, name, build_root, build_zig, args);
1750 return dependencyInner(b, name, build_root, build_zig, &.{}, args);
17411751}
17421752
17431753fn userValuesAreSame(lhs: UserValue, rhs: UserValue) bool {
......@@ -1792,6 +1802,7 @@ pub fn dependencyInner(
17921802 name: []const u8,
17931803 build_root_string: []const u8,
17941804 comptime build_zig: type,
1805 pkg_deps: AvailableDeps,
17951806 args: anytype,
17961807) *Dependency {
17971808 const user_input_options = userInputOptionsFromArgs(b.allocator, args);
......@@ -1810,7 +1821,7 @@ pub fn dependencyInner(
18101821 process.exit(1);
18111822 },
18121823 };
1813 const sub_builder = b.createChild(name, build_root, user_input_options) catch @panic("unhandled error");
1824 const sub_builder = b.createChild(name, build_root, pkg_deps, user_input_options) catch @panic("unhandled error");
18141825 sub_builder.runBuild(build_zig) catch @panic("unhandled error");
18151826
18161827 if (sub_builder.validateUserInputDidItFail()) {
lib/std/Build/Step/Options.zig+1
......@@ -314,6 +314,7 @@ test Options {
314314 .{ .path = "test", .handle = std.fs.cwd() },
315315 host,
316316 &cache,
317 &.{},
317318 );
318319 defer builder.destroy();
319320
src/Package.zig+78-29
......@@ -214,6 +214,8 @@ pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8
214214
215215pub const build_zig_basename = "build.zig";
216216
217/// Fetches a package and all of its dependencies recursively. Writes the
218/// corresponding datastructures for the build runner into `dependencies_source`.
217219pub fn fetchAndAddDependencies(
218220 pkg: *Package,
219221 deps_pkg: *Package,
......@@ -224,11 +226,11 @@ pub fn fetchAndAddDependencies(
224226 global_cache_directory: Compilation.Directory,
225227 local_cache_directory: Compilation.Directory,
226228 dependencies_source: *std.ArrayList(u8),
227 build_roots_source: *std.ArrayList(u8),
228 name_prefix: []const u8,
229229 error_bundle: *std.zig.ErrorBundle.Wip,
230230 all_modules: *AllModules,
231231 root_prog_node: *std.Progress.Node,
232 /// null for the root package
233 this_hash: ?[]const u8,
232234) !void {
233235 const max_bytes = 10 * 1024 * 1024;
234236 const gpa = thread_pool.allocator;
......@@ -242,6 +244,28 @@ pub fn fetchAndAddDependencies(
242244 ) catch |err| switch (err) {
243245 error.FileNotFound => {
244246 // Handle the same as no dependencies.
247 if (this_hash) |hash| {
248 const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len];
249 const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path});
250 try dependencies_source.writer().print(
251 \\ pub const {} = struct {{
252 \\ pub const build_root = "{}";
253 \\ pub const build_zig = @import("{}");
254 \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}};
255 \\ }};
256 \\
257 , .{
258 std.zig.fmtId(hash),
259 std.zig.fmtEscapes(build_root),
260 std.zig.fmtEscapes(hash),
261 });
262 } else {
263 try dependencies_source.writer().writeAll(
264 \\pub const packages = struct {};
265 \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
266 \\
267 );
268 }
245269 return;
246270 },
247271 else => |e| return e,
......@@ -284,23 +308,23 @@ pub fn fetchAndAddDependencies(
284308
285309 root_prog_node.setEstimatedTotalItems(all_modules.count());
286310
311 if (this_hash == null) {
312 try dependencies_source.writer().writeAll("pub const packages = struct {\n");
313 }
314
287315 const deps_list = manifest.dependencies.values();
288316 for (manifest.dependencies.keys(), 0..) |name, i| {
289317 const dep = deps_list[i];
290318
291 const sub_prefix = try std.fmt.allocPrint(arena, "{s}{s}.", .{ name_prefix, name });
292 const fqn = sub_prefix[0 .. sub_prefix.len - 1];
293
294319 const sub = try fetchAndUnpack(
295320 thread_pool,
296321 http_client,
297322 global_cache_directory,
298323 dep,
299324 report,
300 build_roots_source,
301 fqn,
302325 all_modules,
303326 root_prog_node,
327 name,
304328 );
305329
306330 if (!sub.found_existing) {
......@@ -313,11 +337,10 @@ pub fn fetchAndAddDependencies(
313337 global_cache_directory,
314338 local_cache_directory,
315339 dependencies_source,
316 build_roots_source,
317 sub_prefix,
318340 error_bundle,
319341 all_modules,
320342 root_prog_node,
343 dep.hash.?,
321344 );
322345 }
323346
......@@ -329,10 +352,47 @@ pub fn fetchAndAddDependencies(
329352 } else {
330353 try deps_pkg.add(gpa, dep.hash.?, sub.mod);
331354 }
355 }
332356
333 try dependencies_source.writer().print(" pub const {s} = @import(\"{}\");\n", .{
334 std.zig.fmtId(fqn), std.zig.fmtEscapes(dep.hash.?),
357 if (this_hash) |hash| {
358 const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len];
359 const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path});
360 try dependencies_source.writer().print(
361 \\ pub const {} = struct {{
362 \\ pub const build_root = "{}";
363 \\ pub const build_zig = @import("{}");
364 \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{
365 \\
366 , .{
367 std.zig.fmtId(hash),
368 std.zig.fmtEscapes(build_root),
369 std.zig.fmtEscapes(hash),
335370 });
371 for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {
372 try dependencies_source.writer().print(
373 " .{{ \"{}\", \"{}\" }},\n",
374 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) },
375 );
376 }
377 try dependencies_source.writer().writeAll(
378 \\ };
379 \\ };
380 \\
381 );
382 } else {
383 try dependencies_source.writer().writeAll(
384 \\};
385 \\
386 \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
387 \\
388 );
389 for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {
390 try dependencies_source.writer().print(
391 " .{{ \"{}\", \"{}\" }},\n",
392 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) },
393 );
394 }
395 try dependencies_source.writer().writeAll("};\n");
336396 }
337397}
338398
......@@ -470,10 +530,11 @@ fn fetchAndUnpack(
470530 global_cache_directory: Compilation.Directory,
471531 dep: Manifest.Dependency,
472532 report: Report,
473 build_roots_source: *std.ArrayList(u8),
474 fqn: []const u8,
475533 all_modules: *AllModules,
476534 root_prog_node: *std.Progress.Node,
535 /// This does not have to be any form of canonical or fully-qualified name: it
536 /// is only intended to be human-readable for progress reporting.
537 name_for_prog: []const u8,
477538) !struct { mod: *Package, found_existing: bool } {
478539 const gpa = http_client.allocator;
479540 const s = fs.path.sep_str;
......@@ -484,25 +545,17 @@ fn fetchAndUnpack(
484545 const hex_digest = h[0..hex_multihash_len];
485546 const pkg_dir_sub_path = "p" ++ s ++ hex_digest;
486547
487 const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
488 errdefer gpa.free(build_root);
489
490548 var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) {
491549 error.FileNotFound => break :cached,
492550 else => |e| return e,
493551 };
494552 errdefer pkg_dir.close();
495553
496 try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{
497 std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
498 });
499
500554 // The compiler has a rule that a file must not be included in multiple modules,
501555 // so we must detect if a module has been created for this package and reuse it.
502556 const gop = try all_modules.getOrPut(gpa, hex_digest.*);
503557 if (gop.found_existing) {
504558 if (gop.value_ptr.*) |mod| {
505 gpa.free(build_root);
506559 return .{
507560 .mod = mod,
508561 .found_existing = true,
......@@ -510,6 +563,9 @@ fn fetchAndUnpack(
510563 }
511564 }
512565
566 const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
567 errdefer gpa.free(build_root);
568
513569 root_prog_node.completeOne();
514570
515571 const ptr = try gpa.create(Package);
......@@ -534,7 +590,7 @@ fn fetchAndUnpack(
534590 };
535591 }
536592
537 var pkg_prog_node = root_prog_node.start(fqn, 0);
593 var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
538594 defer pkg_prog_node.end();
539595 pkg_prog_node.activate();
540596 pkg_prog_node.context.refresh();
......@@ -666,13 +722,6 @@ fn fetchAndUnpack(
666722 return error.PackageFetchFailed;
667723 }
668724
669 const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
670 defer gpa.free(build_root);
671
672 try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{
673 std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
674 });
675
676725 const mod = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename);
677726 try all_modules.put(gpa, actual_hex, mod);
678727 return .{
src/main.zig+9-13
......@@ -4708,7 +4708,14 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
47084708 .root_src_directory = build_directory,
47094709 .root_src_path = build_zig_basename,
47104710 };
4711 if (!build_options.only_core_functionality) {
4711 if (build_options.only_core_functionality) {
4712 const deps_pkg = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig",
4713 \\pub const packages = struct {};
4714 \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
4715 \\
4716 );
4717 try main_pkg.add(gpa, "@dependencies", deps_pkg);
4718 } else {
47124719 var http_client: std.http.Client = .{ .allocator = gpa };
47134720 defer http_client.deinit();
47144721
......@@ -4717,12 +4724,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
47174724 // access dependencies by name, since `@import` requires string literals.
47184725 var dependencies_source = std.ArrayList(u8).init(gpa);
47194726 defer dependencies_source.deinit();
4720 try dependencies_source.appendSlice("pub const imports = struct {\n");
4721
4722 // This will go into the same package. It contains the file system paths
4723 // to all the build.zig files.
4724 var build_roots_source = std.ArrayList(u8).init(gpa);
4725 defer build_roots_source.deinit();
47264727
47274728 var all_modules: Package.AllModules = .{};
47284729 defer all_modules.deinit(gpa);
......@@ -4746,11 +4747,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
47464747 global_cache_directory,
47474748 local_cache_directory,
47484749 &dependencies_source,
4749 &build_roots_source,
4750 "",
47514750 &wip_errors,
47524751 &all_modules,
47534752 root_prog_node,
4753 null,
47544754 );
47554755 if (wip_errors.root_list.items.len > 0) {
47564756 var errors = try wip_errors.toOwnedBundle("");
......@@ -4760,10 +4760,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
47604760 }
47614761 try fetch_result;
47624762
4763 try dependencies_source.appendSlice("};\npub const build_root = struct {\n");
4764 try dependencies_source.appendSlice(build_roots_source.items);
4765 try dependencies_source.appendSlice("};\n");
4766
47674763 const deps_pkg = try Package.createFilePkg(
47684764 gpa,
47694765 local_cache_directory,