authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-05-20 22:58:20-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-05-20 22:58:20-07:00
logb7cb88384c39a15e3ce9dbeaa5a191786d186d60
tree262dd30843c537e027f997fb46975294c67f259f
parent7621e56938ed6d86827c11ee6272db722f350fed
parentdb7496d6efb64d8210816e49008753b5a81d46f4
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15407 from mlugg/feat/pkg-dedup

Deduplicate uses of the same package across dependencies closes #15755

2 files changed, 57 insertions(+), 26 deletions(-)

lib/std/Build.zig+16
......@@ -124,6 +124,9 @@ host: NativeTargetInfo,
124124dep_prefix: []const u8 = "",
125125
126126modules: std.StringArrayHashMap(*Module),
127/// A map from build root dirs to the corresponding `*Dependency`. This is shared with all child
128/// `Build`s.
129initialized_deps: *std.StringHashMap(*Dependency),
127130
128131pub const ExecError = error{
129132 ReadFailure,
......@@ -209,6 +212,9 @@ pub fn create(
209212 const env_map = try allocator.create(EnvMap);
210213 env_map.* = try process.getEnvMap(allocator);
211214
215 const initialized_deps = try allocator.create(std.StringHashMap(*Dependency));
216 initialized_deps.* = std.StringHashMap(*Dependency).init(allocator);
217
212218 const self = try allocator.create(Build);
213219 self.* = .{
214220 .zig_exe = zig_exe,
......@@ -261,6 +267,7 @@ pub fn create(
261267 .args = null,
262268 .host = host,
263269 .modules = std.StringArrayHashMap(*Module).init(allocator),
270 .initialized_deps = initialized_deps,
264271 };
265272 try self.top_level_steps.put(allocator, self.install_tls.step.name, &self.install_tls);
266273 try self.top_level_steps.put(allocator, self.uninstall_tls.step.name, &self.uninstall_tls);
......@@ -345,6 +352,7 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc
345352 .host = parent.host,
346353 .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }),
347354 .modules = std.StringArrayHashMap(*Module).init(allocator),
355 .initialized_deps = parent.initialized_deps,
348356 };
349357 try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls);
350358 try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls);
......@@ -1560,6 +1568,11 @@ pub fn dependencyInner(
15601568 comptime build_zig: type,
15611569 args: anytype,
15621570) *Dependency {
1571 if (b.initialized_deps.get(build_root_string)) |dep| {
1572 // TODO: check args are the same
1573 return dep;
1574 }
1575
15631576 const build_root: std.Build.Cache.Directory = .{
15641577 .path = build_root_string,
15651578 .handle = std.fs.cwd().openDir(build_root_string, .{}) catch |err| {
......@@ -1578,6 +1591,9 @@ pub fn dependencyInner(
15781591
15791592 const dep = b.allocator.create(Dependency) catch @panic("OOM");
15801593 dep.* = .{ .builder = sub_builder };
1594
1595 b.initialized_deps.put(build_root_string, dep) catch @panic("OOM");
1596
15811597 return dep;
15821598}
15831599
src/Package.zig+41-26
......@@ -216,7 +216,7 @@ pub const build_zig_basename = "build.zig";
216216
217217pub fn fetchAndAddDependencies(
218218 pkg: *Package,
219 root_pkg: *Package,
219 deps_pkg: *Package,
220220 arena: Allocator,
221221 thread_pool: *ThreadPool,
222222 http_client: *std.http.Client,
......@@ -272,7 +272,6 @@ pub fn fetchAndAddDependencies(
272272 .error_bundle = error_bundle,
273273 };
274274
275 var any_error = false;
276275 const deps_list = manifest.dependencies.values();
277276 for (manifest.dependencies.keys(), 0..) |name, i| {
278277 const dep = deps_list[i];
......@@ -280,7 +279,7 @@ pub fn fetchAndAddDependencies(
280279 const sub_prefix = try std.fmt.allocPrint(arena, "{s}{s}.", .{ name_prefix, name });
281280 const fqn = sub_prefix[0 .. sub_prefix.len - 1];
282281
283 const sub_pkg = try fetchAndUnpack(
282 const sub = try fetchAndUnpack(
284283 thread_pool,
285284 http_client,
286285 global_cache_directory,
......@@ -291,30 +290,36 @@ pub fn fetchAndAddDependencies(
291290 all_modules,
292291 );
293292
294 try sub_pkg.fetchAndAddDependencies(
295 root_pkg,
296 arena,
297 thread_pool,
298 http_client,
299 sub_pkg.root_src_directory,
300 global_cache_directory,
301 local_cache_directory,
302 dependencies_source,
303 build_roots_source,
304 sub_prefix,
305 error_bundle,
306 all_modules,
307 );
293 if (!sub.found_existing) {
294 try sub.mod.fetchAndAddDependencies(
295 deps_pkg,
296 arena,
297 thread_pool,
298 http_client,
299 sub.mod.root_src_directory,
300 global_cache_directory,
301 local_cache_directory,
302 dependencies_source,
303 build_roots_source,
304 sub_prefix,
305 error_bundle,
306 all_modules,
307 );
308 }
308309
309 try pkg.add(gpa, name, sub_pkg);
310 try root_pkg.add(gpa, fqn, sub_pkg);
310 try pkg.add(gpa, name, sub.mod);
311 if (deps_pkg.table.get(dep.hash.?)) |other_sub| {
312 // This should be the same package (and hence module) since it's the same hash
313 // TODO: dedup multiple versions of the same package
314 assert(other_sub == sub.mod);
315 } else {
316 try deps_pkg.add(gpa, dep.hash.?, sub.mod);
317 }
311318
312319 try dependencies_source.writer().print(" pub const {s} = @import(\"{}\");\n", .{
313 std.zig.fmtId(fqn), std.zig.fmtEscapes(fqn),
320 std.zig.fmtId(fqn), std.zig.fmtEscapes(dep.hash.?),
314321 });
315322 }
316
317 if (any_error) return error.InvalidBuildManifestFile;
318323}
319324
320325pub fn createFilePkg(
......@@ -410,7 +415,7 @@ fn fetchAndUnpack(
410415 build_roots_source: *std.ArrayList(u8),
411416 fqn: []const u8,
412417 all_modules: *AllModules,
413) !*Package {
418) !struct { mod: *Package, found_existing: bool } {
414419 const gpa = http_client.allocator;
415420 const s = fs.path.sep_str;
416421
......@@ -438,7 +443,10 @@ fn fetchAndUnpack(
438443 const gop = try all_modules.getOrPut(gpa, hex_digest.*);
439444 if (gop.found_existing) {
440445 gpa.free(build_root);
441 return gop.value_ptr.*;
446 return .{
447 .mod = gop.value_ptr.*,
448 .found_existing = true,
449 };
442450 }
443451
444452 const ptr = try gpa.create(Package);
......@@ -457,7 +465,10 @@ fn fetchAndUnpack(
457465 };
458466
459467 gop.value_ptr.* = ptr;
460 return ptr;
468 return .{
469 .mod = ptr,
470 .found_existing = false,
471 };
461472 }
462473
463474 const uri = try std.Uri.parse(dep.url);
......@@ -572,7 +583,11 @@ fn fetchAndUnpack(
572583 std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
573584 });
574585
575 return createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename);
586 const mod = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename);
587 return .{
588 .mod = mod,
589 .found_existing = false,
590 };
576591}
577592
578593fn unpackTarball(