Commit f65e8c78621c90c9b9932903a9ed99c973dbcf63 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2023-04-22 18:15:19+01:00
committer	Andrew Kelley <andrew@ziglang.org> 2023-05-18 00:27:21-07:00
log	f65e8c78621c90c9b9932903a9ed99c973dbcf63
tree	027dc7bdf661ae06bc3f75fb14fd71d5c394a511
parent	378264d404e71da878f9e6934c045ad64193877f

Deduplicate uses of the same package across dependencies

2 files changed, 26 insertions(+), 7 deletions(-)

lib
- std
  - Build.zig
src
- Package.zig

lib/std/Build.zig+16

...	@@ -124,6 +124,9 @@ host: NativeTargetInfo,	...	@@ -124,6 +124,9 @@ host: NativeTargetInfo,
124	dep_prefix: []const u8 = "",	124	dep_prefix: []const u8 = "",
125		125
126	modules: std.StringArrayHashMap(*Module),	126	modules: std.StringArrayHashMap(*Module),
		127	/// A map from build root dirs to the corresponding `*Dependency`. This is shared with all child
		128	/// `Build`s.
		129	initialized_deps: std.StringHashMap(Dependency),
127		130
128	pub const ExecError = error{	131	pub const ExecError = error{
129	ReadFailure,	132	ReadFailure,
...	@@ -209,6 +212,9 @@ pub fn create(	...	@@ -209,6 +212,9 @@ pub fn create(
209	const env_map = try allocator.create(EnvMap);	212	const env_map = try allocator.create(EnvMap);
210	env_map.* = try process.getEnvMap(allocator);	213	env_map.* = try process.getEnvMap(allocator);
211		214
		215	const initialized_deps = try allocator.create(std.StringHashMap(*Dependency));
		216	initialized_deps.* = std.StringHashMap(*Dependency).init(allocator);
		217
212	const self = try allocator.create(Build);	218	const self = try allocator.create(Build);
213	self.* = .{	219	self.* = .{
214	.zig_exe = zig_exe,	220	.zig_exe = zig_exe,
...	@@ -261,6 +267,7 @@ pub fn create(	...	@@ -261,6 +267,7 @@ pub fn create(
261	.args = null,	267	.args = null,
262	.host = host,	268	.host = host,
263	.modules = std.StringArrayHashMap(*Module).init(allocator),	269	.modules = std.StringArrayHashMap(*Module).init(allocator),
		270	.initialized_deps = initialized_deps,
264	};	271	};
265	try self.top_level_steps.put(allocator, self.install_tls.step.name, &self.install_tls);	272	try self.top_level_steps.put(allocator, self.install_tls.step.name, &self.install_tls);
266	try self.top_level_steps.put(allocator, self.uninstall_tls.step.name, &self.uninstall_tls);	273	try self.top_level_steps.put(allocator, self.uninstall_tls.step.name, &self.uninstall_tls);
...	@@ -345,6 +352,7 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc	...	@@ -345,6 +352,7 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc
345	.host = parent.host,	352	.host = parent.host,
346	.dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }),	353	.dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }),
347	.modules = std.StringArrayHashMap(*Module).init(allocator),	354	.modules = std.StringArrayHashMap(*Module).init(allocator),
		355	.initialized_deps = parent.initialized_deps,
348	};	356	};
349	try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls);	357	try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls);
350	try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls);	358	try child.top_level_steps.put(allocator, child.uninstall_tls.step.name, &child.uninstall_tls);
...	@@ -1560,6 +1568,11 @@ pub fn dependencyInner(	...	@@ -1560,6 +1568,11 @@ pub fn dependencyInner(
1560	comptime build_zig: type,	1568	comptime build_zig: type,
1561	args: anytype,	1569	args: anytype,
1562	) *Dependency {	1570	) *Dependency {
		1571	if (b.initialized_deps.get(build_root_string)) \|dep\| {
		1572	// TODO: check args are the same
		1573	return dep;
		1574	}
		1575
1563	const build_root: std.Build.Cache.Directory = .{	1576	const build_root: std.Build.Cache.Directory = .{
1564	.path = build_root_string,	1577	.path = build_root_string,
1565	.handle = std.fs.cwd().openDir(build_root_string, .{}) catch \|err\| {	1578	.handle = std.fs.cwd().openDir(build_root_string, .{}) catch \|err\| {
...	@@ -1578,6 +1591,9 @@ pub fn dependencyInner(	...	@@ -1578,6 +1591,9 @@ pub fn dependencyInner(
1578		1591
1579	const dep = b.allocator.create(Dependency) catch @panic("OOM");	1592	const dep = b.allocator.create(Dependency) catch @panic("OOM");
1580	dep.* = .{ .builder = sub_builder };	1593	dep.* = .{ .builder = sub_builder };
		1594
		1595	b.initialized_deps.put(build_root_string, dep) catch @panic("OOM");
		1596
1581	return dep;	1597	return dep;
1582	}	1598	}
1583		1599

src/Package.zig+10-7

 @@ -216,7 +216,7 @@ pub const build_zig_basename = "build.zig";
 pub fn fetchAndAddDependencies(
     pkg: *Package,
-    root_pkg: *Package,
+    deps_pkg: *Package,
     arena: Allocator,
     thread_pool: *ThreadPool,
     http_client: *std.http.Client,
 @@ -272,7 +272,6 @@ pub fn fetchAndAddDependencies(
         .error_bundle = error_bundle,
     };
-    var any_error = false;
     const deps_list = manifest.dependencies.values();
     for (manifest.dependencies.keys(), 0..) |name, i| {
         const dep = deps_list[i];
 @@ -292,7 +291,7 @@ pub fn fetchAndAddDependencies(
         );
         try sub_pkg.fetchAndAddDependencies(
-            root_pkg,
+            deps_pkg,
             arena,
             thread_pool,
             http_client,
 @@ -307,14 +306,18 @@ pub fn fetchAndAddDependencies(
         );
         try pkg.add(gpa, name, sub_pkg);
-        try root_pkg.add(gpa, fqn, sub_pkg);
+        if (deps_pkg.table.get(dep.hash.?)) |other_sub| {
+            // This should be the same package (and hence module) since it's the same hash
+            // TODO: dedup multiple versions of the same package
+            assert(other_sub == sub_pkg);
+        } else {
+            try deps_pkg.add(gpa, dep.hash.?, sub_pkg);
+        }
         try dependencies_source.writer().print("    pub const {s} = @import(\"{}\");\n", .{
-            std.zig.fmtId(fqn), std.zig.fmtEscapes(fqn),
+            std.zig.fmtId(fqn), std.zig.fmtEscapes(dep.hash.?),
         });
     }
-    if (any_error) return error.InvalidBuildManifestFile;
 }
 pub fn createFilePkg(