| author |  Matthew Lugg <mlugg@mlugg.co.uk> 2024-03-06 21:24:38+00:00 |
| committer | Matthew Lugg <mlugg@mlugg.co.uk> 2024-03-06 21:26:39+00:00 |
| log | b41a0b4768d368d81d0d33c779f919d8f315e622 |
| tree | 36bcd210c4eedce2b3bb68fe6c097dc79219e84f |
| parent | 4c05a9a892d68749f3d7da26ee0e884158640720 |
| signature |  Commit is signed but in an unrecognized format. |
Package.Module: deduplicate identical builtin modules
Previously, when multiple modules had builtin modules with identical
sources, two distinct `Module`s and `File`s were created pointing at the
same file path. This led to a bug later in the frontend. These modules
are now deduplicated with a simple hashmap on the builtin source.9 files changed, 41 insertions(+), 4 deletions(-)
src/Compilation.zig+4
| ... | @@ -1325,6 +1325,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil | ... | @@ -1325,6 +1325,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil |
| 1325 | .global = options.config, | 1325 | .global = options.config, |
| 1326 | .parent = options.root_mod, | 1326 | .parent = options.root_mod, |
| 1327 | .builtin_mod = options.root_mod.getBuiltinDependency(), | 1327 | .builtin_mod = options.root_mod.getBuiltinDependency(), |
| 1328 | .builtin_modules = null, // `builtin_mod` is set | ||
| 1328 | }); | 1329 | }); |
| 1329 | try options.root_mod.deps.putNoClobber(arena, "compiler_rt", compiler_rt_mod); | 1330 | try options.root_mod.deps.putNoClobber(arena, "compiler_rt", compiler_rt_mod); |
| 1330 | } | 1331 | } |
| ... | @@ -1429,6 +1430,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil | ... | @@ -1429,6 +1430,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil |
| 1429 | .global = options.config, | 1430 | .global = options.config, |
| 1430 | .parent = options.root_mod, | 1431 | .parent = options.root_mod, |
| 1431 | .builtin_mod = options.root_mod.getBuiltinDependency(), | 1432 | .builtin_mod = options.root_mod.getBuiltinDependency(), |
| 1433 | .builtin_modules = null, // `builtin_mod` is set | ||
| 1432 | }); | 1434 | }); |
| 1433 | 1435 | ||
| 1434 | const zcu = try arena.create(Module); | 1436 | const zcu = try arena.create(Module); |
| ... | @@ -6104,6 +6106,7 @@ fn buildOutputFromZig( | ... | @@ -6104,6 +6106,7 @@ fn buildOutputFromZig( |
| 6104 | .cc_argv = &.{}, | 6106 | .cc_argv = &.{}, |
| 6105 | .parent = null, | 6107 | .parent = null, |
| 6106 | .builtin_mod = null, | 6108 | .builtin_mod = null, |
| 6109 | .builtin_modules = null, // there is only one module in this compilation | ||
| 6107 | }); | 6110 | }); |
| 6108 | const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; | 6111 | const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; |
| 6109 | const target = comp.getTarget(); | 6112 | const target = comp.getTarget(); |
| ... | @@ -6216,6 +6219,7 @@ pub fn build_crt_file( | ... | @@ -6216,6 +6219,7 @@ pub fn build_crt_file( |
| 6216 | .cc_argv = &.{}, | 6219 | .cc_argv = &.{}, |
| 6217 | .parent = null, | 6220 | .parent = null, |
| 6218 | .builtin_mod = null, | 6221 | .builtin_mod = null, |
| 6222 | .builtin_modules = null, // there is only one module in this compilation | ||
| 6219 | }); | 6223 | }); |
| 6220 | 6224 | ||
| 6221 | for (c_source_files) |*item| { | 6225 | for (c_source_files) |*item| { |
src/Package/Module.zig+15-1
| ... | @@ -63,6 +63,11 @@ pub const CreateOptions = struct { | ... | @@ -63,6 +63,11 @@ pub const CreateOptions = struct { |
| 63 | 63 | ||
| 64 | builtin_mod: ?*Package.Module, | 64 | builtin_mod: ?*Package.Module, |
| 65 | 65 | ||
| 66 | /// Allocated into the given `arena`. Should be shared across all module creations in a Compilation. | ||
| 67 | /// Ignored if `builtin_mod` is passed or if `!have_zcu`. | ||
| 68 | /// Otherwise, may be `null` only if this Compilation consists of a single module. | ||
| 69 | builtin_modules: ?*std.StringHashMapUnmanaged(*Module), | ||
| 70 | |||
| 66 | pub const Paths = struct { | 71 | pub const Paths = struct { |
| 67 | root: Package.Path, | 72 | root: Package.Path, |
| 68 | /// Relative to `root`. May contain path separators. | 73 | /// Relative to `root`. May contain path separators. |
| ... | @@ -364,11 +369,20 @@ pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { | ... | @@ -364,11 +369,20 @@ pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { |
| 364 | .wasi_exec_model = options.global.wasi_exec_model, | 369 | .wasi_exec_model = options.global.wasi_exec_model, |
| 365 | }, arena); | 370 | }, arena); |
| 366 | 371 | ||
| 372 | const new = if (options.builtin_modules) |builtins| new: { | ||
| 373 | const gop = try builtins.getOrPut(arena, generated_builtin_source); | ||
| 374 | if (gop.found_existing) break :b gop.value_ptr.*; | ||
| 375 | errdefer builtins.removeByPtr(gop.key_ptr); | ||
| 376 | const new = try arena.create(Module); | ||
| 377 | gop.value_ptr.* = new; | ||
| 378 | break :new new; | ||
| 379 | } else try arena.create(Module); | ||
| 380 | errdefer if (options.builtin_modules) |builtins| assert(builtins.remove(generated_builtin_source)); | ||
| 381 | |||
| 367 | const new_file = try arena.create(File); | 382 | const new_file = try arena.create(File); |
| 368 | 383 | ||
| 369 | const digest = Cache.HashHelper.oneShot(generated_builtin_source); | 384 | const digest = Cache.HashHelper.oneShot(generated_builtin_source); |
| 370 | const builtin_sub_path = try arena.dupe(u8, "b" ++ std.fs.path.sep_str ++ digest); | 385 | const builtin_sub_path = try arena.dupe(u8, "b" ++ std.fs.path.sep_str ++ digest); |
| 371 | const new = try arena.create(Module); | ||
| 372 | new.* = .{ | 386 | new.* = .{ |
| 373 | .root = .{ | 387 | .root = .{ |
| 374 | .root_dir = options.global_cache_directory, | 388 | .root_dir = options.global_cache_directory, |
src/Sema.zig+1
| ... | @@ -5858,6 +5858,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr | ... | @@ -5858,6 +5858,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr |
| 5858 | .global = comp.config, | 5858 | .global = comp.config, |
| 5859 | .parent = parent_mod, | 5859 | .parent = parent_mod, |
| 5860 | .builtin_mod = parent_mod.getBuiltinDependency(), | 5860 | .builtin_mod = parent_mod.getBuiltinDependency(), |
| 5861 | .builtin_modules = null, // `builtin_mod` is set | ||
| 5861 | }) catch |err| switch (err) { | 5862 | }) catch |err| switch (err) { |
| 5862 | // None of these are possible because we are creating a package with | 5863 | // None of these are possible because we are creating a package with |
| 5863 | // the exact same configuration as the parent package, which already | 5864 | // the exact same configuration as the parent package, which already |
src/glibc.zig+1
| ... | @@ -1118,6 +1118,7 @@ fn buildSharedLib( | ... | @@ -1118,6 +1118,7 @@ fn buildSharedLib( |
| 1118 | .cc_argv = &.{}, | 1118 | .cc_argv = &.{}, |
| 1119 | .parent = null, | 1119 | .parent = null, |
| 1120 | .builtin_mod = null, | 1120 | .builtin_mod = null, |
| 1121 | .builtin_modules = null, // there is only one module in this compilation | ||
| 1121 | }); | 1122 | }); |
| 1122 | 1123 | ||
| 1123 | const c_source_files = [1]Compilation.CSourceFile{ | 1124 | const c_source_files = [1]Compilation.CSourceFile{ |
src/libcxx.zig+2
| ... | @@ -181,6 +181,7 @@ pub fn buildLibCXX(comp: *Compilation, prog_node: *std.Progress.Node) !void { | ... | @@ -181,6 +181,7 @@ pub fn buildLibCXX(comp: *Compilation, prog_node: *std.Progress.Node) !void { |
| 181 | .cc_argv = &.{}, | 181 | .cc_argv = &.{}, |
| 182 | .parent = null, | 182 | .parent = null, |
| 183 | .builtin_mod = null, | 183 | .builtin_mod = null, |
| 184 | .builtin_modules = null, // there is only one module in this compilation | ||
| 184 | }); | 185 | }); |
| 185 | 186 | ||
| 186 | var c_source_files = try std.ArrayList(Compilation.CSourceFile).initCapacity(arena, libcxx_files.len); | 187 | var c_source_files = try std.ArrayList(Compilation.CSourceFile).initCapacity(arena, libcxx_files.len); |
| ... | @@ -395,6 +396,7 @@ pub fn buildLibCXXABI(comp: *Compilation, prog_node: *std.Progress.Node) !void { | ... | @@ -395,6 +396,7 @@ pub fn buildLibCXXABI(comp: *Compilation, prog_node: *std.Progress.Node) !void { |
| 395 | .cc_argv = &.{}, | 396 | .cc_argv = &.{}, |
| 396 | .parent = null, | 397 | .parent = null, |
| 397 | .builtin_mod = null, | 398 | .builtin_mod = null, |
| 399 | .builtin_modules = null, // there is only one module in this compilation | ||
| 398 | }); | 400 | }); |
| 399 | 401 | ||
| 400 | var c_source_files = try std.ArrayList(Compilation.CSourceFile).initCapacity(arena, libcxxabi_files.len); | 402 | var c_source_files = try std.ArrayList(Compilation.CSourceFile).initCapacity(arena, libcxxabi_files.len); |
src/libtsan.zig+1
| ... | @@ -92,6 +92,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) BuildError!v | ... | @@ -92,6 +92,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) BuildError!v |
| 92 | .cc_argv = &common_flags, | 92 | .cc_argv = &common_flags, |
| 93 | .parent = null, | 93 | .parent = null, |
| 94 | .builtin_mod = null, | 94 | .builtin_mod = null, |
| 95 | .builtin_modules = null, // there is only one module in this compilation | ||
| 95 | }) catch |err| { | 96 | }) catch |err| { |
| 96 | comp.setMiscFailure( | 97 | comp.setMiscFailure( |
| 97 | .libtsan, | 98 | .libtsan, |
src/libunwind.zig+1
| ... | @@ -58,6 +58,7 @@ pub fn buildStaticLib(comp: *Compilation, prog_node: *std.Progress.Node) !void { | ... | @@ -58,6 +58,7 @@ pub fn buildStaticLib(comp: *Compilation, prog_node: *std.Progress.Node) !void { |
| 58 | .cc_argv = &.{}, | 58 | .cc_argv = &.{}, |
| 59 | .parent = null, | 59 | .parent = null, |
| 60 | .builtin_mod = null, | 60 | .builtin_mod = null, |
| 61 | .builtin_modules = null, // there is only one module in this compilation | ||
| 61 | }); | 62 | }); |
| 62 | 63 | ||
| 63 | const root_name = "unwind"; | 64 | const root_name = "unwind"; |
src/main.zig+15-3
| ... | @@ -2697,7 +2697,9 @@ fn buildOutputType( | ... | @@ -2697,7 +2697,9 @@ fn buildOutputType( |
| 2697 | create_module.opts.emit_bin = emit_bin != .no; | 2697 | create_module.opts.emit_bin = emit_bin != .no; |
| 2698 | create_module.opts.any_c_source_files = create_module.c_source_files.items.len != 0; | 2698 | create_module.opts.any_c_source_files = create_module.c_source_files.items.len != 0; |
| 2699 | 2699 | ||
| 2700 | const main_mod = try createModule(gpa, arena, &create_module, 0, null, zig_lib_directory); | 2700 | var builtin_modules: std.StringHashMapUnmanaged(*Package.Module) = .{}; |
| 2701 | // `builtin_modules` allocated into `arena`, so no deinit | ||
| 2702 | const main_mod = try createModule(gpa, arena, &create_module, 0, null, zig_lib_directory, &builtin_modules); | ||
| 2701 | for (create_module.modules.keys(), create_module.modules.values()) |key, cli_mod| { | 2703 | for (create_module.modules.keys(), create_module.modules.values()) |key, cli_mod| { |
| 2702 | if (cli_mod.resolved == null) | 2704 | if (cli_mod.resolved == null) |
| 2703 | fatal("module '{s}' declared but not used", .{key}); | 2705 | fatal("module '{s}' declared but not used", .{key}); |
| ... | @@ -2742,6 +2744,7 @@ fn buildOutputType( | ... | @@ -2742,6 +2744,7 @@ fn buildOutputType( |
| 2742 | .global = create_module.resolved_options, | 2744 | .global = create_module.resolved_options, |
| 2743 | .parent = main_mod, | 2745 | .parent = main_mod, |
| 2744 | .builtin_mod = main_mod.getBuiltinDependency(), | 2746 | .builtin_mod = main_mod.getBuiltinDependency(), |
| 2747 | .builtin_modules = null, // `builtin_mod` is specified | ||
| 2745 | }); | 2748 | }); |
| 2746 | test_mod.deps = try main_mod.deps.clone(arena); | 2749 | test_mod.deps = try main_mod.deps.clone(arena); |
| 2747 | break :test_mod test_mod; | 2750 | break :test_mod test_mod; |
| ... | @@ -2760,6 +2763,7 @@ fn buildOutputType( | ... | @@ -2760,6 +2763,7 @@ fn buildOutputType( |
| 2760 | .global = create_module.resolved_options, | 2763 | .global = create_module.resolved_options, |
| 2761 | .parent = main_mod, | 2764 | .parent = main_mod, |
| 2762 | .builtin_mod = main_mod.getBuiltinDependency(), | 2765 | .builtin_mod = main_mod.getBuiltinDependency(), |
| 2766 | .builtin_modules = null, // `builtin_mod` is specified | ||
| 2763 | }); | 2767 | }); |
| 2764 | 2768 | ||
| 2765 | break :root_mod test_mod; | 2769 | break :root_mod test_mod; |
| ... | @@ -3467,6 +3471,7 @@ fn createModule( | ... | @@ -3467,6 +3471,7 @@ fn createModule( |
| 3467 | index: usize, | 3471 | index: usize, |
| 3468 | parent: ?*Package.Module, | 3472 | parent: ?*Package.Module, |
| 3469 | zig_lib_directory: Cache.Directory, | 3473 | zig_lib_directory: Cache.Directory, |
| 3474 | builtin_modules: *std.StringHashMapUnmanaged(*Package.Module), | ||
| 3470 | ) Allocator.Error!*Package.Module { | 3475 | ) Allocator.Error!*Package.Module { |
| 3471 | const cli_mod = &create_module.modules.values()[index]; | 3476 | const cli_mod = &create_module.modules.values()[index]; |
| 3472 | if (cli_mod.resolved) |m| return m; | 3477 | if (cli_mod.resolved) |m| return m; |
| ... | @@ -3919,6 +3924,7 @@ fn createModule( | ... | @@ -3919,6 +3924,7 @@ fn createModule( |
| 3919 | .global = create_module.resolved_options, | 3924 | .global = create_module.resolved_options, |
| 3920 | .parent = parent, | 3925 | .parent = parent, |
| 3921 | .builtin_mod = null, | 3926 | .builtin_mod = null, |
| 3927 | .builtin_modules = builtin_modules, | ||
| 3922 | }) catch |err| switch (err) { | 3928 | }) catch |err| switch (err) { |
| 3923 | error.ValgrindUnsupportedOnTarget => fatal("unable to create module '{s}': valgrind does not support the selected target CPU architecture", .{name}), | 3929 | error.ValgrindUnsupportedOnTarget => fatal("unable to create module '{s}': valgrind does not support the selected target CPU architecture", .{name}), |
| 3924 | error.TargetRequiresSingleThreaded => fatal("unable to create module '{s}': the selected target does not support multithreading", .{name}), | 3930 | error.TargetRequiresSingleThreaded => fatal("unable to create module '{s}': the selected target does not support multithreading", .{name}), |
| ... | @@ -3941,7 +3947,7 @@ fn createModule( | ... | @@ -3941,7 +3947,7 @@ fn createModule( |
| 3941 | for (cli_mod.deps) |dep| { | 3947 | for (cli_mod.deps) |dep| { |
| 3942 | const dep_index = create_module.modules.getIndex(dep.value) orelse | 3948 | const dep_index = create_module.modules.getIndex(dep.value) orelse |
| 3943 | fatal("module '{s}' depends on non-existent module '{s}'", .{ name, dep.key }); | 3949 | fatal("module '{s}' depends on non-existent module '{s}'", .{ name, dep.key }); |
| 3944 | const dep_mod = try createModule(gpa, arena, create_module, dep_index, mod, zig_lib_directory); | 3950 | const dep_mod = try createModule(gpa, arena, create_module, dep_index, mod, zig_lib_directory, builtin_modules); |
| 3945 | try mod.deps.put(arena, dep.key, dep_mod); | 3951 | try mod.deps.put(arena, dep.key, dep_mod); |
| 3946 | } | 3952 | } |
| 3947 | 3953 | ||
| ... | @@ -5237,6 +5243,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { | ... | @@ -5237,6 +5243,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { |
| 5237 | .global = config, | 5243 | .global = config, |
| 5238 | .parent = null, | 5244 | .parent = null, |
| 5239 | .builtin_mod = null, | 5245 | .builtin_mod = null, |
| 5246 | .builtin_modules = null, // all modules will inherit this one's builtin | ||
| 5240 | }); | 5247 | }); |
| 5241 | 5248 | ||
| 5242 | const builtin_mod = root_mod.getBuiltinDependency(); | 5249 | const builtin_mod = root_mod.getBuiltinDependency(); |
| ... | @@ -5253,6 +5260,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { | ... | @@ -5253,6 +5260,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { |
| 5253 | .global = config, | 5260 | .global = config, |
| 5254 | .parent = root_mod, | 5261 | .parent = root_mod, |
| 5255 | .builtin_mod = builtin_mod, | 5262 | .builtin_mod = builtin_mod, |
| 5263 | .builtin_modules = null, // `builtin_mod` is specified | ||
| 5256 | }); | 5264 | }); |
| 5257 | 5265 | ||
| 5258 | var cleanup_build_dir: ?fs.Dir = null; | 5266 | var cleanup_build_dir: ?fs.Dir = null; |
| ... | @@ -5387,6 +5395,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { | ... | @@ -5387,6 +5395,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { |
| 5387 | .global = config, | 5395 | .global = config, |
| 5388 | .parent = root_mod, | 5396 | .parent = root_mod, |
| 5389 | .builtin_mod = builtin_mod, | 5397 | .builtin_mod = builtin_mod, |
| 5398 | .builtin_modules = null, // `builtin_mod` is specified | ||
| 5390 | }); | 5399 | }); |
| 5391 | const hash_cloned = try arena.dupe(u8, &hash); | 5400 | const hash_cloned = try arena.dupe(u8, &hash); |
| 5392 | deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); | 5401 | deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); |
| ... | @@ -5636,6 +5645,7 @@ fn jitCmd( | ... | @@ -5636,6 +5645,7 @@ fn jitCmd( |
| 5636 | .global = config, | 5645 | .global = config, |
| 5637 | .parent = null, | 5646 | .parent = null, |
| 5638 | .builtin_mod = null, | 5647 | .builtin_mod = null, |
| 5648 | .builtin_modules = null, // all modules will inherit this one's builtin | ||
| 5639 | }); | 5649 | }); |
| 5640 | 5650 | ||
| 5641 | if (options.depend_on_aro) { | 5651 | if (options.depend_on_aro) { |
| ... | @@ -5658,6 +5668,7 @@ fn jitCmd( | ... | @@ -5658,6 +5668,7 @@ fn jitCmd( |
| 5658 | .global = config, | 5668 | .global = config, |
| 5659 | .parent = null, | 5669 | .parent = null, |
| 5660 | .builtin_mod = root_mod.getBuiltinDependency(), | 5670 | .builtin_mod = root_mod.getBuiltinDependency(), |
| 5671 | .builtin_modules = null, // `builtin_mod` is specified | ||
| 5661 | }); | 5672 | }); |
| 5662 | try root_mod.deps.put(arena, "aro", aro_mod); | 5673 | try root_mod.deps.put(arena, "aro", aro_mod); |
| 5663 | } | 5674 | } |
| ... | @@ -7204,10 +7215,11 @@ fn createDependenciesModule( | ... | @@ -7204,10 +7215,11 @@ fn createDependenciesModule( |
| 7204 | }, | 7215 | }, |
| 7205 | .fully_qualified_name = "root.@dependencies", | 7216 | .fully_qualified_name = "root.@dependencies", |
| 7206 | .parent = main_mod, | 7217 | .parent = main_mod, |
| 7207 | .builtin_mod = builtin_mod, | ||
| 7208 | .cc_argv = &.{}, | 7218 | .cc_argv = &.{}, |
| 7209 | .inherited = .{}, | 7219 | .inherited = .{}, |
| 7210 | .global = global_options, | 7220 | .global = global_options, |
| 7221 | .builtin_mod = builtin_mod, | ||
| 7222 | .builtin_modules = null, // `builtin_mod` is specified | ||
| 7211 | }); | 7223 | }); |
| 7212 | try main_mod.deps.put(arena, "@dependencies", deps_mod); | 7224 | try main_mod.deps.put(arena, "@dependencies", deps_mod); |
| 7213 | return deps_mod; | 7225 | return deps_mod; |
src/musl.zig+1
| ... | @@ -250,6 +250,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile, prog_node: *std.Progr | ... | @@ -250,6 +250,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile, prog_node: *std.Progr |
| 250 | .cc_argv = cc_argv, | 250 | .cc_argv = cc_argv, |
| 251 | .parent = null, | 251 | .parent = null, |
| 252 | .builtin_mod = null, | 252 | .builtin_mod = null, |
| 253 | .builtin_modules = null, // there is only one module in this compilation | ||
| 253 | }); | 254 | }); |
| 254 | 255 | ||
| 255 | const sub_compilation = try Compilation.create(comp.gpa, arena, .{ | 256 | const sub_compilation = try Compilation.create(comp.gpa, arena, .{ |