authorgravatar for jagt@live.comjagt <jagt@live.com> 2022-03-25 22:53:58+08:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-03-27 11:23:11+03:00
log29c32b3dc50218d15f779201e154d425fffcefeb
treeb16349f3069554879e6bb060d29357b3d93cc98f
parent23507f20b943e7923978b293cb03bee886f913ed

`Namespace.decls` use context to save memory

change `Module.Namespace.decls` from `AutoArrayHashMapUnmanaged` to `ArrayHashMapUnmanaged(*Decl, void)` with custom context to eliminate duplicated decl name strings. Also see: https://zig.news/andrewrk/how-to-use-hash-map-contexts-to-save-memory-when-doing-a-string-table-3l33

2 files changed, 40 insertions(+), 16 deletions(-)

src/Module.zig+36-11
...@@ -1591,6 +1591,19 @@ pub const Var = struct {...@@ -1591,6 +1591,19 @@ pub const Var = struct {
1591 }1591 }
1592};1592};
15931593
1594pub const DeclAdapter = struct {
1595 pub fn hash(self: @This(), s: []const u8) u32 {
1596 _ = self;
1597 return @truncate(u32, std.hash.Wyhash.hash(0, s));
1598 }
1599
1600 pub fn eql(self: @This(), a: []const u8, b_decl: *Decl, b_index: usize) bool {
1601 _ = self;
1602 _ = b_index;
1603 return mem.eql(u8, a, mem.sliceTo(b_decl.name, 0));
1604 }
1605};
1606
1594/// The container that structs, enums, unions, and opaques have.1607/// The container that structs, enums, unions, and opaques have.
1595pub const Namespace = struct {1608pub const Namespace = struct {
1596 parent: ?*Namespace,1609 parent: ?*Namespace,
...@@ -1601,9 +1614,8 @@ pub const Namespace = struct {...@@ -1601,9 +1614,8 @@ pub const Namespace = struct {
1601 /// which decls have been added/removed from source.1614 /// which decls have been added/removed from source.
1602 /// Declaration order is preserved via entry order.1615 /// Declaration order is preserved via entry order.
1603 /// Key memory is owned by `decl.name`.1616 /// Key memory is owned by `decl.name`.
1604 /// TODO save memory with https://github.com/ziglang/zig/issues/8619.
1605 /// Anonymous decls are not stored here; they are kept in `anon_decls` instead.1617 /// Anonymous decls are not stored here; they are kept in `anon_decls` instead.
1606 decls: std.StringArrayHashMapUnmanaged(*Decl) = .{},1618 decls: std.ArrayHashMapUnmanaged(*Decl, void, DeclContext, true) = .{},
16071619
1608 anon_decls: std.AutoArrayHashMapUnmanaged(*Decl, void) = .{},1620 anon_decls: std.AutoArrayHashMapUnmanaged(*Decl, void) = .{},
16091621
...@@ -1612,6 +1624,19 @@ pub const Namespace = struct {...@@ -1612,6 +1624,19 @@ pub const Namespace = struct {
1612 /// Value is whether the usingnamespace decl is marked `pub`.1624 /// Value is whether the usingnamespace decl is marked `pub`.
1613 usingnamespace_set: std.AutoHashMapUnmanaged(*Decl, bool) = .{},1625 usingnamespace_set: std.AutoHashMapUnmanaged(*Decl, bool) = .{},
16141626
1627 const DeclContext = struct {
1628 pub fn hash(self: @This(), decl: *Decl) u32 {
1629 _ = self;
1630 return @truncate(u32, std.hash.Wyhash.hash(0, mem.sliceTo(decl.name, 0)));
1631 }
1632
1633 pub fn eql(self: @This(), a: *Decl, b: *Decl, b_index: usize) bool {
1634 _ = self;
1635 _ = b_index;
1636 return mem.eql(u8, mem.sliceTo(a.name, 0), mem.sliceTo(b.name, 0));
1637 }
1638 };
1639
1615 pub fn deinit(ns: *Namespace, mod: *Module) void {1640 pub fn deinit(ns: *Namespace, mod: *Module) void {
1616 ns.destroyDecls(mod);1641 ns.destroyDecls(mod);
1617 ns.* = undefined;1642 ns.* = undefined;
...@@ -1628,8 +1653,8 @@ pub const Namespace = struct {...@@ -1628,8 +1653,8 @@ pub const Namespace = struct {
1628 var anon_decls = ns.anon_decls;1653 var anon_decls = ns.anon_decls;
1629 ns.anon_decls = .{};1654 ns.anon_decls = .{};
16301655
1631 for (decls.values()) |value| {1656 for (decls.keys()) |decl| {
1632 value.destroy(mod);1657 decl.destroy(mod);
1633 }1658 }
1634 decls.deinit(gpa);1659 decls.deinit(gpa);
16351660
...@@ -1658,7 +1683,7 @@ pub const Namespace = struct {...@@ -1658,7 +1683,7 @@ pub const Namespace = struct {
1658 // TODO rework this code to not panic on OOM.1683 // TODO rework this code to not panic on OOM.
1659 // (might want to coordinate with the clearDecl function)1684 // (might want to coordinate with the clearDecl function)
16601685
1661 for (decls.values()) |child_decl| {1686 for (decls.keys()) |child_decl| {
1662 mod.clearDecl(child_decl, outdated_decls) catch @panic("out of memory");1687 mod.clearDecl(child_decl, outdated_decls) catch @panic("out of memory");
1663 child_decl.destroy(mod);1688 child_decl.destroy(mod);
1664 }1689 }
...@@ -3325,7 +3350,7 @@ fn updateZirRefs(gpa: Allocator, file: *File, old_zir: Zir) !void {...@@ -3325,7 +3350,7 @@ fn updateZirRefs(gpa: Allocator, file: *File, old_zir: Zir) !void {
3325 }3350 }
33263351
3327 if (decl.getInnerNamespace()) |namespace| {3352 if (decl.getInnerNamespace()) |namespace| {
3328 for (namespace.decls.values()) |sub_decl| {3353 for (namespace.decls.keys()) |sub_decl| {
3329 try decl_stack.append(gpa, sub_decl);3354 try decl_stack.append(gpa, sub_decl);
3330 }3355 }
3331 for (namespace.anon_decls.keys()) |sub_decl| {3356 for (namespace.anon_decls.keys()) |sub_decl| {
...@@ -4420,7 +4445,7 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) SemaError!voi...@@ -4420,7 +4445,7 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) SemaError!voi
4420 if (is_usingnamespace) try namespace.usingnamespace_set.ensureUnusedCapacity(gpa, 1);4445 if (is_usingnamespace) try namespace.usingnamespace_set.ensureUnusedCapacity(gpa, 1);
44214446
4422 // We create a Decl for it regardless of analysis status.4447 // We create a Decl for it regardless of analysis status.
4423 const gop = try namespace.decls.getOrPut(gpa, decl_name);4448 const gop = try namespace.decls.getOrPutAdapted(gpa, @as([]const u8, mem.sliceTo(decl_name, 0)), DeclAdapter{});
4424 if (!gop.found_existing) {4449 if (!gop.found_existing) {
4425 const new_decl = try mod.allocateNewDecl(decl_name, namespace, decl_node, iter.parent_decl.src_scope);4450 const new_decl = try mod.allocateNewDecl(decl_name, namespace, decl_node, iter.parent_decl.src_scope);
4426 if (is_usingnamespace) {4451 if (is_usingnamespace) {
...@@ -4428,7 +4453,7 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) SemaError!voi...@@ -4428,7 +4453,7 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) SemaError!voi
4428 }4453 }
4429 log.debug("scan new {*} ({s}) into {*}", .{ new_decl, decl_name, namespace });4454 log.debug("scan new {*} ({s}) into {*}", .{ new_decl, decl_name, namespace });
4430 new_decl.src_line = line;4455 new_decl.src_line = line;
4431 gop.value_ptr.* = new_decl;4456 gop.key_ptr.* = new_decl;
4432 // Exported decls, comptime decls, usingnamespace decls, and4457 // Exported decls, comptime decls, usingnamespace decls, and
4433 // test decls if in test mode, get analyzed.4458 // test decls if in test mode, get analyzed.
4434 const decl_pkg = namespace.file_scope.pkg;4459 const decl_pkg = namespace.file_scope.pkg;
...@@ -4464,7 +4489,7 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) SemaError!voi...@@ -4464,7 +4489,7 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) SemaError!voi
4464 return;4489 return;
4465 }4490 }
4466 gpa.free(decl_name);4491 gpa.free(decl_name);
4467 const decl = gop.value_ptr.*;4492 const decl = gop.key_ptr.*;
4468 log.debug("scan existing {*} ({s}) of {*}", .{ decl, decl.name, namespace });4493 log.debug("scan existing {*} ({s}) of {*}", .{ decl, decl.name, namespace });
4469 // Update the AST node of the decl; even if its contents are unchanged, it may4494 // Update the AST node of the decl; even if its contents are unchanged, it may
4470 // have been re-ordered.4495 // have been re-ordered.
...@@ -5333,7 +5358,7 @@ pub fn processOutdatedAndDeletedDecls(mod: *Module) !void {...@@ -5333,7 +5358,7 @@ pub fn processOutdatedAndDeletedDecls(mod: *Module) !void {
53335358
5334 // Remove from the namespace it resides in, preserving declaration order.5359 // Remove from the namespace it resides in, preserving declaration order.
5335 assert(decl.zir_decl_index != 0);5360 assert(decl.zir_decl_index != 0);
5336 _ = decl.src_namespace.decls.orderedRemove(mem.sliceTo(decl.name, 0));5361 _ = decl.src_namespace.decls.orderedRemoveAdapted(@as([]const u8, mem.sliceTo(decl.name, 0)), DeclAdapter{});
53375362
5338 try mod.clearDecl(decl, &outdated_decls);5363 try mod.clearDecl(decl, &outdated_decls);
5339 decl.destroy(mod);5364 decl.destroy(mod);
...@@ -5400,7 +5425,7 @@ pub fn populateTestFunctions(mod: *Module) !void {...@@ -5400,7 +5425,7 @@ pub fn populateTestFunctions(mod: *Module) !void {
5400 const builtin_pkg = mod.main_pkg.table.get("builtin").?;5425 const builtin_pkg = mod.main_pkg.table.get("builtin").?;
5401 const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file;5426 const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file;
5402 const builtin_namespace = builtin_file.root_decl.?.src_namespace;5427 const builtin_namespace = builtin_file.root_decl.?.src_namespace;
5403 const decl = builtin_namespace.decls.get("test_functions").?;5428 const decl = builtin_namespace.decls.getKeyAdapted(@as([]const u8, "test_functions"), DeclAdapter{}).?;
5404 var buf: Type.SlicePtrFieldTypeBuffer = undefined;5429 var buf: Type.SlicePtrFieldTypeBuffer = undefined;
5405 const tmp_test_fn_ty = decl.ty.slicePtrFieldType(&buf).elemType();5430 const tmp_test_fn_ty = decl.ty.slicePtrFieldType(&buf).elemType();
54065431
src/Sema.zig+4-5
...@@ -4516,7 +4516,7 @@ fn lookupInNamespace(...@@ -4516,7 +4516,7 @@ fn lookupInNamespace(
45164516
4517 while (check_i < checked_namespaces.count()) : (check_i += 1) {4517 while (check_i < checked_namespaces.count()) : (check_i += 1) {
4518 const check_ns = checked_namespaces.keys()[check_i];4518 const check_ns = checked_namespaces.keys()[check_i];
4519 if (check_ns.decls.get(ident_name)) |decl| {4519 if (check_ns.decls.getKeyAdapted(ident_name, Module.DeclAdapter{})) |decl| {
4520 // Skip decls which are not marked pub, which are in a different4520 // Skip decls which are not marked pub, which are in a different
4521 // file than the `a.b`/`@hasDecl` syntax.4521 // file than the `a.b`/`@hasDecl` syntax.
4522 if (decl.is_pub or src_file == decl.getFileScope()) {4522 if (decl.is_pub or src_file == decl.getFileScope()) {
...@@ -4559,7 +4559,7 @@ fn lookupInNamespace(...@@ -4559,7 +4559,7 @@ fn lookupInNamespace(
4559 return sema.failWithOwnedErrorMsg(block, msg);4559 return sema.failWithOwnedErrorMsg(block, msg);
4560 },4560 },
4561 }4561 }
4562 } else if (namespace.decls.get(ident_name)) |decl| {4562 } else if (namespace.decls.getKeyAdapted(ident_name, Module.DeclAdapter{})) |decl| {
4563 try mod.declareDeclDependency(sema.owner_decl, decl);4563 try mod.declareDeclDependency(sema.owner_decl, decl);
4564 return decl;4564 return decl;
4565 }4565 }
...@@ -11502,12 +11502,11 @@ fn typeInfoDecls(...@@ -11502,12 +11502,11 @@ fn typeInfoDecls(
11502 const decls_len = if (opt_namespace) |ns| ns.decls.count() else 0;11502 const decls_len = if (opt_namespace) |ns| ns.decls.count() else 0;
11503 const decls_vals = try decls_anon_decl.arena().alloc(Value, decls_len);11503 const decls_vals = try decls_anon_decl.arena().alloc(Value, decls_len);
11504 for (decls_vals) |*decls_val, i| {11504 for (decls_vals) |*decls_val, i| {
11505 const decl = opt_namespace.?.decls.values()[i];11505 const decl = opt_namespace.?.decls.keys()[i];
11506 const name = opt_namespace.?.decls.keys()[i];
11507 const name_val = v: {11506 const name_val = v: {
11508 var anon_decl = try block.startAnonDecl(src);11507 var anon_decl = try block.startAnonDecl(src);
11509 defer anon_decl.deinit();11508 defer anon_decl.deinit();
11510 const bytes = try anon_decl.arena().dupeZ(u8, name);11509 const bytes = try anon_decl.arena().dupeZ(u8, mem.sliceTo(decl.name, 0));
11511 const new_decl = try anon_decl.finish(11510 const new_decl = try anon_decl.finish(
11512 try Type.Tag.array_u8_sentinel_0.create(anon_decl.arena(), bytes.len),11511 try Type.Tag.array_u8_sentinel_0.create(anon_decl.arena(), bytes.len),
11513 try Value.Tag.bytes.create(anon_decl.arena(), bytes[0 .. bytes.len + 1]),11512 try Value.Tag.bytes.create(anon_decl.arena(), bytes[0 .. bytes.len + 1]),