authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-10-04 05:05:31-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-10-05 00:27:39-04:00
log7de67e6802d4cec114d5ede1892abe428e331cac
tree45371f4ba0d7dd52d80e611e40c7262e8240a95a
parent2700af2aeb804c332fe11c9c26116769e78c3dbd

InternPool: use sequential string indices instead of byte offsets

This allows more bytes to be referenced by a smaller index range. Closes #22867 Closes #25297 Closes #25339

5 files changed, 102 insertions(+), 63 deletions(-)

src/Compilation.zig+7-2
...@@ -3666,6 +3666,7 @@ const Header = extern struct {...@@ -3666,6 +3666,7 @@ const Header = extern struct {
3666 items_len: u32,3666 items_len: u32,
3667 extra_len: u32,3667 extra_len: u32,
3668 limbs_len: u32,3668 limbs_len: u32,
3669 strings_len: u32,
3669 string_bytes_len: u32,3670 string_bytes_len: u32,
3670 tracked_insts_len: u32,3671 tracked_insts_len: u32,
3671 files_len: u32,3672 files_len: u32,
...@@ -3714,7 +3715,8 @@ pub fn saveState(comp: *Compilation) !void {...@@ -3714,7 +3715,8 @@ pub fn saveState(comp: *Compilation) !void {
3714 .items_len = @intCast(local.mutate.items.len),3715 .items_len = @intCast(local.mutate.items.len),
3715 .extra_len = @intCast(local.mutate.extra.len),3716 .extra_len = @intCast(local.mutate.extra.len),
3716 .limbs_len = @intCast(local.mutate.limbs.len),3717 .limbs_len = @intCast(local.mutate.limbs.len),
3717 .string_bytes_len = @intCast(local.mutate.strings.len),3718 .strings_len = @intCast(local.mutate.strings.len),
3719 .string_bytes_len = @intCast(local.mutate.string_bytes.len),
3718 .tracked_insts_len = @intCast(local.mutate.tracked_insts.len),3720 .tracked_insts_len = @intCast(local.mutate.tracked_insts.len),
3719 .files_len = @intCast(local.mutate.files.len),3721 .files_len = @intCast(local.mutate.files.len),
3720 },3722 },
...@@ -3757,8 +3759,11 @@ pub fn saveState(comp: *Compilation) !void {...@@ -3757,8 +3759,11 @@ pub fn saveState(comp: *Compilation) !void {
3757 addBuf(&bufs, @ptrCast(local.shared.items.view().items(.data)[0..pt_header.intern_pool.items_len]));3759 addBuf(&bufs, @ptrCast(local.shared.items.view().items(.data)[0..pt_header.intern_pool.items_len]));
3758 addBuf(&bufs, @ptrCast(local.shared.items.view().items(.tag)[0..pt_header.intern_pool.items_len]));3760 addBuf(&bufs, @ptrCast(local.shared.items.view().items(.tag)[0..pt_header.intern_pool.items_len]));
3759 }3761 }
3762 if (pt_header.intern_pool.strings_len > 0) {
3763 addBuf(&bufs, @ptrCast(local.shared.strings.view().items(.@"0")[0..pt_header.intern_pool.strings_len]));
3764 }
3760 if (pt_header.intern_pool.string_bytes_len > 0) {3765 if (pt_header.intern_pool.string_bytes_len > 0) {
3761 addBuf(&bufs, local.shared.strings.view().items(.@"0")[0..pt_header.intern_pool.string_bytes_len]);3766 addBuf(&bufs, local.shared.string_bytes.view().items(.@"0")[0..pt_header.intern_pool.string_bytes_len]);
3762 }3767 }
3763 if (pt_header.intern_pool.tracked_insts_len > 0) {3768 if (pt_header.intern_pool.tracked_insts_len > 0) {
3764 addBuf(&bufs, @ptrCast(local.shared.tracked_insts.view().items(.@"0")[0..pt_header.intern_pool.tracked_insts_len]));3769 addBuf(&bufs, @ptrCast(local.shared.tracked_insts.view().items(.@"0")[0..pt_header.intern_pool.tracked_insts_len]));
src/InternPool.zig+82-48
...@@ -182,7 +182,7 @@ pub const TrackedInst = extern struct {...@@ -182,7 +182,7 @@ pub const TrackedInst = extern struct {
182 pub fn wrap(unwrapped: Unwrapped, ip: *const InternPool) TrackedInst.Index {182 pub fn wrap(unwrapped: Unwrapped, ip: *const InternPool) TrackedInst.Index {
183 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());183 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
184 assert(unwrapped.index <= ip.getIndexMask(u32));184 assert(unwrapped.index <= ip.getIndexMask(u32));
185 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |185 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
186 unwrapped.index);186 unwrapped.index);
187 }187 }
188 };188 };
...@@ -480,7 +480,7 @@ pub const ComptimeUnit = extern struct {...@@ -480,7 +480,7 @@ pub const ComptimeUnit = extern struct {
480 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) ComptimeUnit.Id {480 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) ComptimeUnit.Id {
481 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());481 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
482 assert(unwrapped.index <= ip.getIndexMask(u32));482 assert(unwrapped.index <= ip.getIndexMask(u32));
483 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |483 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
484 unwrapped.index);484 unwrapped.index);
485 }485 }
486 };486 };
...@@ -699,7 +699,7 @@ pub const Nav = struct {...@@ -699,7 +699,7 @@ pub const Nav = struct {
699 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Nav.Index {699 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Nav.Index {
700 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());700 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
701 assert(unwrapped.index <= ip.getIndexMask(u32));701 assert(unwrapped.index <= ip.getIndexMask(u32));
702 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |702 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
703 unwrapped.index);703 unwrapped.index);
704 }704 }
705 };705 };
...@@ -1047,6 +1047,7 @@ const Local = struct {...@@ -1047,6 +1047,7 @@ const Local = struct {
1047 extra: ListMutate,1047 extra: ListMutate,
1048 limbs: ListMutate,1048 limbs: ListMutate,
1049 strings: ListMutate,1049 strings: ListMutate,
1050 string_bytes: ListMutate,
1050 tracked_insts: ListMutate,1051 tracked_insts: ListMutate,
1051 files: ListMutate,1052 files: ListMutate,
1052 maps: ListMutate,1053 maps: ListMutate,
...@@ -1061,6 +1062,7 @@ const Local = struct {...@@ -1061,6 +1062,7 @@ const Local = struct {
1061 extra: Extra,1062 extra: Extra,
1062 limbs: Limbs,1063 limbs: Limbs,
1063 strings: Strings,1064 strings: Strings,
1065 string_bytes: StringBytes,
1064 tracked_insts: TrackedInsts,1066 tracked_insts: TrackedInsts,
1065 files: List(File),1067 files: List(File),
1066 maps: Maps,1068 maps: Maps,
...@@ -1084,7 +1086,8 @@ const Local = struct {...@@ -1084,7 +1086,8 @@ const Local = struct {
1084 @sizeOf(u64) => List(struct { u64 }),1086 @sizeOf(u64) => List(struct { u64 }),
1085 else => @compileError("unsupported host"),1087 else => @compileError("unsupported host"),
1086 };1088 };
1087 const Strings = List(struct { u8 });1089 const Strings = List(struct { u32 });
1090 const StringBytes = List(struct { u8 });
1088 const TrackedInsts = List(struct { TrackedInst.MaybeLost });1091 const TrackedInsts = List(struct { TrackedInst.MaybeLost });
1089 const Maps = List(struct { FieldMap });1092 const Maps = List(struct { FieldMap });
1090 const Navs = List(Nav.Repr);1093 const Navs = List(Nav.Repr);
...@@ -1414,17 +1417,27 @@ const Local = struct {...@@ -1414,17 +1417,27 @@ const Local = struct {
1414 };1417 };
1415 }1418 }
14161419
1420 /// A list of offsets into `string_bytes` for each string.
1421 pub fn getMutableStrings(local: *Local, gpa: Allocator) Strings.Mutable {
1422 return .{
1423 .gpa = gpa,
1424 .arena = &local.mutate.arena,
1425 .mutate = &local.mutate.strings,
1426 .list = &local.shared.strings,
1427 };
1428 }
1429
1417 /// In order to store references to strings in fewer bytes, we copy all1430 /// In order to store references to strings in fewer bytes, we copy all
1418 /// string bytes into here. String bytes can be null. It is up to whomever1431 /// string bytes into here. String bytes can be null. It is up to whomever
1419 /// is referencing the data here whether they want to store both index and length,1432 /// is referencing the data here whether they want to store both index and length,
1420 /// thus allowing null bytes, or store only index, and use null-termination. The1433 /// thus allowing null bytes, or store only index, and use null-termination. The
1421 /// `strings` array is agnostic to either usage.1434 /// `strings_bytes` array is agnostic to either usage.
1422 pub fn getMutableStrings(local: *Local, gpa: Allocator) Strings.Mutable {1435 pub fn getMutableStringBytes(local: *Local, gpa: Allocator) StringBytes.Mutable {
1423 return .{1436 return .{
1424 .gpa = gpa,1437 .gpa = gpa,
1425 .arena = &local.mutate.arena,1438 .arena = &local.mutate.arena,
1426 .mutate = &local.mutate.strings,1439 .mutate = &local.mutate.string_bytes,
1427 .list = &local.shared.strings,1440 .list = &local.shared.string_bytes,
1428 };1441 };
1429 }1442 }
14301443
...@@ -1597,7 +1610,7 @@ const Shard = struct {...@@ -1597,7 +1610,7 @@ const Shard = struct {
1597};1610};
15981611
1599fn getTidMask(ip: *const InternPool) u32 {1612fn getTidMask(ip: *const InternPool) u32 {
1600 return (@as(u32, 1) << ip.tid_width) - 1;1613 return @shlExact(@as(u32, 1), ip.tid_width) - 1;
1601}1614}
16021615
1603fn getIndexMask(ip: *const InternPool, comptime BackingInt: type) u32 {1616fn getIndexMask(ip: *const InternPool, comptime BackingInt: type) u32 {
...@@ -1652,7 +1665,7 @@ pub const MapIndex = enum(u32) {...@@ -1652,7 +1665,7 @@ pub const MapIndex = enum(u32) {
1652 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) MapIndex {1665 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) MapIndex {
1653 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());1666 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1654 assert(unwrapped.index <= ip.getIndexMask(u32));1667 assert(unwrapped.index <= ip.getIndexMask(u32));
1655 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |1668 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
1656 unwrapped.index);1669 unwrapped.index);
1657 }1670 }
1658 };1671 };
...@@ -1678,7 +1691,7 @@ pub const NamespaceIndex = enum(u32) {...@@ -1678,7 +1691,7 @@ pub const NamespaceIndex = enum(u32) {
1678 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());1691 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1679 assert(unwrapped.bucket_index <= ip.getIndexMask(u32) >> Local.namespaces_bucket_width);1692 assert(unwrapped.bucket_index <= ip.getIndexMask(u32) >> Local.namespaces_bucket_width);
1680 assert(unwrapped.index <= Local.namespaces_bucket_mask);1693 assert(unwrapped.index <= Local.namespaces_bucket_mask);
1681 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |1694 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
1682 unwrapped.bucket_index << Local.namespaces_bucket_width |1695 unwrapped.bucket_index << Local.namespaces_bucket_width |
1683 unwrapped.index);1696 unwrapped.index);
1684 }1697 }
...@@ -1721,7 +1734,7 @@ pub const FileIndex = enum(u32) {...@@ -1721,7 +1734,7 @@ pub const FileIndex = enum(u32) {
1721 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) FileIndex {1734 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) FileIndex {
1722 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());1735 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1723 assert(unwrapped.index <= ip.getIndexMask(u32));1736 assert(unwrapped.index <= ip.getIndexMask(u32));
1724 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |1737 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
1725 unwrapped.index);1738 unwrapped.index);
1726 }1739 }
1727 };1740 };
...@@ -1780,7 +1793,8 @@ pub const String = enum(u32) {...@@ -1780,7 +1793,8 @@ pub const String = enum(u32) {
1780 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) String {1793 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) String {
1781 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());1794 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1782 assert(unwrapped.index <= ip.getIndexMask(u32));1795 assert(unwrapped.index <= ip.getIndexMask(u32));
1783 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | unwrapped.index);1796 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
1797 unwrapped.index);
1784 }1798 }
1785 };1799 };
1786 fn unwrap(string: String, ip: *const InternPool) Unwrapped {1800 fn unwrap(string: String, ip: *const InternPool) Unwrapped {
...@@ -1791,9 +1805,11 @@ pub const String = enum(u32) {...@@ -1791,9 +1805,11 @@ pub const String = enum(u32) {
1791 }1805 }
17921806
1793 fn toOverlongSlice(string: String, ip: *const InternPool) []const u8 {1807 fn toOverlongSlice(string: String, ip: *const InternPool) []const u8 {
1794 const unwrapped_string = string.unwrap(ip);1808 const unwrapped = string.unwrap(ip);
1795 const strings = ip.getLocalShared(unwrapped_string.tid).strings.acquire();1809 const local_shared = ip.getLocalShared(unwrapped.tid);
1796 return strings.view().items(.@"0")[unwrapped_string.index..];1810 const strings = local_shared.strings.acquire().view().items(.@"0");
1811 const string_bytes = local_shared.string_bytes.acquire().view().items(.@"0");
1812 return string_bytes[strings[unwrapped.index]..];
1797 }1813 }
17981814
1799 const debug_state = InternPool.debug_state;1815 const debug_state = InternPool.debug_state;
...@@ -1848,12 +1864,18 @@ pub const NullTerminatedString = enum(u32) {...@@ -1848,12 +1864,18 @@ pub const NullTerminatedString = enum(u32) {
1848 }1864 }
18491865
1850 pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {1866 pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {
1851 const overlong_slice = string.toString().toOverlongSlice(ip);1867 const unwrapped = string.toString().unwrap(ip);
1852 return overlong_slice[0..std.mem.indexOfScalar(u8, overlong_slice, 0).? :0];1868 const local_shared = ip.getLocalShared(unwrapped.tid);
1869 const strings = local_shared.strings.acquire().view().items(.@"0");
1870 const string_bytes = local_shared.string_bytes.acquire().view().items(.@"0");
1871 return string_bytes[strings[unwrapped.index] .. strings[unwrapped.index + 1] - 1 :0];
1853 }1872 }
18541873
1855 pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {1874 pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {
1856 return @intCast(string.toSlice(ip).len);1875 const unwrapped = string.toString().unwrap(ip);
1876 const local_shared = ip.getLocalShared(unwrapped.tid);
1877 const strings = local_shared.strings.acquire().view().items(.@"0");
1878 return strings[unwrapped.index + 1] - 1 - strings[unwrapped.index];
1857 }1879 }
18581880
1859 pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {1881 pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {
...@@ -4767,7 +4789,8 @@ pub const Index = enum(u32) {...@@ -4767,7 +4789,8 @@ pub const Index = enum(u32) {
4767 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Index {4789 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Index {
4768 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());4790 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
4769 assert(unwrapped.index <= ip.getIndexMask(u30));4791 assert(unwrapped.index <= ip.getIndexMask(u30));
4770 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_30 | unwrapped.index);4792 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_30) |
4793 unwrapped.index);
4771 }4794 }
47724795
4773 pub fn getExtra(unwrapped: Unwrapped, ip: *const InternPool) Local.Extra {4796 pub fn getExtra(unwrapped: Unwrapped, ip: *const InternPool) Local.Extra {
...@@ -6784,6 +6807,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {...@@ -6784,6 +6807,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
6784 .extra = .empty,6807 .extra = .empty,
6785 .limbs = .empty,6808 .limbs = .empty,
6786 .strings = .empty,6809 .strings = .empty,
6810 .string_bytes = .empty,
6787 .tracked_insts = .empty,6811 .tracked_insts = .empty,
6788 .files = .empty,6812 .files = .empty,
6789 .maps = .empty,6813 .maps = .empty,
...@@ -6799,6 +6823,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {...@@ -6799,6 +6823,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
6799 .extra = .empty,6823 .extra = .empty,
6800 .limbs = .empty,6824 .limbs = .empty,
6801 .strings = .empty,6825 .strings = .empty,
6826 .string_bytes = .empty,
6802 .tracked_insts = .empty,6827 .tracked_insts = .empty,
6803 .files = .empty,6828 .files = .empty,
6804 .maps = .empty,6829 .maps = .empty,
...@@ -6808,6 +6833,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {...@@ -6808,6 +6833,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
6808 .namespaces = .empty,6833 .namespaces = .empty,
6809 },6834 },
6810 });6835 });
6836 for (ip.locals) |*local| try local.getMutableStrings(gpa).append(.{0});
68116837
6812 ip.tid_width = @intCast(std.math.log2_int_ceil(usize, used_threads));6838 ip.tid_width = @intCast(std.math.log2_int_ceil(usize, used_threads));
6813 ip.tid_shift_30 = if (single_threaded) 0 else 30 - ip.tid_width;6839 ip.tid_shift_30 = if (single_threaded) 0 else 30 - ip.tid_width;
...@@ -8515,30 +8541,30 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All...@@ -8515,30 +8541,30 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
8515 }8541 }
85168542
8517 if (child == .u8_type) bytes: {8543 if (child == .u8_type) bytes: {
8518 const strings = ip.getLocal(tid).getMutableStrings(gpa);8544 const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa);
8519 const start = strings.mutate.len;8545 const start = string_bytes.mutate.len;
8520 try strings.ensureUnusedCapacity(@intCast(len_including_sentinel + 1));8546 try string_bytes.ensureUnusedCapacity(@intCast(len_including_sentinel + 1));
8521 try extra.ensureUnusedCapacity(@typeInfo(Bytes).@"struct".fields.len);8547 try extra.ensureUnusedCapacity(@typeInfo(Bytes).@"struct".fields.len);
8522 switch (aggregate.storage) {8548 switch (aggregate.storage) {
8523 .bytes => |bytes| strings.appendSliceAssumeCapacity(.{bytes.toSlice(len, ip)}),8549 .bytes => |bytes| string_bytes.appendSliceAssumeCapacity(.{bytes.toSlice(len, ip)}),
8524 .elems => |elems| for (elems[0..@intCast(len)]) |elem| switch (ip.indexToKey(elem)) {8550 .elems => |elems| for (elems[0..@intCast(len)]) |elem| switch (ip.indexToKey(elem)) {
8525 .undef => {8551 .undef => {
8526 strings.shrinkRetainingCapacity(start);8552 string_bytes.shrinkRetainingCapacity(start);
8527 break :bytes;8553 break :bytes;
8528 },8554 },
8529 .int => |int| strings.appendAssumeCapacity(.{@intCast(int.storage.u64)}),8555 .int => |int| string_bytes.appendAssumeCapacity(.{@intCast(int.storage.u64)}),
8530 else => unreachable,8556 else => unreachable,
8531 },8557 },
8532 .repeated_elem => |elem| switch (ip.indexToKey(elem)) {8558 .repeated_elem => |elem| switch (ip.indexToKey(elem)) {
8533 .undef => break :bytes,8559 .undef => break :bytes,
8534 .int => |int| @memset(8560 .int => |int| @memset(
8535 strings.addManyAsSliceAssumeCapacity(@intCast(len))[0],8561 string_bytes.addManyAsSliceAssumeCapacity(@intCast(len))[0],
8536 @intCast(int.storage.u64),8562 @intCast(int.storage.u64),
8537 ),8563 ),
8538 else => unreachable,8564 else => unreachable,
8539 },8565 },
8540 }8566 }
8541 if (sentinel != .none) strings.appendAssumeCapacity(.{8567 if (sentinel != .none) string_bytes.appendAssumeCapacity(.{
8542 @intCast(ip.indexToKey(sentinel).int.storage.u64),8568 @intCast(ip.indexToKey(sentinel).int.storage.u64),
8543 });8569 });
8544 const string = try ip.getOrPutTrailingString(8570 const string = try ip.getOrPutTrailingString(
...@@ -11754,10 +11780,10 @@ pub fn getOrPutString(...@@ -11754,10 +11780,10 @@ pub fn getOrPutString(
11754 slice: []const u8,11780 slice: []const u8,
11755 comptime embedded_nulls: EmbeddedNulls,11781 comptime embedded_nulls: EmbeddedNulls,
11756) Allocator.Error!embedded_nulls.StringType() {11782) Allocator.Error!embedded_nulls.StringType() {
11757 const strings = ip.getLocal(tid).getMutableStrings(gpa);11783 const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa);
11758 try strings.ensureUnusedCapacity(slice.len + 1);11784 try string_bytes.ensureUnusedCapacity(slice.len + 1);
11759 strings.appendSliceAssumeCapacity(.{slice});11785 string_bytes.appendSliceAssumeCapacity(.{slice});
11760 strings.appendAssumeCapacity(.{0});11786 string_bytes.appendAssumeCapacity(.{0});
11761 return ip.getOrPutTrailingString(gpa, tid, @intCast(slice.len + 1), embedded_nulls);11787 return ip.getOrPutTrailingString(gpa, tid, @intCast(slice.len + 1), embedded_nulls);
11762}11788}
1176311789
...@@ -11772,8 +11798,8 @@ pub fn getOrPutStringFmt(...@@ -11772,8 +11798,8 @@ pub fn getOrPutStringFmt(
11772 // ensure that references to strings in args do not get invalidated11798 // ensure that references to strings in args do not get invalidated
11773 const format_z = format ++ .{0};11799 const format_z = format ++ .{0};
11774 const len: u32 = @intCast(std.fmt.count(format_z, args));11800 const len: u32 = @intCast(std.fmt.count(format_z, args));
11775 const strings = ip.getLocal(tid).getMutableStrings(gpa);11801 const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa);
11776 const slice = try strings.addManyAsSlice(len);11802 const slice = try string_bytes.addManyAsSlice(len);
11777 assert((std.fmt.bufPrint(slice[0], format_z, args) catch unreachable).len == len);11803 assert((std.fmt.bufPrint(slice[0], format_z, args) catch unreachable).len == len);
11778 return ip.getOrPutTrailingString(gpa, tid, len, embedded_nulls);11804 return ip.getOrPutTrailingString(gpa, tid, len, embedded_nulls);
11779}11805}
...@@ -11797,21 +11823,27 @@ pub fn getOrPutTrailingString(...@@ -11797,21 +11823,27 @@ pub fn getOrPutTrailingString(
11797 len: u32,11823 len: u32,
11798 comptime embedded_nulls: EmbeddedNulls,11824 comptime embedded_nulls: EmbeddedNulls,
11799) Allocator.Error!embedded_nulls.StringType() {11825) Allocator.Error!embedded_nulls.StringType() {
11800 const strings = ip.getLocal(tid).getMutableStrings(gpa);11826 const local = ip.getLocal(tid);
11801 const start: u32 = @intCast(strings.mutate.len - len);11827 const strings = local.getMutableStrings(gpa);
11802 if (len > 0 and strings.view().items(.@"0")[strings.mutate.len - 1] == 0) {11828 try strings.ensureUnusedCapacity(1);
11803 strings.mutate.len -= 1;11829 const string_bytes = local.getMutableStringBytes(gpa);
11830 const start: u32 = @intCast(string_bytes.mutate.len - len);
11831 if (len > 0 and string_bytes.view().items(.@"0")[string_bytes.mutate.len - 1] == 0) {
11832 string_bytes.mutate.len -= 1;
11804 } else {11833 } else {
11805 try strings.ensureUnusedCapacity(1);11834 try string_bytes.ensureUnusedCapacity(1);
11806 }11835 }
11807 const key: []const u8 = strings.view().items(.@"0")[start..];11836 const key: []const u8 = string_bytes.view().items(.@"0")[start..];
11808 const value: embedded_nulls.StringType() =11837 const value: embedded_nulls.StringType() = @enumFromInt(@intFromEnum((String.Unwrapped{
11809 @enumFromInt(@intFromEnum((String.Unwrapped{ .tid = tid, .index = start }).wrap(ip)));11838 .tid = tid,
11839 .index = strings.mutate.len - 1,
11840 }).wrap(ip)));
11810 const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;11841 const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;
11811 switch (embedded_nulls) {11842 switch (embedded_nulls) {
11812 .no_embedded_nulls => assert(!has_embedded_null),11843 .no_embedded_nulls => assert(!has_embedded_null),
11813 .maybe_embedded_nulls => if (has_embedded_null) {11844 .maybe_embedded_nulls => if (has_embedded_null) {
11814 strings.appendAssumeCapacity(.{0});11845 string_bytes.appendAssumeCapacity(.{0});
11846 strings.appendAssumeCapacity(.{string_bytes.mutate.len});
11815 return value;11847 return value;
11816 },11848 },
11817 }11849 }
...@@ -11829,7 +11861,7 @@ pub fn getOrPutTrailingString(...@@ -11829,7 +11861,7 @@ pub fn getOrPutTrailingString(
11829 const index = entry.acquire().unwrap() orelse break;11861 const index = entry.acquire().unwrap() orelse break;
11830 if (entry.hash != hash) continue;11862 if (entry.hash != hash) continue;
11831 if (!index.eqlSlice(key, ip)) continue;11863 if (!index.eqlSlice(key, ip)) continue;
11832 strings.shrinkRetainingCapacity(start);11864 string_bytes.shrinkRetainingCapacity(start);
11833 return @enumFromInt(@intFromEnum(index));11865 return @enumFromInt(@intFromEnum(index));
11834 }11866 }
11835 shard.mutate.string_map.mutex.lock();11867 shard.mutate.string_map.mutex.lock();
...@@ -11845,19 +11877,20 @@ pub fn getOrPutTrailingString(...@@ -11845,19 +11877,20 @@ pub fn getOrPutTrailingString(
11845 const index = entry.acquire().unwrap() orelse break;11877 const index = entry.acquire().unwrap() orelse break;
11846 if (entry.hash != hash) continue;11878 if (entry.hash != hash) continue;
11847 if (!index.eqlSlice(key, ip)) continue;11879 if (!index.eqlSlice(key, ip)) continue;
11848 strings.shrinkRetainingCapacity(start);11880 string_bytes.shrinkRetainingCapacity(start);
11849 return @enumFromInt(@intFromEnum(index));11881 return @enumFromInt(@intFromEnum(index));
11850 }11882 }
11851 defer shard.mutate.string_map.len += 1;11883 defer shard.mutate.string_map.len += 1;
11852 const map_header = map.header().*;11884 const map_header = map.header().*;
11853 if (shard.mutate.string_map.len < map_header.capacity * 3 / 5) {11885 if (shard.mutate.string_map.len < map_header.capacity * 3 / 5) {
11854 strings.appendAssumeCapacity(.{0});11886 string_bytes.appendAssumeCapacity(.{0});
11887 strings.appendAssumeCapacity(.{string_bytes.mutate.len});
11855 const entry = &map.entries[map_index];11888 const entry = &map.entries[map_index];
11856 entry.hash = hash;11889 entry.hash = hash;
11857 entry.release(@enumFromInt(@intFromEnum(value)));11890 entry.release(@enumFromInt(@intFromEnum(value)));
11858 return value;11891 return value;
11859 }11892 }
11860 const arena_state = &ip.getLocal(tid).mutate.arena;11893 const arena_state = &local.mutate.arena;
11861 var arena = arena_state.promote(gpa);11894 var arena = arena_state.promote(gpa);
11862 defer arena_state.* = arena.state;11895 defer arena_state.* = arena.state;
11863 const new_map_capacity = map_header.capacity * 2;11896 const new_map_capacity = map_header.capacity * 2;
...@@ -11893,7 +11926,8 @@ pub fn getOrPutTrailingString(...@@ -11893,7 +11926,8 @@ pub fn getOrPutTrailingString(
11893 map_index &= new_map_mask;11926 map_index &= new_map_mask;
11894 if (map.entries[map_index].value == .none) break;11927 if (map.entries[map_index].value == .none) break;
11895 }11928 }
11896 strings.appendAssumeCapacity(.{0});11929 string_bytes.appendAssumeCapacity(.{0});
11930 strings.appendAssumeCapacity(.{string_bytes.mutate.len});
11897 map.entries[map_index] = .{11931 map.entries[map_index] = .{
11898 .value = @enumFromInt(@intFromEnum(value)),11932 .value = @enumFromInt(@intFromEnum(value)),
11899 .hash = hash,11933 .hash = hash,
src/Value.zig+7-7
...@@ -66,8 +66,8 @@ pub fn toIpString(val: Value, ty: Type, pt: Zcu.PerThread) !InternPool.NullTermi...@@ -66,8 +66,8 @@ pub fn toIpString(val: Value, ty: Type, pt: Zcu.PerThread) !InternPool.NullTermi
66 .repeated_elem => |elem| {66 .repeated_elem => |elem| {
67 const byte: u8 = @intCast(Value.fromInterned(elem).toUnsignedInt(zcu));67 const byte: u8 = @intCast(Value.fromInterned(elem).toUnsignedInt(zcu));
68 const len: u32 = @intCast(ty.arrayLen(zcu));68 const len: u32 = @intCast(ty.arrayLen(zcu));
69 const strings = ip.getLocal(pt.tid).getMutableStrings(zcu.gpa);69 const string_bytes = ip.getLocal(pt.tid).getMutableStringBytes(zcu.gpa);
70 try strings.appendNTimes(.{byte}, len);70 try string_bytes.appendNTimes(.{byte}, len);
71 return ip.getOrPutTrailingString(zcu.gpa, pt.tid, len, .no_embedded_nulls);71 return ip.getOrPutTrailingString(zcu.gpa, pt.tid, len, .no_embedded_nulls);
72 },72 },
73 }73 }
...@@ -109,16 +109,16 @@ fn arrayToIpString(val: Value, len_u64: u64, pt: Zcu.PerThread) !InternPool.Null...@@ -109,16 +109,16 @@ fn arrayToIpString(val: Value, len_u64: u64, pt: Zcu.PerThread) !InternPool.Null
109 const gpa = zcu.gpa;109 const gpa = zcu.gpa;
110 const ip = &zcu.intern_pool;110 const ip = &zcu.intern_pool;
111 const len: u32 = @intCast(len_u64);111 const len: u32 = @intCast(len_u64);
112 const strings = ip.getLocal(pt.tid).getMutableStrings(gpa);112 const string_bytes = ip.getLocal(pt.tid).getMutableStringBytes(gpa);
113 try strings.ensureUnusedCapacity(len);113 try string_bytes.ensureUnusedCapacity(len);
114 for (0..len) |i| {114 for (0..len) |i| {
115 // I don't think elemValue has the possibility to affect ip.string_bytes. Let's115 // I don't think elemValue has the possibility to affect ip.string_bytes. Let's
116 // assert just to be sure.116 // assert just to be sure.
117 const prev_len = strings.mutate.len;117 const prev_len = string_bytes.mutate.len;
118 const elem_val = try val.elemValue(pt, i);118 const elem_val = try val.elemValue(pt, i);
119 assert(strings.mutate.len == prev_len);119 assert(string_bytes.mutate.len == prev_len);
120 const byte: u8 = @intCast(elem_val.toUnsignedInt(zcu));120 const byte: u8 = @intCast(elem_val.toUnsignedInt(zcu));
121 strings.appendAssumeCapacity(.{byte});121 string_bytes.appendAssumeCapacity(.{byte});
122 }122 }
123 return ip.getOrPutTrailingString(gpa, pt.tid, len, .no_embedded_nulls);123 return ip.getOrPutTrailingString(gpa, pt.tid, len, .no_embedded_nulls);
124}124}
src/Zcu.zig+2-2
...@@ -1116,8 +1116,8 @@ pub const File = struct {...@@ -1116,8 +1116,8 @@ pub const File = struct {
1116 pub fn internFullyQualifiedName(file: File, pt: Zcu.PerThread) !InternPool.NullTerminatedString {1116 pub fn internFullyQualifiedName(file: File, pt: Zcu.PerThread) !InternPool.NullTerminatedString {
1117 const gpa = pt.zcu.gpa;1117 const gpa = pt.zcu.gpa;
1118 const ip = &pt.zcu.intern_pool;1118 const ip = &pt.zcu.intern_pool;
1119 const strings = ip.getLocal(pt.tid).getMutableStrings(gpa);1119 const string_bytes = ip.getLocal(pt.tid).getMutableStringBytes(gpa);
1120 var w: Writer = .fixed((try strings.addManyAsSlice(file.fullyQualifiedNameLen()))[0]);1120 var w: Writer = .fixed((try string_bytes.addManyAsSlice(file.fullyQualifiedNameLen()))[0]);
1121 file.renderFullyQualifiedName(&w) catch unreachable;1121 file.renderFullyQualifiedName(&w) catch unreachable;
1122 assert(w.end == w.buffer.len);1122 assert(w.end == w.buffer.len);
1123 return ip.getOrPutTrailingString(gpa, pt.tid, @intCast(w.end), .no_embedded_nulls);1123 return ip.getOrPutTrailingString(gpa, pt.tid, @intCast(w.end), .no_embedded_nulls);
src/Zcu/PerThread.zig+4-4
...@@ -2459,10 +2459,10 @@ fn updateEmbedFileInner(...@@ -2459,10 +2459,10 @@ fn updateEmbedFileInner(
24592459
2460 // The loaded bytes of the file, including a sentinel 0 byte.2460 // The loaded bytes of the file, including a sentinel 0 byte.
2461 const ip_str: InternPool.String = str: {2461 const ip_str: InternPool.String = str: {
2462 const strings = ip.getLocal(tid).getMutableStrings(gpa);2462 const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa);
2463 const old_len = strings.mutate.len;2463 const old_len = string_bytes.mutate.len;
2464 errdefer strings.shrinkRetainingCapacity(old_len);2464 errdefer string_bytes.shrinkRetainingCapacity(old_len);
2465 const bytes = (try strings.addManyAsSlice(size_plus_one))[0];2465 const bytes = (try string_bytes.addManyAsSlice(size_plus_one))[0];
2466 var fr = file.reader(&.{});2466 var fr = file.reader(&.{});
2467 fr.size = stat.size;2467 fr.size = stat.size;
2468 fr.interface.readSliceAll(bytes[0..size]) catch |err| switch (err) {2468 fr.interface.readSliceAll(bytes[0..size]) catch |err| switch (err) {