authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-07-10 15:37:04-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-07-10 19:02:55-04:00
log2c89f3b65427e8ca733a9977ded4e8c6f0a89650
tree31649a654f955efd0dc8b5e8c25622fb3f257c45
parentafa66fa392f5a32d16da7f4705c59dad369f6d48

InternPool: make `maps` thread-safe

1 files changed, 115 insertions(+), 55 deletions(-)

src/InternPool.zig+115-55
......@@ -13,13 +13,6 @@ tid_shift_31: if (single_threaded) u0 else std.math.Log2Int(u32) = if (single_th
1313/// Cached shift amount to put a `tid` in the top bits of a 32-bit value.
1414tid_shift_32: if (single_threaded) u0 else std.math.Log2Int(u32) = if (single_threaded) 0 else 31,
1515
16/// Some types such as enums, structs, and unions need to store mappings from field names
17/// to field index, or value to field index. In such cases, they will store the underlying
18/// field names and values directly, relying on one of these maps, stored separately,
19/// to provide lookup.
20/// These are not serialized; it is computed upon deserialization.
21maps: std.ArrayListUnmanaged(FieldMap) = .{},
22
2316/// Dependencies on the source code hash associated with a ZIR instruction.
2417/// * For a `declaration`, this is the entire declaration body.
2518/// * For a `struct_decl`, `union_decl`, etc, this is the source of the fields (but not declarations).
......@@ -428,6 +421,7 @@ const Local = struct {
428421 strings: ListMutate,
429422 tracked_insts: MutexListMutate,
430423 files: ListMutate,
424 maps: ListMutate,
431425
432426 decls: BucketListMutate,
433427 namespaces: BucketListMutate,
......@@ -440,6 +434,7 @@ const Local = struct {
440434 strings: Strings,
441435 tracked_insts: TrackedInsts,
442436 files: List(File),
437 maps: Maps,
443438
444439 decls: Decls,
445440 namespaces: Namespaces,
......@@ -461,6 +456,7 @@ const Local = struct {
461456 };
462457 const Strings = List(struct { u8 });
463458 const TrackedInsts = List(struct { TrackedInst });
459 const Maps = List(struct { FieldMap });
464460
465461 const decls_bucket_width = 8;
466462 const decls_bucket_mask = (1 << decls_bucket_width) - 1;
......@@ -536,14 +532,17 @@ const Local = struct {
536532 .is_tuple = elem_info.is_tuple,
537533 } });
538534 }
539 fn SliceElem(comptime opts: struct { is_const: bool = false }) type {
535 fn PtrElem(comptime opts: struct {
536 size: std.builtin.Type.Pointer.Size,
537 is_const: bool = false,
538 }) type {
540539 const elem_info = @typeInfo(Elem).Struct;
541540 const elem_fields = elem_info.fields;
542541 var new_fields: [elem_fields.len]std.builtin.Type.StructField = undefined;
543542 for (&new_fields, elem_fields) |*new_field, elem_field| new_field.* = .{
544543 .name = elem_field.name,
545544 .type = @Type(.{ .Pointer = .{
546 .size = .Slice,
545 .size = opts.size,
547546 .is_const = opts.is_const,
548547 .is_volatile = false,
549548 .alignment = 0,
......@@ -564,6 +563,23 @@ const Local = struct {
564563 } });
565564 }
566565
566 pub fn addOne(mutable: Mutable) Allocator.Error!PtrElem(.{ .size = .One }) {
567 try mutable.ensureUnusedCapacity(1);
568 return mutable.addOneAssumeCapacity();
569 }
570
571 pub fn addOneAssumeCapacity(mutable: Mutable) PtrElem(.{ .size = .One }) {
572 const index = mutable.mutate.len;
573 assert(index < mutable.list.header().capacity);
574 mutable.mutate.len = index + 1;
575 const mutable_view = mutable.view().slice();
576 var ptr: PtrElem(.{ .size = .One }) = undefined;
577 inline for (fields) |field| {
578 @field(ptr, @tagName(field)) = &mutable_view.items(field)[index];
579 }
580 return ptr;
581 }
582
567583 pub fn append(mutable: Mutable, elem: Elem) Allocator.Error!void {
568584 try mutable.ensureUnusedCapacity(1);
569585 mutable.appendAssumeCapacity(elem);
......@@ -577,14 +593,14 @@ const Local = struct {
577593
578594 pub fn appendSliceAssumeCapacity(
579595 mutable: Mutable,
580 slice: SliceElem(.{ .is_const = true }),
596 slice: PtrElem(.{ .size = .Slice, .is_const = true }),
581597 ) void {
582598 if (fields.len == 0) return;
583599 const start = mutable.mutate.len;
584600 const slice_len = @field(slice, @tagName(fields[0])).len;
585601 assert(slice_len <= mutable.list.header().capacity - start);
586602 mutable.mutate.len = @intCast(start + slice_len);
587 const mutable_view = mutable.view();
603 const mutable_view = mutable.view().slice();
588604 inline for (fields) |field| {
589605 const field_slice = @field(slice, @tagName(field));
590606 assert(field_slice.len == slice_len);
......@@ -601,7 +617,7 @@ const Local = struct {
601617 const start = mutable.mutate.len;
602618 assert(len <= mutable.list.header().capacity - start);
603619 mutable.mutate.len = @intCast(start + len);
604 const mutable_view = mutable.view();
620 const mutable_view = mutable.view().slice();
605621 inline for (fields) |field| {
606622 @memset(mutable_view.items(field)[start..][0..len], @field(elem, @tagName(field)));
607623 }
......@@ -616,7 +632,7 @@ const Local = struct {
616632 const start = mutable.mutate.len;
617633 assert(len <= mutable.list.header().capacity - start);
618634 mutable.mutate.len = @intCast(start + len);
619 const mutable_view = mutable.view();
635 const mutable_view = mutable.view().slice();
620636 var ptr_array: PtrArrayElem(len) = undefined;
621637 inline for (fields) |field| {
622638 @field(ptr_array, @tagName(field)) = mutable_view.items(field)[start..][0..len];
......@@ -624,17 +640,17 @@ const Local = struct {
624640 return ptr_array;
625641 }
626642
627 pub fn addManyAsSlice(mutable: Mutable, len: usize) Allocator.Error!SliceElem(.{}) {
643 pub fn addManyAsSlice(mutable: Mutable, len: usize) Allocator.Error!PtrElem(.{ .size = .Slice }) {
628644 try mutable.ensureUnusedCapacity(len);
629645 return mutable.addManyAsSliceAssumeCapacity(len);
630646 }
631647
632 pub fn addManyAsSliceAssumeCapacity(mutable: Mutable, len: usize) SliceElem(.{}) {
648 pub fn addManyAsSliceAssumeCapacity(mutable: Mutable, len: usize) PtrElem(.{ .size = .Slice }) {
633649 const start = mutable.mutate.len;
634650 assert(len <= mutable.list.header().capacity - start);
635651 mutable.mutate.len = @intCast(start + len);
636 const mutable_view = mutable.view();
637 var slice: SliceElem(.{}) = undefined;
652 const mutable_view = mutable.view().slice();
653 var slice: PtrElem(.{ .size = .Slice }) = undefined;
638654 inline for (fields) |field| {
639655 @field(slice, @tagName(field)) = mutable_view.items(field)[start..][0..len];
640656 }
......@@ -807,6 +823,20 @@ const Local = struct {
807823 };
808824 }
809825
826 /// Some types such as enums, structs, and unions need to store mappings from field names
827 /// to field index, or value to field index. In such cases, they will store the underlying
828 /// field names and values directly, relying on one of these maps, stored separately,
829 /// to provide lookup.
830 /// These are not serialized; it is computed upon deserialization.
831 pub fn getMutableMaps(local: *Local, gpa: Allocator) Maps.Mutable {
832 return .{
833 .gpa = gpa,
834 .arena = &local.mutate.arena,
835 .mutate = &local.mutate.maps,
836 .list = &local.shared.maps,
837 };
838 }
839
810840 /// Rather than allocating Decl objects with an Allocator, we instead allocate
811841 /// them with this BucketList. This provides four advantages:
812842 /// * Stable memory so that one thread can access a Decl object while another
......@@ -961,9 +991,37 @@ pub const OptionalMapIndex = enum(u32) {
961991pub const MapIndex = enum(u32) {
962992 _,
963993
994 pub fn get(map_index: MapIndex, ip: *InternPool) *FieldMap {
995 const unwrapped_map_index = map_index.unwrap(ip);
996 const maps = ip.getLocalShared(unwrapped_map_index.tid).maps.acquire();
997 return &maps.view().items(.@"0")[unwrapped_map_index.index];
998 }
999
1000 pub fn getConst(map_index: MapIndex, ip: *const InternPool) FieldMap {
1001 return map_index.get(@constCast(ip)).*;
1002 }
1003
9641004 pub fn toOptional(i: MapIndex) OptionalMapIndex {
9651005 return @enumFromInt(@intFromEnum(i));
9661006 }
1007
1008 const Unwrapped = struct {
1009 tid: Zcu.PerThread.Id,
1010 index: u32,
1011
1012 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) MapIndex {
1013 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1014 assert(unwrapped.index <= ip.getIndexMask(u32));
1015 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |
1016 unwrapped.index);
1017 }
1018 };
1019 fn unwrap(map_index: MapIndex, ip: *const InternPool) Unwrapped {
1020 return .{
1021 .tid = @enumFromInt(@intFromEnum(map_index) >> ip.tid_shift_32 & ip.getTidMask()),
1022 .index = @intFromEnum(map_index) & ip.getIndexMask(u32),
1023 };
1024 }
9671025};
9681026
9691027pub const RuntimeIndex = enum(u32) {
......@@ -1398,7 +1456,7 @@ pub const Key = union(enum) {
13981456
13991457 /// Look up field index based on field name.
14001458 pub fn nameIndex(self: ErrorSetType, ip: *const InternPool, name: NullTerminatedString) ?u32 {
1401 const map = &ip.maps.items[@intFromEnum(self.names_map.unwrap().?)];
1459 const map = self.names_map.unwrap().?.getConst(ip);
14021460 const adapter: NullTerminatedString.Adapter = .{ .strings = self.names.get(ip) };
14031461 const field_index = map.getIndexAdapted(name, adapter) orelse return null;
14041462 return @intCast(field_index);
......@@ -2823,7 +2881,7 @@ pub const LoadedStructType = struct {
28232881 if (i >= self.field_types.len) return null;
28242882 return i;
28252883 };
2826 const map = &ip.maps.items[@intFromEnum(names_map)];
2884 const map = names_map.getConst(ip);
28272885 const adapter: NullTerminatedString.Adapter = .{ .strings = self.field_names.get(ip) };
28282886 const field_index = map.getIndexAdapted(name, adapter) orelse return null;
28292887 return @intCast(field_index);
......@@ -3350,7 +3408,7 @@ const LoadedEnumType = struct {
33503408
33513409 /// Look up field index based on field name.
33523410 pub fn nameIndex(self: LoadedEnumType, ip: *const InternPool, name: NullTerminatedString) ?u32 {
3353 const map = &ip.maps.items[@intFromEnum(self.names_map)];
3411 const map = self.names_map.getConst(ip);
33543412 const adapter: NullTerminatedString.Adapter = .{ .strings = self.names.get(ip) };
33553413 const field_index = map.getIndexAdapted(name, adapter) orelse return null;
33563414 return @intCast(field_index);
......@@ -3370,7 +3428,7 @@ const LoadedEnumType = struct {
33703428 else => unreachable,
33713429 };
33723430 if (self.values_map.unwrap()) |values_map| {
3373 const map = &ip.maps.items[@intFromEnum(values_map)];
3431 const map = values_map.getConst(ip);
33743432 const adapter: Index.Adapter = .{ .indexes = self.values.get(ip) };
33753433 const field_index = map.getIndexAdapted(int_tag_val, adapter) orelse return null;
33763434 return @intCast(field_index);
......@@ -5370,6 +5428,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
53705428 .strings = Local.Strings.empty,
53715429 .tracked_insts = Local.TrackedInsts.empty,
53725430 .files = Local.List(File).empty,
5431 .maps = Local.Maps.empty,
53735432
53745433 .decls = Local.Decls.empty,
53755434 .namespaces = Local.Namespaces.empty,
......@@ -5383,6 +5442,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
53835442 .strings = Local.ListMutate.empty,
53845443 .tracked_insts = Local.MutexListMutate.empty,
53855444 .files = Local.ListMutate.empty,
5445 .maps = Local.ListMutate.empty,
53865446
53875447 .decls = Local.BucketListMutate.empty,
53885448 .namespaces = Local.BucketListMutate.empty,
......@@ -5440,9 +5500,6 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
54405500}
54415501
54425502pub fn deinit(ip: *InternPool, gpa: Allocator) void {
5443 for (ip.maps.items) |*map| map.deinit(gpa);
5444 ip.maps.deinit(gpa);
5445
54465503 ip.src_hash_deps.deinit(gpa);
54475504 ip.decl_val_deps.deinit(gpa);
54485505 ip.func_ies_deps.deinit(gpa);
......@@ -5470,6 +5527,8 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {
54705527 namespace.usingnamespace_set.deinit(gpa);
54715528 }
54725529 };
5530 const maps = local.getMutableMaps(gpa);
5531 if (maps.mutate.len > 0) for (maps.view().items(.@"0")) |*map| map.deinit(gpa);
54735532 local.mutate.arena.promote(gpa).deinit();
54745533 }
54755534 gpa.free(ip.locals);
......@@ -6386,8 +6445,8 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
63866445 assert(error_set_type.names_map == .none);
63876446 assert(std.sort.isSorted(NullTerminatedString, error_set_type.names.get(ip), {}, NullTerminatedString.indexLessThan));
63886447 const names = error_set_type.names.get(ip);
6389 const names_map = try ip.addMap(gpa, names.len);
6390 addStringsToMap(ip, names_map, names);
6448 const names_map = try ip.addMap(gpa, tid, names.len);
6449 ip.addStringsToMap(names_map, names);
63916450 const names_len = error_set_type.names.len;
63926451 try extra.ensureUnusedCapacity(@typeInfo(Tag.ErrorSet).Struct.fields.len + names_len);
63936452 items.appendAssumeCapacity(.{
......@@ -7287,8 +7346,8 @@ pub fn getStructType(
72877346 const items = local.getMutableItems(gpa);
72887347 const extra = local.getMutableExtra(gpa);
72897348
7290 const names_map = try ip.addMap(gpa, ini.fields_len);
7291 errdefer _ = ip.maps.pop();
7349 const names_map = try ip.addMap(gpa, tid, ini.fields_len);
7350 errdefer local.mutate.maps.len -= 1;
72927351
72937352 const zir_index = switch (ini.key) {
72947353 inline else => |x| x.zir_index,
......@@ -7835,17 +7894,18 @@ pub fn getErrorSetType(
78357894 const extra = local.getMutableExtra(gpa);
78367895 try extra.ensureUnusedCapacity(@typeInfo(Tag.ErrorSet).Struct.fields.len + names.len);
78377896
7897 const names_map = try ip.addMap(gpa, tid, names.len);
7898 errdefer local.mutate.maps.len -= 1;
7899
78387900 // The strategy here is to add the type unconditionally, then to ask if it
78397901 // already exists, and if so, revert the lengths of the mutated arrays.
78407902 // This is similar to what `getOrPutTrailingString` does.
78417903 const prev_extra_len = extra.mutate.len;
78427904 errdefer extra.mutate.len = prev_extra_len;
78437905
7844 const predicted_names_map: MapIndex = @enumFromInt(ip.maps.items.len);
7845
78467906 const error_set_extra_index = addExtraAssumeCapacity(extra, Tag.ErrorSet{
78477907 .names_len = @intCast(names.len),
7848 .names_map = predicted_names_map,
7908 .names_map = names_map,
78497909 });
78507910 extra.appendSliceAssumeCapacity(.{@ptrCast(names)});
78517911 errdefer extra.mutate.len = prev_extra_len;
......@@ -7865,11 +7925,7 @@ pub fn getErrorSetType(
78657925 });
78667926 errdefer items.mutate.len -= 1;
78677927
7868 const names_map = try ip.addMap(gpa, names.len);
7869 assert(names_map == predicted_names_map);
7870 errdefer _ = ip.maps.pop();
7871
7872 addStringsToMap(ip, names_map, names);
7928 ip.addStringsToMap(names_map, names);
78737929
78747930 return gop.put();
78757931}
......@@ -8235,7 +8291,7 @@ pub const WipEnumType = struct {
82358291 return null;
82368292 }
82378293 assert(ip.typeOf(value) == @as(Index, @enumFromInt(extra_items[wip.tag_ty_index])));
8238 const map = &ip.maps.items[@intFromEnum(wip.values_map.unwrap().?)];
8294 const map = wip.values_map.unwrap().?.get(ip);
82398295 const field_index = map.count();
82408296 const indexes = extra_items[wip.values_start..][0..field_index];
82418297 const adapter: Index.Adapter = .{ .indexes = @ptrCast(indexes) };
......@@ -8281,8 +8337,8 @@ pub fn getEnumType(
82818337 try items.ensureUnusedCapacity(1);
82828338 const extra = local.getMutableExtra(gpa);
82838339
8284 const names_map = try ip.addMap(gpa, ini.fields_len);
8285 errdefer _ = ip.maps.pop();
8340 const names_map = try ip.addMap(gpa, tid, ini.fields_len);
8341 errdefer local.mutate.maps.len -= 1;
82868342
82878343 switch (ini.tag_mode) {
82888344 .auto => {
......@@ -8335,11 +8391,11 @@ pub fn getEnumType(
83358391 },
83368392 .explicit, .nonexhaustive => {
83378393 const values_map: OptionalMapIndex = if (!ini.has_values) .none else m: {
8338 const values_map = try ip.addMap(gpa, ini.fields_len);
8394 const values_map = try ip.addMap(gpa, tid, ini.fields_len);
83398395 break :m values_map.toOptional();
83408396 };
83418397 errdefer if (ini.has_values) {
8342 _ = ip.maps.pop();
8398 local.mutate.maps.len -= 1;
83438399 };
83448400
83458401 try extra.ensureUnusedCapacity(@typeInfo(EnumExplicit).Struct.fields.len +
......@@ -8428,8 +8484,8 @@ pub fn getGeneratedTagEnumType(
84288484 try items.ensureUnusedCapacity(1);
84298485 const extra = local.getMutableExtra(gpa);
84308486
8431 const names_map = try ip.addMap(gpa, ini.names.len);
8432 errdefer _ = ip.maps.pop();
8487 const names_map = try ip.addMap(gpa, tid, ini.names.len);
8488 errdefer local.mutate.maps.len -= 1;
84338489 ip.addStringsToMap(names_map, ini.names);
84348490
84358491 const fields_len: u32 = @intCast(ini.names.len);
......@@ -8462,8 +8518,8 @@ pub fn getGeneratedTagEnumType(
84628518 ini.values.len); // field values
84638519
84648520 const values_map: OptionalMapIndex = if (ini.values.len != 0) m: {
8465 const map = try ip.addMap(gpa, ini.values.len);
8466 addIndexesToMap(ip, map, ini.values);
8521 const map = try ip.addMap(gpa, tid, ini.values.len);
8522 ip.addIndexesToMap(map, ini.values);
84678523 break :m map.toOptional();
84688524 } else .none;
84698525 // We don't clean up the values map on error!
......@@ -8494,7 +8550,9 @@ pub fn getGeneratedTagEnumType(
84948550 errdefer extra.mutate.len = prev_extra_len;
84958551 errdefer switch (ini.tag_mode) {
84968552 .auto => {},
8497 .explicit, .nonexhaustive => _ = if (ini.values.len != 0) ip.maps.pop(),
8553 .explicit, .nonexhaustive => if (ini.values.len != 0) {
8554 local.mutate.maps.len -= 1;
8555 },
84988556 };
84998557
85008558 var gop = try ip.getOrPutKey(gpa, tid, .{ .enum_type = .{
......@@ -8598,7 +8656,7 @@ fn addStringsToMap(
85988656 map_index: MapIndex,
85998657 strings: []const NullTerminatedString,
86008658) void {
8601 const map = &ip.maps.items[@intFromEnum(map_index)];
8659 const map = map_index.get(ip);
86028660 const adapter: NullTerminatedString.Adapter = .{ .strings = strings };
86038661 for (strings) |string| {
86048662 const gop = map.getOrPutAssumeCapacityAdapted(string, adapter);
......@@ -8611,7 +8669,7 @@ fn addIndexesToMap(
86118669 map_index: MapIndex,
86128670 indexes: []const Index,
86138671) void {
8614 const map = &ip.maps.items[@intFromEnum(map_index)];
8672 const map = map_index.get(ip);
86158673 const adapter: Index.Adapter = .{ .indexes = indexes };
86168674 for (indexes) |index| {
86178675 const gop = map.getOrPutAssumeCapacityAdapted(index, adapter);
......@@ -8619,12 +8677,14 @@ fn addIndexesToMap(
86198677 }
86208678}
86218679
8622fn addMap(ip: *InternPool, gpa: Allocator, cap: usize) Allocator.Error!MapIndex {
8623 const ptr = try ip.maps.addOne(gpa);
8624 errdefer _ = ip.maps.pop();
8625 ptr.* = .{};
8626 try ptr.ensureTotalCapacity(gpa, cap);
8627 return @enumFromInt(ip.maps.items.len - 1);
8680fn addMap(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, cap: usize) Allocator.Error!MapIndex {
8681 const maps = ip.getLocal(tid).getMutableMaps(gpa);
8682 const unwrapped: MapIndex.Unwrapped = .{ .tid = tid, .index = maps.mutate.len };
8683 const ptr = try maps.addOne();
8684 errdefer maps.mutate.len = unwrapped.index;
8685 ptr[0].* = .{};
8686 try ptr[0].ensureTotalCapacity(gpa, cap);
8687 return unwrapped.wrap(ip);
86288688}
86298689
86308690/// This operation only happens under compile error conditions.
......@@ -10858,7 +10918,7 @@ pub fn addFieldName(
1085810918 name: NullTerminatedString,
1085910919) ?u32 {
1086010920 const extra_items = extra.view().items(.@"0");
10861 const map = &ip.maps.items[@intFromEnum(names_map)];
10921 const map = names_map.get(ip);
1086210922 const field_index = map.count();
1086310923 const strings = extra_items[names_start..][0..field_index];
1086410924 const adapter: NullTerminatedString.Adapter = .{ .strings = @ptrCast(strings) };