authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-07-15 03:19:15-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-07-16 05:14:38-04:00
logb0fe7eef54dcaab8f7f0c18be042bf1876274ca4
tree2522ef39e4bb572dc474f2c61b8604202485aa69
parente32454796cb601be8558cd6d078854a7dc2d1a14

InternPool: fix various data structure invariants

1 files changed, 92 insertions(+), 41 deletions(-)

src/InternPool.zig+92-41
......@@ -939,8 +939,12 @@ const Shard = struct {
939939 return @atomicLoad(Value, &entry.value, .acquire);
940940 }
941941 fn release(entry: *Entry, value: Value) void {
942 assert(value != .none);
942943 @atomicStore(Value, &entry.value, value, .release);
943944 }
945 fn resetUnordered(entry: *Entry) void {
946 @atomicStore(Value, &entry.value, .none, .unordered);
947 }
944948 };
945949 };
946950 }
......@@ -6583,35 +6587,55 @@ const GetOrPutKey = union(enum) {
65836587 },
65846588
65856589 fn put(gop: *GetOrPutKey) Index {
6586 return gop.putAt(0);
6587 }
6588 fn putAt(gop: *GetOrPutKey, offset: u32) Index {
65896590 switch (gop.*) {
65906591 .existing => unreachable,
6591 .new => |info| {
6592 .new => |*info| {
65926593 const index = Index.Unwrapped.wrap(.{
65936594 .tid = info.tid,
6594 .index = info.ip.getLocal(info.tid).mutate.items.len - 1 - offset,
6595 .index = info.ip.getLocal(info.tid).mutate.items.len - 1,
65956596 }, info.ip);
6596 info.shard.shared.map.entries[info.map_index].release(index);
6597 gop.putTentative(index);
6598 gop.putFinal(index);
6599 return index;
6600 },
6601 }
6602 }
6603
6604 fn putTentative(gop: *GetOrPutKey, index: Index) void {
6605 assert(index != .none);
6606 switch (gop.*) {
6607 .existing => unreachable,
6608 .new => |*info| gop.new.shard.shared.map.entries[info.map_index].release(index),
6609 }
6610 }
6611
6612 fn putFinal(gop: *GetOrPutKey, index: Index) void {
6613 assert(index != .none);
6614 switch (gop.*) {
6615 .existing => unreachable,
6616 .new => |info| {
6617 assert(info.shard.shared.map.entries[info.map_index].value == index);
65976618 info.shard.mutate.map.len += 1;
65986619 info.shard.mutate.map.mutex.unlock();
65996620 gop.* = .{ .existing = index };
6600 return index;
66016621 },
66026622 }
66036623 }
66046624
6605 fn assign(gop: *GetOrPutKey, new_gop: GetOrPutKey) void {
6606 gop.deinit();
6607 gop.* = new_gop;
6625 fn cancel(gop: *GetOrPutKey) void {
6626 switch (gop.*) {
6627 .existing => {},
6628 .new => |info| info.shard.mutate.map.mutex.unlock(),
6629 }
6630 gop.* = .{ .existing = undefined };
66086631 }
66096632
66106633 fn deinit(gop: *GetOrPutKey) void {
66116634 switch (gop.*) {
66126635 .existing => {},
6613 .new => |info| info.shard.mutate.map.mutex.unlock(),
6636 .new => |info| info.shard.shared.map.entries[info.map_index].resetUnordered(),
66146637 }
6638 gop.cancel();
66156639 gop.* = undefined;
66166640 }
66176641};
......@@ -6620,6 +6644,15 @@ fn getOrPutKey(
66206644 gpa: Allocator,
66216645 tid: Zcu.PerThread.Id,
66226646 key: Key,
6647) Allocator.Error!GetOrPutKey {
6648 return ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, key, 0);
6649}
6650fn getOrPutKeyEnsuringAdditionalCapacity(
6651 ip: *InternPool,
6652 gpa: Allocator,
6653 tid: Zcu.PerThread.Id,
6654 key: Key,
6655 additional_capacity: u32,
66236656) Allocator.Error!GetOrPutKey {
66246657 const full_hash = key.hash64(ip);
66256658 const hash: u32 = @truncate(full_hash >> 32);
......@@ -6655,11 +6688,16 @@ fn getOrPutKey(
66556688 }
66566689 }
66576690 const map_header = map.header().*;
6658 if (shard.mutate.map.len >= map_header.capacity * 3 / 5) {
6691 const required = shard.mutate.map.len + additional_capacity;
6692 if (required >= map_header.capacity * 3 / 5) {
66596693 const arena_state = &ip.getLocal(tid).mutate.arena;
66606694 var arena = arena_state.promote(gpa);
66616695 defer arena_state.* = arena.state;
6662 const new_map_capacity = map_header.capacity * 2;
6696 var new_map_capacity = map_header.capacity;
6697 while (true) {
6698 new_map_capacity *= 2;
6699 if (required < new_map_capacity * 3 / 5) break;
6700 }
66636701 const new_map_buf = try arena.allocator().alignedAlloc(
66646702 u8,
66656703 Map.alignment,
......@@ -6728,10 +6766,11 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
67286766 assert(ptr_type.sentinel == .none or ip.typeOf(ptr_type.sentinel) == ptr_type.child);
67296767
67306768 if (ptr_type.flags.size == .Slice) {
6769 gop.cancel();
67316770 var new_key = key;
67326771 new_key.ptr_type.flags.size = .Many;
67336772 const ptr_type_index = try ip.get(gpa, tid, new_key);
6734 gop.assign(try ip.getOrPutKey(gpa, tid, key));
6773 gop = try ip.getOrPutKey(gpa, tid, key);
67356774
67366775 try items.ensureUnusedCapacity(1);
67376776 items.appendAssumeCapacity(.{
......@@ -6911,9 +6950,10 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
69116950 },
69126951 .anon_decl => |anon_decl| if (ptrsHaveSameAlignment(ip, ptr.ty, ptr_type, anon_decl.orig_ty)) item: {
69136952 if (ptr.ty != anon_decl.orig_ty) {
6953 gop.cancel();
69146954 var new_key = key;
69156955 new_key.ptr.base_addr.anon_decl.orig_ty = ptr.ty;
6916 gop.assign(try ip.getOrPutKey(gpa, tid, new_key));
6956 gop = try ip.getOrPutKey(gpa, tid, new_key);
69176957 if (gop == .existing) return gop.existing;
69186958 }
69196959 break :item .{
......@@ -6984,11 +7024,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
69847024 },
69857025 else => unreachable,
69867026 }
7027 gop.cancel();
69877028 const index_index = try ip.get(gpa, tid, .{ .int = .{
69887029 .ty = .usize_type,
69897030 .storage = .{ .u64 = base_index.index },
69907031 } });
6991 gop.assign(try ip.getOrPutKey(gpa, tid, key));
7032 gop = try ip.getOrPutKey(gpa, tid, key);
69927033 try items.ensureUnusedCapacity(1);
69937034 items.appendAssumeCapacity(.{
69947035 .tag = switch (ptr.base_addr) {
......@@ -7397,11 +7438,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
73977438 }
73987439 const elem = switch (aggregate.storage) {
73997440 .bytes => |bytes| elem: {
7441 gop.cancel();
74007442 const elem = try ip.get(gpa, tid, .{ .int = .{
74017443 .ty = .u8_type,
74027444 .storage = .{ .u64 = bytes.at(0, ip) },
74037445 } });
7404 gop.assign(try ip.getOrPutKey(gpa, tid, key));
7446 gop = try ip.getOrPutKey(gpa, tid, key);
74057447 try items.ensureUnusedCapacity(1);
74067448 break :elem elem;
74077449 },
......@@ -8219,9 +8261,9 @@ pub fn getFuncDeclIes(
82198261 extra.mutate.len = prev_extra_len;
82208262 }
82218263
8222 var func_gop = try ip.getOrPutKey(gpa, tid, .{
8264 var func_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
82238265 .func = extraFuncDecl(tid, extra.list.*, func_decl_extra_index),
8224 });
8266 }, 3);
82258267 defer func_gop.deinit();
82268268 if (func_gop == .existing) {
82278269 // An existing function type was found; undo the additions to our two arrays.
......@@ -8229,23 +8271,28 @@ pub fn getFuncDeclIes(
82298271 extra.mutate.len = prev_extra_len;
82308272 return func_gop.existing;
82318273 }
8232 var error_union_type_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{
8274 func_gop.putTentative(func_index);
8275 var error_union_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{ .error_union_type = .{
82338276 .error_set_type = error_set_type,
82348277 .payload_type = key.bare_return_type,
8235 } });
8278 } }, 2);
82368279 defer error_union_type_gop.deinit();
8237 var error_set_type_gop = try ip.getOrPutKey(gpa, tid, .{
8280 error_union_type_gop.putTentative(error_union_type);
8281 var error_set_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
82388282 .inferred_error_set_type = func_index,
8239 });
8283 }, 1);
82408284 defer error_set_type_gop.deinit();
8285 error_set_type_gop.putTentative(error_set_type);
82418286 var func_ty_gop = try ip.getOrPutKey(gpa, tid, .{
82428287 .func_type = extraFuncType(tid, extra.list.*, func_type_extra_index),
82438288 });
82448289 defer func_ty_gop.deinit();
8245 assert(func_gop.putAt(3) == func_index);
8246 assert(error_union_type_gop.putAt(2) == error_union_type);
8247 assert(error_set_type_gop.putAt(1) == error_set_type);
8248 assert(func_ty_gop.putAt(0) == func_ty);
8290 func_ty_gop.putTentative(func_ty);
8291
8292 func_gop.putFinal(func_index);
8293 error_union_type_gop.putFinal(error_union_type);
8294 error_set_type_gop.putFinal(error_set_type);
8295 func_ty_gop.putFinal(func_ty);
82498296 return func_index;
82508297}
82518298
......@@ -8504,9 +8551,9 @@ pub fn getFuncInstanceIes(
85048551 extra.mutate.len = prev_extra_len;
85058552 }
85068553
8507 var func_gop = try ip.getOrPutKey(gpa, tid, .{
8554 var func_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
85088555 .func = ip.extraFuncInstance(tid, extra.list.*, func_extra_index),
8509 });
8556 }, 3);
85108557 defer func_gop.deinit();
85118558 if (func_gop == .existing) {
85128559 // Hot path: undo the additions to our two arrays.
......@@ -8514,19 +8561,23 @@ pub fn getFuncInstanceIes(
85148561 extra.mutate.len = prev_extra_len;
85158562 return func_gop.existing;
85168563 }
8517 var error_union_type_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{
8564 func_gop.putTentative(func_index);
8565 var error_union_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{ .error_union_type = .{
85188566 .error_set_type = error_set_type,
85198567 .payload_type = arg.bare_return_type,
8520 } });
8568 } }, 2);
85218569 defer error_union_type_gop.deinit();
8522 var error_set_type_gop = try ip.getOrPutKey(gpa, tid, .{
8570 error_union_type_gop.putTentative(error_union_type);
8571 var error_set_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
85238572 .inferred_error_set_type = func_index,
8524 });
8573 }, 1);
85258574 defer error_set_type_gop.deinit();
8575 error_set_type_gop.putTentative(error_set_type);
85268576 var func_ty_gop = try ip.getOrPutKey(gpa, tid, .{
85278577 .func_type = extraFuncType(tid, extra.list.*, func_type_extra_index),
85288578 });
85298579 defer func_ty_gop.deinit();
8580 func_ty_gop.putTentative(func_ty);
85308581 try finishFuncInstance(
85318582 ip,
85328583 gpa,
......@@ -8538,10 +8589,11 @@ pub fn getFuncInstanceIes(
85388589 arg.alignment,
85398590 arg.section,
85408591 );
8541 assert(func_gop.putAt(3) == func_index);
8542 assert(error_union_type_gop.putAt(2) == error_union_type);
8543 assert(error_set_type_gop.putAt(1) == error_set_type);
8544 assert(func_ty_gop.putAt(0) == func_ty);
8592
8593 func_gop.putFinal(func_index);
8594 error_union_type_gop.putFinal(error_union_type);
8595 error_set_type_gop.putFinal(error_set_type);
8596 func_ty_gop.putFinal(func_ty);
85458597 return func_index;
85468598}
85478599
......@@ -10837,19 +10889,18 @@ pub fn getBackingDecl(ip: *const InternPool, val: Index) OptionalDeclIndex {
1083710889 while (true) {
1083810890 const unwrapped_base = base.unwrap(ip);
1083910891 const base_item = unwrapped_base.getItem(ip);
10840 const base_extra_items = unwrapped_base.getExtra(ip).view().items(.@"0");
1084110892 switch (base_item.tag) {
10842 .ptr_decl => return @enumFromInt(base_extra_items[
10893 .ptr_decl => return @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
1084310894 base_item.data + std.meta.fieldIndex(PtrDecl, "decl").?
1084410895 ]),
1084510896 inline .ptr_eu_payload,
1084610897 .ptr_opt_payload,
1084710898 .ptr_elem,
1084810899 .ptr_field,
10849 => |tag| base = @enumFromInt(base_extra_items[
10900 => |tag| base = @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
1085010901 base_item.data + std.meta.fieldIndex(tag.Payload(), "base").?
1085110902 ]),
10852 .ptr_slice => base = @enumFromInt(base_extra_items[
10903 .ptr_slice => base = @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
1085310904 base_item.data + std.meta.fieldIndex(PtrSlice, "ptr").?
1085410905 ]),
1085510906 else => return .none,