authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-07-15 03:19:15-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-07-16 05:14:38-04:00
logb0fe7eef54dcaab8f7f0c18be042bf1876274ca4
tree2522ef39e4bb572dc474f2c61b8604202485aa69
parente32454796cb601be8558cd6d078854a7dc2d1a14

InternPool: fix various data structure invariants


1 files changed, 92 insertions(+), 41 deletions(-)

src/InternPool.zig+92-41
...@@ -939,8 +939,12 @@ const Shard = struct {...@@ -939,8 +939,12 @@ const Shard = struct {
939 return @atomicLoad(Value, &entry.value, .acquire);939 return @atomicLoad(Value, &entry.value, .acquire);
940 }940 }
941 fn release(entry: *Entry, value: Value) void {941 fn release(entry: *Entry, value: Value) void {
942 assert(value != .none);
942 @atomicStore(Value, &entry.value, value, .release);943 @atomicStore(Value, &entry.value, value, .release);
943 }944 }
945 fn resetUnordered(entry: *Entry) void {
946 @atomicStore(Value, &entry.value, .none, .unordered);
947 }
944 };948 };
945 };949 };
946 }950 }
...@@ -6583,35 +6587,55 @@ const GetOrPutKey = union(enum) {...@@ -6583,35 +6587,55 @@ const GetOrPutKey = union(enum) {
6583 },6587 },
65846588
6585 fn put(gop: *GetOrPutKey) Index {6589 fn put(gop: *GetOrPutKey) Index {
6586 return gop.putAt(0);
6587 }
6588 fn putAt(gop: *GetOrPutKey, offset: u32) Index {
6589 switch (gop.*) {6590 switch (gop.*) {
6590 .existing => unreachable,6591 .existing => unreachable,
6591 .new => |info| {6592 .new => |*info| {
6592 const index = Index.Unwrapped.wrap(.{6593 const index = Index.Unwrapped.wrap(.{
6593 .tid = info.tid,6594 .tid = info.tid,
6594 .index = info.ip.getLocal(info.tid).mutate.items.len - 1 - offset,6595 .index = info.ip.getLocal(info.tid).mutate.items.len - 1,
6595 }, info.ip);6596 }, info.ip);
6596 info.shard.shared.map.entries[info.map_index].release(index);6597 gop.putTentative(index);
6598 gop.putFinal(index);
6599 return index;
6600 },
6601 }
6602 }
6603
6604 fn putTentative(gop: *GetOrPutKey, index: Index) void {
6605 assert(index != .none);
6606 switch (gop.*) {
6607 .existing => unreachable,
6608 .new => |*info| gop.new.shard.shared.map.entries[info.map_index].release(index),
6609 }
6610 }
6611
6612 fn putFinal(gop: *GetOrPutKey, index: Index) void {
6613 assert(index != .none);
6614 switch (gop.*) {
6615 .existing => unreachable,
6616 .new => |info| {
6617 assert(info.shard.shared.map.entries[info.map_index].value == index);
6597 info.shard.mutate.map.len += 1;6618 info.shard.mutate.map.len += 1;
6598 info.shard.mutate.map.mutex.unlock();6619 info.shard.mutate.map.mutex.unlock();
6599 gop.* = .{ .existing = index };6620 gop.* = .{ .existing = index };
6600 return index;
6601 },6621 },
6602 }6622 }
6603 }6623 }
66046624
6605 fn assign(gop: *GetOrPutKey, new_gop: GetOrPutKey) void {6625 fn cancel(gop: *GetOrPutKey) void {
6606 gop.deinit();6626 switch (gop.*) {
6607 gop.* = new_gop;6627 .existing => {},
6628 .new => |info| info.shard.mutate.map.mutex.unlock(),
6629 }
6630 gop.* = .{ .existing = undefined };
6608 }6631 }
66096632
6610 fn deinit(gop: *GetOrPutKey) void {6633 fn deinit(gop: *GetOrPutKey) void {
6611 switch (gop.*) {6634 switch (gop.*) {
6612 .existing => {},6635 .existing => {},
6613 .new => |info| info.shard.mutate.map.mutex.unlock(),6636 .new => |info| info.shard.shared.map.entries[info.map_index].resetUnordered(),
6614 }6637 }
6638 gop.cancel();
6615 gop.* = undefined;6639 gop.* = undefined;
6616 }6640 }
6617};6641};
...@@ -6620,6 +6644,15 @@ fn getOrPutKey(...@@ -6620,6 +6644,15 @@ fn getOrPutKey(
6620 gpa: Allocator,6644 gpa: Allocator,
6621 tid: Zcu.PerThread.Id,6645 tid: Zcu.PerThread.Id,
6622 key: Key,6646 key: Key,
6647) Allocator.Error!GetOrPutKey {
6648 return ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, key, 0);
6649}
6650fn getOrPutKeyEnsuringAdditionalCapacity(
6651 ip: *InternPool,
6652 gpa: Allocator,
6653 tid: Zcu.PerThread.Id,
6654 key: Key,
6655 additional_capacity: u32,
6623) Allocator.Error!GetOrPutKey {6656) Allocator.Error!GetOrPutKey {
6624 const full_hash = key.hash64(ip);6657 const full_hash = key.hash64(ip);
6625 const hash: u32 = @truncate(full_hash >> 32);6658 const hash: u32 = @truncate(full_hash >> 32);
...@@ -6655,11 +6688,16 @@ fn getOrPutKey(...@@ -6655,11 +6688,16 @@ fn getOrPutKey(
6655 }6688 }
6656 }6689 }
6657 const map_header = map.header().*;6690 const map_header = map.header().*;
6658 if (shard.mutate.map.len >= map_header.capacity * 3 / 5) {6691 const required = shard.mutate.map.len + additional_capacity;
6692 if (required >= map_header.capacity * 3 / 5) {
6659 const arena_state = &ip.getLocal(tid).mutate.arena;6693 const arena_state = &ip.getLocal(tid).mutate.arena;
6660 var arena = arena_state.promote(gpa);6694 var arena = arena_state.promote(gpa);
6661 defer arena_state.* = arena.state;6695 defer arena_state.* = arena.state;
6662 const new_map_capacity = map_header.capacity * 2;6696 var new_map_capacity = map_header.capacity;
6697 while (true) {
6698 new_map_capacity *= 2;
6699 if (required < new_map_capacity * 3 / 5) break;
6700 }
6663 const new_map_buf = try arena.allocator().alignedAlloc(6701 const new_map_buf = try arena.allocator().alignedAlloc(
6664 u8,6702 u8,
6665 Map.alignment,6703 Map.alignment,
...@@ -6728,10 +6766,11 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All...@@ -6728,10 +6766,11 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
6728 assert(ptr_type.sentinel == .none or ip.typeOf(ptr_type.sentinel) == ptr_type.child);6766 assert(ptr_type.sentinel == .none or ip.typeOf(ptr_type.sentinel) == ptr_type.child);
67296767
6730 if (ptr_type.flags.size == .Slice) {6768 if (ptr_type.flags.size == .Slice) {
6769 gop.cancel();
6731 var new_key = key;6770 var new_key = key;
6732 new_key.ptr_type.flags.size = .Many;6771 new_key.ptr_type.flags.size = .Many;
6733 const ptr_type_index = try ip.get(gpa, tid, new_key);6772 const ptr_type_index = try ip.get(gpa, tid, new_key);
6734 gop.assign(try ip.getOrPutKey(gpa, tid, key));6773 gop = try ip.getOrPutKey(gpa, tid, key);
67356774
6736 try items.ensureUnusedCapacity(1);6775 try items.ensureUnusedCapacity(1);
6737 items.appendAssumeCapacity(.{6776 items.appendAssumeCapacity(.{
...@@ -6911,9 +6950,10 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All...@@ -6911,9 +6950,10 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
6911 },6950 },
6912 .anon_decl => |anon_decl| if (ptrsHaveSameAlignment(ip, ptr.ty, ptr_type, anon_decl.orig_ty)) item: {6951 .anon_decl => |anon_decl| if (ptrsHaveSameAlignment(ip, ptr.ty, ptr_type, anon_decl.orig_ty)) item: {
6913 if (ptr.ty != anon_decl.orig_ty) {6952 if (ptr.ty != anon_decl.orig_ty) {
6953 gop.cancel();
6914 var new_key = key;6954 var new_key = key;
6915 new_key.ptr.base_addr.anon_decl.orig_ty = ptr.ty;6955 new_key.ptr.base_addr.anon_decl.orig_ty = ptr.ty;
6916 gop.assign(try ip.getOrPutKey(gpa, tid, new_key));6956 gop = try ip.getOrPutKey(gpa, tid, new_key);
6917 if (gop == .existing) return gop.existing;6957 if (gop == .existing) return gop.existing;
6918 }6958 }
6919 break :item .{6959 break :item .{
...@@ -6984,11 +7024,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All...@@ -6984,11 +7024,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
6984 },7024 },
6985 else => unreachable,7025 else => unreachable,
6986 }7026 }
7027 gop.cancel();
6987 const index_index = try ip.get(gpa, tid, .{ .int = .{7028 const index_index = try ip.get(gpa, tid, .{ .int = .{
6988 .ty = .usize_type,7029 .ty = .usize_type,
6989 .storage = .{ .u64 = base_index.index },7030 .storage = .{ .u64 = base_index.index },
6990 } });7031 } });
6991 gop.assign(try ip.getOrPutKey(gpa, tid, key));7032 gop = try ip.getOrPutKey(gpa, tid, key);
6992 try items.ensureUnusedCapacity(1);7033 try items.ensureUnusedCapacity(1);
6993 items.appendAssumeCapacity(.{7034 items.appendAssumeCapacity(.{
6994 .tag = switch (ptr.base_addr) {7035 .tag = switch (ptr.base_addr) {
...@@ -7397,11 +7438,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All...@@ -7397,11 +7438,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
7397 }7438 }
7398 const elem = switch (aggregate.storage) {7439 const elem = switch (aggregate.storage) {
7399 .bytes => |bytes| elem: {7440 .bytes => |bytes| elem: {
7441 gop.cancel();
7400 const elem = try ip.get(gpa, tid, .{ .int = .{7442 const elem = try ip.get(gpa, tid, .{ .int = .{
7401 .ty = .u8_type,7443 .ty = .u8_type,
7402 .storage = .{ .u64 = bytes.at(0, ip) },7444 .storage = .{ .u64 = bytes.at(0, ip) },
7403 } });7445 } });
7404 gop.assign(try ip.getOrPutKey(gpa, tid, key));7446 gop = try ip.getOrPutKey(gpa, tid, key);
7405 try items.ensureUnusedCapacity(1);7447 try items.ensureUnusedCapacity(1);
7406 break :elem elem;7448 break :elem elem;
7407 },7449 },
...@@ -8219,9 +8261,9 @@ pub fn getFuncDeclIes(...@@ -8219,9 +8261,9 @@ pub fn getFuncDeclIes(
8219 extra.mutate.len = prev_extra_len;8261 extra.mutate.len = prev_extra_len;
8220 }8262 }
82218263
8222 var func_gop = try ip.getOrPutKey(gpa, tid, .{8264 var func_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
8223 .func = extraFuncDecl(tid, extra.list.*, func_decl_extra_index),8265 .func = extraFuncDecl(tid, extra.list.*, func_decl_extra_index),
8224 });8266 }, 3);
8225 defer func_gop.deinit();8267 defer func_gop.deinit();
8226 if (func_gop == .existing) {8268 if (func_gop == .existing) {
8227 // An existing function type was found; undo the additions to our two arrays.8269 // An existing function type was found; undo the additions to our two arrays.
...@@ -8229,23 +8271,28 @@ pub fn getFuncDeclIes(...@@ -8229,23 +8271,28 @@ pub fn getFuncDeclIes(
8229 extra.mutate.len = prev_extra_len;8271 extra.mutate.len = prev_extra_len;
8230 return func_gop.existing;8272 return func_gop.existing;
8231 }8273 }
8232 var error_union_type_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{8274 func_gop.putTentative(func_index);
8275 var error_union_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{ .error_union_type = .{
8233 .error_set_type = error_set_type,8276 .error_set_type = error_set_type,
8234 .payload_type = key.bare_return_type,8277 .payload_type = key.bare_return_type,
8235 } });8278 } }, 2);
8236 defer error_union_type_gop.deinit();8279 defer error_union_type_gop.deinit();
8237 var error_set_type_gop = try ip.getOrPutKey(gpa, tid, .{8280 error_union_type_gop.putTentative(error_union_type);
8281 var error_set_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
8238 .inferred_error_set_type = func_index,8282 .inferred_error_set_type = func_index,
8239 });8283 }, 1);
8240 defer error_set_type_gop.deinit();8284 defer error_set_type_gop.deinit();
8285 error_set_type_gop.putTentative(error_set_type);
8241 var func_ty_gop = try ip.getOrPutKey(gpa, tid, .{8286 var func_ty_gop = try ip.getOrPutKey(gpa, tid, .{
8242 .func_type = extraFuncType(tid, extra.list.*, func_type_extra_index),8287 .func_type = extraFuncType(tid, extra.list.*, func_type_extra_index),
8243 });8288 });
8244 defer func_ty_gop.deinit();8289 defer func_ty_gop.deinit();
8245 assert(func_gop.putAt(3) == func_index);8290 func_ty_gop.putTentative(func_ty);
8246 assert(error_union_type_gop.putAt(2) == error_union_type);8291
8247 assert(error_set_type_gop.putAt(1) == error_set_type);8292 func_gop.putFinal(func_index);
8248 assert(func_ty_gop.putAt(0) == func_ty);8293 error_union_type_gop.putFinal(error_union_type);
8294 error_set_type_gop.putFinal(error_set_type);
8295 func_ty_gop.putFinal(func_ty);
8249 return func_index;8296 return func_index;
8250}8297}
82518298
...@@ -8504,9 +8551,9 @@ pub fn getFuncInstanceIes(...@@ -8504,9 +8551,9 @@ pub fn getFuncInstanceIes(
8504 extra.mutate.len = prev_extra_len;8551 extra.mutate.len = prev_extra_len;
8505 }8552 }
85068553
8507 var func_gop = try ip.getOrPutKey(gpa, tid, .{8554 var func_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
8508 .func = ip.extraFuncInstance(tid, extra.list.*, func_extra_index),8555 .func = ip.extraFuncInstance(tid, extra.list.*, func_extra_index),
8509 });8556 }, 3);
8510 defer func_gop.deinit();8557 defer func_gop.deinit();
8511 if (func_gop == .existing) {8558 if (func_gop == .existing) {
8512 // Hot path: undo the additions to our two arrays.8559 // Hot path: undo the additions to our two arrays.
...@@ -8514,19 +8561,23 @@ pub fn getFuncInstanceIes(...@@ -8514,19 +8561,23 @@ pub fn getFuncInstanceIes(
8514 extra.mutate.len = prev_extra_len;8561 extra.mutate.len = prev_extra_len;
8515 return func_gop.existing;8562 return func_gop.existing;
8516 }8563 }
8517 var error_union_type_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{8564 func_gop.putTentative(func_index);
8565 var error_union_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{ .error_union_type = .{
8518 .error_set_type = error_set_type,8566 .error_set_type = error_set_type,
8519 .payload_type = arg.bare_return_type,8567 .payload_type = arg.bare_return_type,
8520 } });8568 } }, 2);
8521 defer error_union_type_gop.deinit();8569 defer error_union_type_gop.deinit();
8522 var error_set_type_gop = try ip.getOrPutKey(gpa, tid, .{8570 error_union_type_gop.putTentative(error_union_type);
8571 var error_set_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
8523 .inferred_error_set_type = func_index,8572 .inferred_error_set_type = func_index,
8524 });8573 }, 1);
8525 defer error_set_type_gop.deinit();8574 defer error_set_type_gop.deinit();
8575 error_set_type_gop.putTentative(error_set_type);
8526 var func_ty_gop = try ip.getOrPutKey(gpa, tid, .{8576 var func_ty_gop = try ip.getOrPutKey(gpa, tid, .{
8527 .func_type = extraFuncType(tid, extra.list.*, func_type_extra_index),8577 .func_type = extraFuncType(tid, extra.list.*, func_type_extra_index),
8528 });8578 });
8529 defer func_ty_gop.deinit();8579 defer func_ty_gop.deinit();
8580 func_ty_gop.putTentative(func_ty);
8530 try finishFuncInstance(8581 try finishFuncInstance(
8531 ip,8582 ip,
8532 gpa,8583 gpa,
...@@ -8538,10 +8589,11 @@ pub fn getFuncInstanceIes(...@@ -8538,10 +8589,11 @@ pub fn getFuncInstanceIes(
8538 arg.alignment,8589 arg.alignment,
8539 arg.section,8590 arg.section,
8540 );8591 );
8541 assert(func_gop.putAt(3) == func_index);8592
8542 assert(error_union_type_gop.putAt(2) == error_union_type);8593 func_gop.putFinal(func_index);
8543 assert(error_set_type_gop.putAt(1) == error_set_type);8594 error_union_type_gop.putFinal(error_union_type);
8544 assert(func_ty_gop.putAt(0) == func_ty);8595 error_set_type_gop.putFinal(error_set_type);
8596 func_ty_gop.putFinal(func_ty);
8545 return func_index;8597 return func_index;
8546}8598}
85478599
...@@ -10837,19 +10889,18 @@ pub fn getBackingDecl(ip: *const InternPool, val: Index) OptionalDeclIndex {...@@ -10837,19 +10889,18 @@ pub fn getBackingDecl(ip: *const InternPool, val: Index) OptionalDeclIndex {
10837 while (true) {10889 while (true) {
10838 const unwrapped_base = base.unwrap(ip);10890 const unwrapped_base = base.unwrap(ip);
10839 const base_item = unwrapped_base.getItem(ip);10891 const base_item = unwrapped_base.getItem(ip);
10840 const base_extra_items = unwrapped_base.getExtra(ip).view().items(.@"0");
10841 switch (base_item.tag) {10892 switch (base_item.tag) {
10842 .ptr_decl => return @enumFromInt(base_extra_items[10893 .ptr_decl => return @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
10843 base_item.data + std.meta.fieldIndex(PtrDecl, "decl").?10894 base_item.data + std.meta.fieldIndex(PtrDecl, "decl").?
10844 ]),10895 ]),
10845 inline .ptr_eu_payload,10896 inline .ptr_eu_payload,
10846 .ptr_opt_payload,10897 .ptr_opt_payload,
10847 .ptr_elem,10898 .ptr_elem,
10848 .ptr_field,10899 .ptr_field,
10849 => |tag| base = @enumFromInt(base_extra_items[10900 => |tag| base = @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
10850 base_item.data + std.meta.fieldIndex(tag.Payload(), "base").?10901 base_item.data + std.meta.fieldIndex(tag.Payload(), "base").?
10851 ]),10902 ]),
10852 .ptr_slice => base = @enumFromInt(base_extra_items[10903 .ptr_slice => base = @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
10853 base_item.data + std.meta.fieldIndex(PtrSlice, "ptr").?10904 base_item.data + std.meta.fieldIndex(PtrSlice, "ptr").?
10854 ]),10905 ]),
10855 else => return .none,10906 else => return .none,