| ... | ... | @@ -2,9 +2,11 @@ |
| 2 | 2 | //! This data structure is self-contained, with the following exceptions: |
| 3 | 3 | //! * Module.Namespace has a pointer to Module.File |
| 4 | 4 | |
| 5 | | local: []Local = &.{}, |
| 6 | | shard_shift: std.math.Log2Int(usize) = 0, |
| 5 | locals: []Local = &.{}, |
| 7 | 6 | shards: []Shard = &.{}, |
| 7 | tid_width: std.math.Log2Int(u32) = 0, |
| 8 | tid_shift_31: std.math.Log2Int(u32) = 31, |
| 9 | tid_shift_32: std.math.Log2Int(u32) = 31, |
| 8 | 10 | |
| 9 | 11 | items: std.MultiArrayList(Item) = .{}, |
| 10 | 12 | extra: std.ArrayListUnmanaged(u32) = .{}, |
| ... | ... | @@ -13,12 +15,6 @@ extra: std.ArrayListUnmanaged(u32) = .{}, |
| 13 | 15 | /// Use the helper methods instead of accessing this directly in order to not |
| 14 | 16 | /// violate the above mechanism. |
| 15 | 17 | limbs: std.ArrayListUnmanaged(u64) = .{}, |
| 16 | | /// In order to store references to strings in fewer bytes, we copy all |
| 17 | | /// string bytes into here. String bytes can be null. It is up to whomever |
| 18 | | /// is referencing the data here whether they want to store both index and length, |
| 19 | | /// thus allowing null bytes, or store only index, and use null-termination. The |
| 20 | | /// `string_bytes` array is agnostic to either usage. |
| 21 | | string_bytes: std.ArrayListUnmanaged(u8) = .{}, |
| 22 | 18 | |
| 23 | 19 | /// Rather than allocating Decl objects with an Allocator, we instead allocate |
| 24 | 20 | /// them with this SegmentedList. This provides four advantages: |
| ... | ... | @@ -345,52 +341,237 @@ pub const DepEntry = extern struct { |
| 345 | 341 | }; |
| 346 | 342 | |
| 347 | 343 | const Local = struct { |
| 348 | | aligned: void align(std.atomic.cache_line) = {}, |
| 344 | shared: Shared align(std.atomic.cache_line), |
| 345 | mutate: struct { |
| 346 | arena: std.heap.ArenaAllocator.State, |
| 347 | strings: Mutate, |
| 348 | } align(std.atomic.cache_line), |
| 349 | 349 | |
| 350 | | /// header: List.Header, |
| 351 | | /// data: [capacity]u32, |
| 352 | | /// tag: [header.capacity]Tag, |
| 353 | | items: List, |
| 350 | const Shared = struct { |
| 351 | strings: Strings, |
| 352 | }; |
| 354 | 353 | |
| 355 | | /// header: List.Header, |
| 356 | | /// extra: [header.capacity]u32, |
| 357 | | extra: List, |
| 354 | const Strings = List(struct { u8 }); |
| 358 | 355 | |
| 359 | | /// header: List.Header, |
| 360 | | /// bytes: [header.capacity]u8, |
| 361 | | strings: List, |
| 356 | const Mutate = struct { |
| 357 | len: u32, |
| 362 | 358 | |
| 363 | | arena: std.heap.ArenaAllocator.State, |
| 359 | const empty: Mutate = .{ |
| 360 | .len = 0, |
| 361 | }; |
| 362 | }; |
| 364 | 363 | |
| 365 | | const List = struct { |
| 366 | | entries: [*]u32, |
| 364 | fn List(comptime Elem: type) type { |
| 365 | assert(@typeInfo(Elem) == .Struct); |
| 366 | return struct { |
| 367 | bytes: [*]align(@alignOf(Elem)) u8, |
| 368 | |
| 369 | const ListSelf = @This(); |
| 370 | const Mutable = struct { |
| 371 | gpa: std.mem.Allocator, |
| 372 | arena: *std.heap.ArenaAllocator.State, |
| 373 | mutate: *Mutate, |
| 374 | list: *ListSelf, |
| 375 | |
| 376 | const fields = std.enums.values(std.meta.FieldEnum(Elem)); |
| 377 | |
| 378 | fn Slice(comptime opts: struct { is_const: bool = false }) type { |
| 379 | const elem_info = @typeInfo(Elem).Struct; |
| 380 | const elem_fields = elem_info.fields; |
| 381 | var new_fields: [elem_fields.len]std.builtin.Type.StructField = undefined; |
| 382 | for (&new_fields, elem_fields) |*new_field, elem_field| new_field.* = .{ |
| 383 | .name = elem_field.name, |
| 384 | .type = @Type(.{ .Pointer = .{ |
| 385 | .size = .Slice, |
| 386 | .is_const = opts.is_const, |
| 387 | .is_volatile = false, |
| 388 | .alignment = 0, |
| 389 | .address_space = .generic, |
| 390 | .child = elem_field.type, |
| 391 | .is_allowzero = false, |
| 392 | .sentinel = null, |
| 393 | } }), |
| 394 | .default_value = null, |
| 395 | .is_comptime = false, |
| 396 | .alignment = 0, |
| 397 | }; |
| 398 | return @Type(.{ .Struct = .{ |
| 399 | .layout = .auto, |
| 400 | .fields = &new_fields, |
| 401 | .decls = &.{}, |
| 402 | .is_tuple = elem_info.is_tuple, |
| 403 | } }); |
| 404 | } |
| 367 | 405 | |
| 368 | | const empty: List = .{ .entries = @constCast(&(extern struct { |
| 369 | | header: Header, |
| 370 | | entries: [0]u32, |
| 371 | | }{ |
| 372 | | .header = .{ .len = 0, .capacity = 0 }, |
| 373 | | .entries = .{}, |
| 374 | | }).entries) }; |
| 406 | pub fn appendAssumeCapacity(mutable: Mutable, elem: Elem) void { |
| 407 | var mutable_view = mutable.view(); |
| 408 | defer mutable.lenPtr().* = @intCast(mutable_view.len); |
| 409 | mutable_view.appendAssumeCapacity(elem); |
| 410 | } |
| 375 | 411 | |
| 376 | | fn acquire(list: *const List) List { |
| 377 | | return .{ .entries = @atomicLoad([*]u32, &list.entries, .acquire) }; |
| 378 | | } |
| 379 | | fn release(list: *List, new_list: List) void { |
| 380 | | @atomicStore([*]u32, &list.entries, new_list.entries, .release); |
| 381 | | } |
| 412 | pub fn appendSliceAssumeCapacity( |
| 413 | mutable: Mutable, |
| 414 | slice: Slice(.{ .is_const = true }), |
| 415 | ) void { |
| 416 | if (fields.len == 0) return; |
| 417 | const mutable_len = mutable.lenPtr(); |
| 418 | const start = mutable_len.*; |
| 419 | const slice_len = @field(slice, @tagName(fields[0])).len; |
| 420 | assert(slice_len < mutable.capacityPtr().* - start); |
| 421 | mutable_len.* = @intCast(start + slice_len); |
| 422 | const mutable_view = mutable.view(); |
| 423 | inline for (fields) |field| { |
| 424 | const field_slice = @field(slice, @tagName(field)); |
| 425 | assert(field_slice.len == slice_len); |
| 426 | @memcpy(mutable_view.items(field)[start..][0..slice_len], field_slice); |
| 427 | } |
| 428 | } |
| 382 | 429 | |
| 383 | | const Header = extern struct { |
| 384 | | len: u32, |
| 385 | | capacity: u32, |
| 430 | pub fn appendNTimes(mutable: Mutable, elem: Elem, len: usize) Allocator.Error!void { |
| 431 | try mutable.ensureUnusedCapacity(len); |
| 432 | mutable.appendNTimesAssumeCapacity(elem, len); |
| 433 | } |
| 434 | |
| 435 | pub fn appendNTimesAssumeCapacity(mutable: Mutable, elem: Elem, len: usize) void { |
| 436 | const mutable_len = mutable.lenPtr(); |
| 437 | const start = mutable_len.*; |
| 438 | assert(len <= mutable.capacityPtr().* - start); |
| 439 | mutable_len.* = @intCast(start + len); |
| 440 | const mutable_view = mutable.view(); |
| 441 | inline for (fields) |field| { |
| 442 | @memset(mutable_view.items(field)[start..][0..len], @field(elem, @tagName(field))); |
| 443 | } |
| 444 | } |
| 445 | |
| 446 | pub fn addManyAsSlice(mutable: Mutable, len: usize) Allocator.Error!Slice(.{}) { |
| 447 | try mutable.ensureUnusedCapacity(len); |
| 448 | return mutable.addManyAsSliceAssumeCapacity(len); |
| 449 | } |
| 450 | |
| 451 | pub fn addManyAsSliceAssumeCapacity(mutable: Mutable, len: usize) Slice(.{}) { |
| 452 | const mutable_len = mutable.lenPtr(); |
| 453 | const start = mutable_len.*; |
| 454 | assert(len <= mutable.capacityPtr().* - start); |
| 455 | mutable_len.* = @intCast(start + len); |
| 456 | const mutable_view = mutable.view(); |
| 457 | var slice: Slice(.{}) = undefined; |
| 458 | inline for (fields) |field| { |
| 459 | @field(slice, @tagName(field)) = mutable_view.items(field)[start..][0..len]; |
| 460 | } |
| 461 | return slice; |
| 462 | } |
| 463 | |
| 464 | pub fn shrinkRetainingCapacity(mutable: Mutable, len: usize) void { |
| 465 | const mutable_len = mutable.lenPtr(); |
| 466 | assert(len <= mutable_len.*); |
| 467 | mutable_len.* = @intCast(len); |
| 468 | } |
| 469 | |
| 470 | pub fn ensureUnusedCapacity(mutable: Mutable, unused_capacity: usize) Allocator.Error!void { |
| 471 | try mutable.ensureTotalCapacity(@intCast(mutable.lenPtr().* + unused_capacity)); |
| 472 | } |
| 473 | |
| 474 | pub fn ensureTotalCapacity(mutable: Mutable, total_capacity: usize) Allocator.Error!void { |
| 475 | const old_capacity = mutable.capacityPtr().*; |
| 476 | if (old_capacity >= total_capacity) return; |
| 477 | var new_capacity = old_capacity; |
| 478 | while (new_capacity < total_capacity) new_capacity = (new_capacity + 10) * 2; |
| 479 | try mutable.setCapacity(new_capacity); |
| 480 | } |
| 481 | |
| 482 | fn setCapacity(mutable: Mutable, capacity: u32) Allocator.Error!void { |
| 483 | var arena = mutable.arena.promote(mutable.gpa); |
| 484 | defer mutable.arena.* = arena.state; |
| 485 | const buf = try arena.allocator().alignedAlloc( |
| 486 | u8, |
| 487 | alignment, |
| 488 | bytes_offset + View.capacityInBytes(capacity), |
| 489 | ); |
| 490 | var new_list: ListSelf = .{ .bytes = @ptrCast(buf[bytes_offset..].ptr) }; |
| 491 | new_list.header().* = .{ .capacity = capacity }; |
| 492 | const len = mutable.lenPtr().*; |
| 493 | const old_slice = mutable.list.view().slice(); |
| 494 | const new_slice = new_list.view().slice(); |
| 495 | inline for (fields) |field| { |
| 496 | @memcpy(new_slice.items(field)[0..len], old_slice.items(field)[0..len]); |
| 497 | } |
| 498 | mutable.list.release(new_list); |
| 499 | } |
| 500 | |
| 501 | fn view(mutable: Mutable) View { |
| 502 | return .{ |
| 503 | .bytes = mutable.list.bytes, |
| 504 | .len = mutable.lenPtr().*, |
| 505 | .capacity = mutable.capacityPtr().*, |
| 506 | }; |
| 507 | } |
| 508 | |
| 509 | pub fn lenPtr(mutable: Mutable) *u32 { |
| 510 | return &mutable.mutate.len; |
| 511 | } |
| 512 | |
| 513 | pub fn capacityPtr(mutable: Mutable) *u32 { |
| 514 | return &mutable.list.header().capacity; |
| 515 | } |
| 516 | }; |
| 517 | |
| 518 | const empty: ListSelf = .{ .bytes = @constCast(&(extern struct { |
| 519 | header: Header, |
| 520 | bytes: [0]u8, |
| 521 | }{ |
| 522 | .header = .{ .capacity = 0 }, |
| 523 | .bytes = .{}, |
| 524 | }).bytes) }; |
| 386 | 525 | |
| 387 | | const fields_len = @typeInfo(Header).Struct.fields.len; |
| 526 | const alignment = @max(@alignOf(Header), @alignOf(Elem)); |
| 527 | const bytes_offset = std.mem.alignForward(usize, @sizeOf(Header), @alignOf(Elem)); |
| 528 | const View = std.MultiArrayList(Elem); |
| 529 | |
| 530 | fn acquire(list: *const ListSelf) ListSelf { |
| 531 | return .{ .bytes = @atomicLoad([*]align(@alignOf(Elem)) u8, &list.bytes, .acquire) }; |
| 532 | } |
| 533 | fn release(list: *ListSelf, new_list: ListSelf) void { |
| 534 | @atomicStore([*]align(@alignOf(Elem)) u8, &list.bytes, new_list.bytes, .release); |
| 535 | } |
| 536 | |
| 537 | const Header = extern struct { |
| 538 | capacity: u32, |
| 539 | }; |
| 540 | fn header(list: ListSelf) *Header { |
| 541 | return @ptrFromInt(@intFromPtr(list.bytes) - bytes_offset); |
| 542 | } |
| 543 | |
| 544 | fn view(list: ListSelf) View { |
| 545 | const capacity = list.header().capacity; |
| 546 | return .{ |
| 547 | .bytes = list.bytes, |
| 548 | .len = capacity, |
| 549 | .capacity = capacity, |
| 550 | }; |
| 551 | } |
| 388 | 552 | }; |
| 389 | | fn header(list: List) *Header { |
| 390 | | return @ptrCast(list.entries - Header.fields_len); |
| 391 | | } |
| 392 | | }; |
| 553 | } |
| 554 | |
| 555 | /// In order to store references to strings in fewer bytes, we copy all |
| 556 | /// string bytes into here. String bytes can be null. It is up to whomever |
| 557 | /// is referencing the data here whether they want to store both index and length, |
| 558 | /// thus allowing null bytes, or store only index, and use null-termination. The |
| 559 | /// `strings` array is agnostic to either usage. |
| 560 | pub fn getMutableStrings(local: *Local, gpa: std.mem.Allocator) Strings.Mutable { |
| 561 | return .{ |
| 562 | .gpa = gpa, |
| 563 | .arena = &local.mutate.arena, |
| 564 | .mutate = &local.mutate.strings, |
| 565 | .list = &local.shared.strings, |
| 566 | }; |
| 567 | } |
| 393 | 568 | }; |
| 569 | pub fn getLocal(ip: *InternPool, tid: Zcu.PerThread.Id) *Local { |
| 570 | return &ip.locals[@intFromEnum(tid)]; |
| 571 | } |
| 572 | pub fn getLocalShared(ip: *const InternPool, tid: Zcu.PerThread.Id) *const Local.Shared { |
| 573 | return &ip.locals[@intFromEnum(tid)].shared; |
| 574 | } |
| 394 | 575 | |
| 395 | 576 | const Shard = struct { |
| 396 | 577 | shared: struct { |
| ... | ... | @@ -448,7 +629,7 @@ const Shard = struct { |
| 448 | 629 | } |
| 449 | 630 | }; |
| 450 | 631 | fn header(map: @This()) *Header { |
| 451 | | return &(@as([*]Header, @ptrCast(map.entries)) - 1)[0]; |
| 632 | return @ptrFromInt(@intFromPtr(map.entries) - entries_offset); |
| 452 | 633 | } |
| 453 | 634 | |
| 454 | 635 | const Entry = extern struct { |
| ... | ... | @@ -465,6 +646,17 @@ const Shard = struct { |
| 465 | 646 | }; |
| 466 | 647 | } |
| 467 | 648 | }; |
| 649 | fn getShard(ip: *InternPool, tid: Zcu.PerThread.Id) *Shard { |
| 650 | return &ip.shards[@intFromEnum(tid)]; |
| 651 | } |
| 652 | |
| 653 | fn getTidMask(ip: *const InternPool) u32 { |
| 654 | assert(std.math.isPowerOfTwo(ip.shards.len)); |
| 655 | return @intCast(ip.shards.len - 1); |
| 656 | } |
| 657 | fn getIndexMask(ip: *const InternPool, comptime BackingInt: type) u32 { |
| 658 | return @as(u32, std.math.maxInt(BackingInt)) >> ip.tid_width; |
| 659 | } |
| 468 | 660 | |
| 469 | 661 | const FieldMap = std.ArrayHashMapUnmanaged(void, void, std.array_hash_map.AutoContext(void), false); |
| 470 | 662 | |
| ... | ... | @@ -560,18 +752,18 @@ pub const OptionalNamespaceIndex = enum(u32) { |
| 560 | 752 | } |
| 561 | 753 | }; |
| 562 | 754 | |
| 563 | | /// An index into `string_bytes`. |
| 755 | /// An index into `strings`. |
| 564 | 756 | pub const String = enum(u32) { |
| 565 | 757 | /// An empty string. |
| 566 | 758 | empty = 0, |
| 567 | 759 | _, |
| 568 | 760 | |
| 569 | 761 | pub fn toSlice(string: String, len: u64, ip: *const InternPool) []const u8 { |
| 570 | | return ip.string_bytes.items[@intFromEnum(string)..][0..@intCast(len)]; |
| 762 | return string.toOverlongSlice(ip)[0..@intCast(len)]; |
| 571 | 763 | } |
| 572 | 764 | |
| 573 | 765 | pub fn at(string: String, index: u64, ip: *const InternPool) u8 { |
| 574 | | return ip.string_bytes.items[@intCast(@intFromEnum(string) + index)]; |
| 766 | return string.toOverlongSlice(ip)[@intCast(index)]; |
| 575 | 767 | } |
| 576 | 768 | |
| 577 | 769 | pub fn toNullTerminatedString(string: String, len: u64, ip: *const InternPool) NullTerminatedString { |
| ... | ... | @@ -579,9 +771,32 @@ pub const String = enum(u32) { |
| 579 | 771 | assert(string.at(len, ip) == 0); |
| 580 | 772 | return @enumFromInt(@intFromEnum(string)); |
| 581 | 773 | } |
| 774 | |
| 775 | const Unwrapped = struct { |
| 776 | tid: Zcu.PerThread.Id, |
| 777 | index: u32, |
| 778 | |
| 779 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) String { |
| 780 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 781 | assert(unwrapped.index <= ip.getIndexMask(u32)); |
| 782 | return @enumFromInt(@intFromEnum(unwrapped.tid) << ip.tid_shift_32 | unwrapped.index); |
| 783 | } |
| 784 | }; |
| 785 | fn unwrap(string: String, ip: *const InternPool) Unwrapped { |
| 786 | return .{ |
| 787 | .tid = @enumFromInt(@intFromEnum(string) >> ip.tid_shift_32 & ip.getTidMask()), |
| 788 | .index = @intFromEnum(string) & ip.getIndexMask(u32), |
| 789 | }; |
| 790 | } |
| 791 | |
| 792 | fn toOverlongSlice(string: String, ip: *const InternPool) []const u8 { |
| 793 | const unwrapped = string.unwrap(ip); |
| 794 | const strings = ip.getLocalShared(unwrapped.tid).strings.acquire(); |
| 795 | return strings.view().items(.@"0")[unwrapped.index..]; |
| 796 | } |
| 582 | 797 | }; |
| 583 | 798 | |
| 584 | | /// An index into `string_bytes` which might be `none`. |
| 799 | /// An index into `strings` which might be `none`. |
| 585 | 800 | pub const OptionalString = enum(u32) { |
| 586 | 801 | /// This is distinct from `none` - it is a valid index that represents empty string. |
| 587 | 802 | empty = 0, |
| ... | ... | @@ -597,7 +812,7 @@ pub const OptionalString = enum(u32) { |
| 597 | 812 | } |
| 598 | 813 | }; |
| 599 | 814 | |
| 600 | | /// An index into `string_bytes`. |
| 815 | /// An index into `strings`. |
| 601 | 816 | pub const NullTerminatedString = enum(u32) { |
| 602 | 817 | /// An empty string. |
| 603 | 818 | empty = 0, |
| ... | ... | @@ -623,12 +838,8 @@ pub const NullTerminatedString = enum(u32) { |
| 623 | 838 | return @enumFromInt(@intFromEnum(self)); |
| 624 | 839 | } |
| 625 | 840 | |
| 626 | | fn toOverlongSlice(string: NullTerminatedString, ip: *const InternPool) []const u8 { |
| 627 | | return ip.string_bytes.items[@intFromEnum(string)..]; |
| 628 | | } |
| 629 | | |
| 630 | 841 | pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 { |
| 631 | | const overlong_slice = string.toOverlongSlice(ip); |
| 842 | const overlong_slice = string.toString().toOverlongSlice(ip); |
| 632 | 843 | return overlong_slice[0..std.mem.indexOfScalar(u8, overlong_slice, 0).? :0]; |
| 633 | 844 | } |
| 634 | 845 | |
| ... | ... | @@ -637,7 +848,7 @@ pub const NullTerminatedString = enum(u32) { |
| 637 | 848 | } |
| 638 | 849 | |
| 639 | 850 | pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool { |
| 640 | | const overlong_slice = string.toOverlongSlice(ip); |
| 851 | const overlong_slice = string.toString().toOverlongSlice(ip); |
| 641 | 852 | return overlong_slice.len > slice.len and |
| 642 | 853 | std.mem.eql(u8, overlong_slice[0..slice.len], slice) and |
| 643 | 854 | overlong_slice[slice.len] == 0; |
| ... | ... | @@ -688,12 +899,12 @@ pub const NullTerminatedString = enum(u32) { |
| 688 | 899 | } else @compileError("invalid format string '" ++ specifier ++ "' for '" ++ @typeName(NullTerminatedString) ++ "'"); |
| 689 | 900 | } |
| 690 | 901 | |
| 691 | | pub fn fmt(self: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) { |
| 692 | | return .{ .data = .{ .string = self, .ip = ip } }; |
| 902 | pub fn fmt(string: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) { |
| 903 | return .{ .data = .{ .string = string, .ip = ip } }; |
| 693 | 904 | } |
| 694 | 905 | }; |
| 695 | 906 | |
| 696 | | /// An index into `string_bytes` which might be `none`. |
| 907 | /// An index into `strings` which might be `none`. |
| 697 | 908 | pub const OptionalNullTerminatedString = enum(u32) { |
| 698 | 909 | /// This is distinct from `none` - it is a valid index that represents empty string. |
| 699 | 910 | empty = 0, |
| ... | ... | @@ -4077,7 +4288,7 @@ pub const FuncAnalysis = packed struct(u32) { |
| 4077 | 4288 | pub const Bytes = struct { |
| 4078 | 4289 | /// The type of the aggregate |
| 4079 | 4290 | ty: Index, |
| 4080 | | /// Index into string_bytes, of len ip.aggregateTypeLen(ty) |
| 4291 | /// Index into strings, of len ip.aggregateTypeLen(ty) |
| 4081 | 4292 | bytes: String, |
| 4082 | 4293 | }; |
| 4083 | 4294 | |
| ... | ... | @@ -4647,16 +4858,21 @@ pub fn init(ip: *InternPool, gpa: Allocator, total_threads: usize) !void { |
| 4647 | 4858 | errdefer ip.deinit(gpa); |
| 4648 | 4859 | assert(ip.items.len == 0); |
| 4649 | 4860 | |
| 4650 | | ip.local = try gpa.alloc(Local, total_threads); |
| 4651 | | @memset(ip.local, .{ |
| 4652 | | .items = Local.List.empty, |
| 4653 | | .extra = Local.List.empty, |
| 4654 | | .strings = Local.List.empty, |
| 4655 | | .arena = .{}, |
| 4861 | ip.locals = try gpa.alloc(Local, total_threads); |
| 4862 | @memset(ip.locals, .{ |
| 4863 | .shared = .{ |
| 4864 | .strings = Local.Strings.empty, |
| 4865 | }, |
| 4866 | .mutate = .{ |
| 4867 | .arena = .{}, |
| 4868 | .strings = Local.Mutate.empty, |
| 4869 | }, |
| 4656 | 4870 | }); |
| 4657 | 4871 | |
| 4658 | | ip.shard_shift = @intCast(std.math.log2_int_ceil(usize, total_threads)); |
| 4659 | | ip.shards = try gpa.alloc(Shard, @as(usize, 1) << ip.shard_shift); |
| 4872 | ip.tid_width = @intCast(std.math.log2_int_ceil(usize, total_threads)); |
| 4873 | ip.tid_shift_31 = 31 - ip.tid_width; |
| 4874 | ip.tid_shift_32 = ip.tid_shift_31 +| 1; |
| 4875 | ip.shards = try gpa.alloc(Shard, @as(usize, 1) << ip.tid_width); |
| 4660 | 4876 | @memset(ip.shards, .{ |
| 4661 | 4877 | .shared = .{ |
| 4662 | 4878 | .map = Shard.Map(Index).empty, |
| ... | ... | @@ -4705,7 +4921,6 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void { |
| 4705 | 4921 | ip.items.deinit(gpa); |
| 4706 | 4922 | ip.extra.deinit(gpa); |
| 4707 | 4923 | ip.limbs.deinit(gpa); |
| 4708 | | ip.string_bytes.deinit(gpa); |
| 4709 | 4924 | |
| 4710 | 4925 | ip.decls_free_list.deinit(gpa); |
| 4711 | 4926 | ip.allocated_decls.deinit(gpa); |
| ... | ... | @@ -4732,8 +4947,8 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void { |
| 4732 | 4947 | ip.files.deinit(gpa); |
| 4733 | 4948 | |
| 4734 | 4949 | gpa.free(ip.shards); |
| 4735 | | for (ip.local) |*local| local.arena.promote(gpa).deinit(); |
| 4736 | | gpa.free(ip.local); |
| 4950 | for (ip.locals) |*local| local.mutate.arena.promote(gpa).deinit(); |
| 4951 | gpa.free(ip.locals); |
| 4737 | 4952 | |
| 4738 | 4953 | ip.* = undefined; |
| 4739 | 4954 | } |
| ... | ... | @@ -5437,8 +5652,9 @@ fn getOrPutKey( |
| 5437 | 5652 | } |
| 5438 | 5653 | const map_header = map.header().*; |
| 5439 | 5654 | if (shard.mutate.map.len >= map_header.capacity * 3 / 5) { |
| 5440 | | var arena = ip.local[@intFromEnum(tid)].arena.promote(gpa); |
| 5441 | | defer ip.local[@intFromEnum(tid)].arena = arena.state; |
| 5655 | const arena_state = &ip.getLocal(tid).mutate.arena; |
| 5656 | var arena = arena_state.promote(gpa); |
| 5657 | defer arena_state.* = arena.state; |
| 5442 | 5658 | const new_map_capacity = map_header.capacity * 2; |
| 5443 | 5659 | const new_map_buf = try arena.allocator().alignedAlloc( |
| 5444 | 5660 | u8, |
| ... | ... | @@ -6194,33 +6410,32 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 6194 | 6410 | } |
| 6195 | 6411 | |
| 6196 | 6412 | if (child == .u8_type) bytes: { |
| 6197 | | const string_bytes_index = ip.string_bytes.items.len; |
| 6198 | | try ip.string_bytes.ensureUnusedCapacity(gpa, @intCast(len_including_sentinel + 1)); |
| 6413 | const strings = ip.getLocal(tid).getMutableStrings(gpa); |
| 6414 | const start = strings.lenPtr().*; |
| 6415 | try strings.ensureUnusedCapacity(@intCast(len_including_sentinel + 1)); |
| 6199 | 6416 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Bytes).Struct.fields.len); |
| 6200 | 6417 | switch (aggregate.storage) { |
| 6201 | | .bytes => |bytes| ip.string_bytes.appendSliceAssumeCapacity(bytes.toSlice(len, ip)), |
| 6418 | .bytes => |bytes| strings.appendSliceAssumeCapacity(.{bytes.toSlice(len, ip)}), |
| 6202 | 6419 | .elems => |elems| for (elems[0..@intCast(len)]) |elem| switch (ip.indexToKey(elem)) { |
| 6203 | 6420 | .undef => { |
| 6204 | | ip.string_bytes.shrinkRetainingCapacity(string_bytes_index); |
| 6421 | strings.shrinkRetainingCapacity(start); |
| 6205 | 6422 | break :bytes; |
| 6206 | 6423 | }, |
| 6207 | | .int => |int| ip.string_bytes.appendAssumeCapacity( |
| 6208 | | @intCast(int.storage.u64), |
| 6209 | | ), |
| 6424 | .int => |int| strings.appendAssumeCapacity(.{@intCast(int.storage.u64)}), |
| 6210 | 6425 | else => unreachable, |
| 6211 | 6426 | }, |
| 6212 | 6427 | .repeated_elem => |elem| switch (ip.indexToKey(elem)) { |
| 6213 | 6428 | .undef => break :bytes, |
| 6214 | 6429 | .int => |int| @memset( |
| 6215 | | ip.string_bytes.addManyAsSliceAssumeCapacity(@intCast(len)), |
| 6430 | strings.addManyAsSliceAssumeCapacity(@intCast(len))[0], |
| 6216 | 6431 | @intCast(int.storage.u64), |
| 6217 | 6432 | ), |
| 6218 | 6433 | else => unreachable, |
| 6219 | 6434 | }, |
| 6220 | 6435 | } |
| 6221 | | if (sentinel != .none) ip.string_bytes.appendAssumeCapacity( |
| 6436 | if (sentinel != .none) strings.appendAssumeCapacity(.{ |
| 6222 | 6437 | @intCast(ip.indexToKey(sentinel).int.storage.u64), |
| 6223 | | ); |
| 6438 | }); |
| 6224 | 6439 | const string = try ip.getOrPutTrailingString( |
| 6225 | 6440 | gpa, |
| 6226 | 6441 | tid, |
| ... | ... | @@ -9050,10 +9265,11 @@ pub fn getOrPutString( |
| 9050 | 9265 | slice: []const u8, |
| 9051 | 9266 | comptime embedded_nulls: EmbeddedNulls, |
| 9052 | 9267 | ) Allocator.Error!embedded_nulls.StringType() { |
| 9053 | | try ip.string_bytes.ensureUnusedCapacity(gpa, slice.len + 1); |
| 9054 | | ip.string_bytes.appendSliceAssumeCapacity(slice); |
| 9055 | | ip.string_bytes.appendAssumeCapacity(0); |
| 9056 | | return ip.getOrPutTrailingString(gpa, tid, slice.len + 1, embedded_nulls); |
| 9268 | const strings = ip.getLocal(tid).getMutableStrings(gpa); |
| 9269 | try strings.ensureUnusedCapacity(slice.len + 1); |
| 9270 | strings.appendSliceAssumeCapacity(.{slice}); |
| 9271 | strings.appendAssumeCapacity(.{0}); |
| 9272 | return ip.getOrPutTrailingString(gpa, tid, @intCast(slice.len + 1), embedded_nulls); |
| 9057 | 9273 | } |
| 9058 | 9274 | |
| 9059 | 9275 | pub fn getOrPutStringFmt( |
| ... | ... | @@ -9064,11 +9280,12 @@ pub fn getOrPutStringFmt( |
| 9064 | 9280 | args: anytype, |
| 9065 | 9281 | comptime embedded_nulls: EmbeddedNulls, |
| 9066 | 9282 | ) Allocator.Error!embedded_nulls.StringType() { |
| 9067 | | // ensure that references to string_bytes in args do not get invalidated |
| 9068 | | const len: usize = @intCast(std.fmt.count(format, args) + 1); |
| 9069 | | try ip.string_bytes.ensureUnusedCapacity(gpa, len); |
| 9070 | | ip.string_bytes.writer(undefined).print(format, args) catch unreachable; |
| 9071 | | ip.string_bytes.appendAssumeCapacity(0); |
| 9283 | // ensure that references to strings in args do not get invalidated |
| 9284 | const format_z = format ++ .{0}; |
| 9285 | const len: u32 = @intCast(std.fmt.count(format_z, args)); |
| 9286 | const strings = ip.getLocal(tid).getMutableStrings(gpa); |
| 9287 | const slice = try strings.addManyAsSlice(len); |
| 9288 | assert((std.fmt.bufPrint(slice[0], format_z, args) catch unreachable).len == len); |
| 9072 | 9289 | return ip.getOrPutTrailingString(gpa, tid, len, embedded_nulls); |
| 9073 | 9290 | } |
| 9074 | 9291 | |
| ... | ... | @@ -9083,47 +9300,33 @@ pub fn getOrPutStringOpt( |
| 9083 | 9300 | return string.toOptional(); |
| 9084 | 9301 | } |
| 9085 | 9302 | |
| 9086 | | /// Uses the last len bytes of ip.string_bytes as the key. |
| 9303 | /// Uses the last len bytes of strings as the key. |
| 9087 | 9304 | pub fn getOrPutTrailingString( |
| 9088 | 9305 | ip: *InternPool, |
| 9089 | 9306 | gpa: Allocator, |
| 9090 | 9307 | tid: Zcu.PerThread.Id, |
| 9091 | | len: usize, |
| 9308 | len: u32, |
| 9092 | 9309 | comptime embedded_nulls: EmbeddedNulls, |
| 9093 | 9310 | ) Allocator.Error!embedded_nulls.StringType() { |
| 9094 | | const string_bytes = &ip.string_bytes; |
| 9095 | | const str_index: u32 = @intCast(string_bytes.items.len - len); |
| 9096 | | if (len > 0 and string_bytes.getLast() == 0) { |
| 9097 | | _ = string_bytes.pop(); |
| 9311 | const strings = ip.getLocal(tid).getMutableStrings(gpa); |
| 9312 | const start: u32 = @intCast(strings.lenPtr().* - len); |
| 9313 | if (len > 0 and strings.view().items(.@"0")[strings.lenPtr().* - 1] == 0) { |
| 9314 | strings.lenPtr().* -= 1; |
| 9098 | 9315 | } else { |
| 9099 | | try string_bytes.ensureUnusedCapacity(gpa, 1); |
| 9316 | try strings.ensureUnusedCapacity(1); |
| 9100 | 9317 | } |
| 9101 | | const key: []const u8 = string_bytes.items[str_index..]; |
| 9318 | const key: []const u8 = strings.view().items(.@"0")[start..]; |
| 9319 | const value: embedded_nulls.StringType() = |
| 9320 | @enumFromInt(@intFromEnum(tid) << ip.tid_shift_32 | start); |
| 9102 | 9321 | const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null; |
| 9103 | 9322 | switch (embedded_nulls) { |
| 9104 | 9323 | .no_embedded_nulls => assert(!has_embedded_null), |
| 9105 | 9324 | .maybe_embedded_nulls => if (has_embedded_null) { |
| 9106 | | string_bytes.appendAssumeCapacity(0); |
| 9107 | | return @enumFromInt(str_index); |
| 9325 | strings.appendAssumeCapacity(.{0}); |
| 9326 | return value; |
| 9108 | 9327 | }, |
| 9109 | 9328 | } |
| 9110 | | const maybe_existing_index = try ip.getOrPutStringValue(gpa, tid, key, @enumFromInt(str_index)); |
| 9111 | | if (maybe_existing_index.unwrap()) |existing_index| { |
| 9112 | | string_bytes.shrinkRetainingCapacity(str_index); |
| 9113 | | return @enumFromInt(@intFromEnum(existing_index)); |
| 9114 | | } else { |
| 9115 | | string_bytes.appendAssumeCapacity(0); |
| 9116 | | return @enumFromInt(str_index); |
| 9117 | | } |
| 9118 | | } |
| 9119 | 9329 | |
| 9120 | | fn getOrPutStringValue( |
| 9121 | | ip: *InternPool, |
| 9122 | | gpa: Allocator, |
| 9123 | | tid: Zcu.PerThread.Id, |
| 9124 | | key: []const u8, |
| 9125 | | value: NullTerminatedString, |
| 9126 | | ) Allocator.Error!OptionalNullTerminatedString { |
| 9127 | 9330 | const full_hash = Hash.hash(0, key); |
| 9128 | 9331 | const hash: u32 = @truncate(full_hash >> 32); |
| 9129 | 9332 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; |
| ... | ... | @@ -9136,7 +9339,9 @@ fn getOrPutStringValue( |
| 9136 | 9339 | const entry = &map.entries[map_index]; |
| 9137 | 9340 | const index = entry.acquire().unwrap() orelse break; |
| 9138 | 9341 | if (entry.hash != hash) continue; |
| 9139 | | if (index.eqlSlice(key, ip)) return index.toOptional(); |
| 9342 | if (!index.eqlSlice(key, ip)) continue; |
| 9343 | strings.shrinkRetainingCapacity(start); |
| 9344 | return @enumFromInt(@intFromEnum(index)); |
| 9140 | 9345 | } |
| 9141 | 9346 | shard.mutate.string_map.mutex.lock(); |
| 9142 | 9347 | defer shard.mutate.string_map.mutex.unlock(); |
| ... | ... | @@ -9151,18 +9356,22 @@ fn getOrPutStringValue( |
| 9151 | 9356 | const entry = &map.entries[map_index]; |
| 9152 | 9357 | const index = entry.acquire().unwrap() orelse break; |
| 9153 | 9358 | if (entry.hash != hash) continue; |
| 9154 | | if (index.eqlSlice(key, ip)) return index.toOptional(); |
| 9359 | if (!index.eqlSlice(key, ip)) continue; |
| 9360 | strings.shrinkRetainingCapacity(start); |
| 9361 | return @enumFromInt(@intFromEnum(index)); |
| 9155 | 9362 | } |
| 9156 | 9363 | defer shard.mutate.string_map.len += 1; |
| 9157 | 9364 | const map_header = map.header().*; |
| 9158 | 9365 | if (shard.mutate.string_map.len < map_header.capacity * 3 / 5) { |
| 9159 | 9366 | const entry = &map.entries[map_index]; |
| 9160 | 9367 | entry.hash = hash; |
| 9161 | | entry.release(value.toOptional()); |
| 9162 | | return .none; |
| 9368 | entry.release(@enumFromInt(@intFromEnum(value))); |
| 9369 | strings.appendAssumeCapacity(.{0}); |
| 9370 | return value; |
| 9163 | 9371 | } |
| 9164 | | var arena = ip.local[@intFromEnum(tid)].arena.promote(gpa); |
| 9165 | | defer ip.local[@intFromEnum(tid)].arena = arena.state; |
| 9372 | const arena_state = &ip.getLocal(tid).mutate.arena; |
| 9373 | var arena = arena_state.promote(gpa); |
| 9374 | defer arena_state.* = arena.state; |
| 9166 | 9375 | const new_map_capacity = map_header.capacity * 2; |
| 9167 | 9376 | const new_map_buf = try arena.allocator().alignedAlloc( |
| 9168 | 9377 | u8, |
| ... | ... | @@ -9197,11 +9406,12 @@ fn getOrPutStringValue( |
| 9197 | 9406 | if (map.entries[map_index].value == .none) break; |
| 9198 | 9407 | } |
| 9199 | 9408 | map.entries[map_index] = .{ |
| 9200 | | .value = value.toOptional(), |
| 9409 | .value = @enumFromInt(@intFromEnum(value)), |
| 9201 | 9410 | .hash = hash, |
| 9202 | 9411 | }; |
| 9203 | 9412 | shard.shared.string_map.release(new_map); |
| 9204 | | return .none; |
| 9413 | strings.appendAssumeCapacity(.{0}); |
| 9414 | return value; |
| 9205 | 9415 | } |
| 9206 | 9416 | |
| 9207 | 9417 | pub fn getString(ip: *InternPool, key: []const u8) OptionalNullTerminatedString { |
Jacob Young <jacobly@ziglang.org> 2024-07-03 22:37:09-04:00