| ... | ... | @@ -682,45 +682,40 @@ test "reset while retaining a buffer" { |
| 682 | 682 | try std.testing.expectEqual(2, arena_allocator.queryCapacity()); |
| 683 | 683 | } |
| 684 | 684 | |
| 685 | | test "fuzz" { |
| 685 | test "fuzz multi threaded" { |
| 686 | 686 | @disableInstrumentation(); |
| 687 | 687 | if (@import("builtin").single_threaded) return error.SkipZigTest; |
| 688 | 688 | |
| 689 | 689 | const gpa = std.heap.smp_allocator; |
| 690 | 690 | |
| 691 | var io_instance: std.Io.Threaded = .init(gpa, .{}); |
| 692 | defer io_instance.deinit(); |
| 693 | |
| 691 | 694 | var arena_state: ArenaAllocator.State = .init; |
| 692 | 695 | // No need to deinit arena_state, all allocations are in `sample_buffer`! |
| 693 | 696 | |
| 694 | | const control_buffer = try gpa.alloc(u8, 64 << 10 << 10); |
| 697 | const buffer_size = FuzzContext.max_alloc_count * FuzzContext.max_alloc_size; |
| 698 | |
| 699 | const control_buffer = try gpa.alloc(u8, buffer_size); |
| 695 | 700 | defer gpa.free(control_buffer); |
| 696 | 701 | var control_instance: std.heap.FixedBufferAllocator = .init(control_buffer); |
| 697 | 702 | |
| 698 | | const sample_buffer = try gpa.alloc(u8, 64 << 10 << 10); |
| 703 | const sample_buffer = try gpa.alloc(u8, buffer_size); |
| 699 | 704 | defer gpa.free(sample_buffer); |
| 700 | 705 | var sample_instance: FuzzAllocator = .init(sample_buffer); |
| 701 | 706 | |
| 702 | | var allocs: FuzzContext.Allocs = try .initCapacity(gpa, FuzzContext.max_alloc_count); |
| 703 | | defer allocs.deinit(gpa); |
| 704 | | |
| 705 | 707 | try std.testing.fuzz(FuzzContext.Init{ |
| 706 | | .gpa = gpa, |
| 707 | | .allocs = &allocs, |
| 708 | .threaded_instance = &io_instance, |
| 708 | 709 | .arena_state = &arena_state, |
| 709 | 710 | .control_instance = &control_instance, |
| 710 | 711 | .sample_instance = &sample_instance, |
| 711 | | }, fuzzArenaAllocator, .{}); |
| 712 | }, fuzzMultiThreaded, .{}); |
| 712 | 713 | } |
| 713 | 714 | |
| 714 | | fn fuzzArenaAllocator(fuzz_init: FuzzContext.Init, smith: *std.testing.Smith) anyerror!void { |
| 715 | fn fuzzMultiThreaded(fuzz_init: FuzzContext.Init, smith: *std.testing.Smith) anyerror!void { |
| 715 | 716 | @disableInstrumentation(); |
| 716 | 717 | const testing = std.testing; |
| 717 | | |
| 718 | | // We use a 'fresh' `Threaded` instance every time to reset threadlocals to |
| 719 | | // their default values. |
| 720 | | |
| 721 | | var io_instance: std.Io.Threaded = .init(fuzz_init.gpa, .{}); |
| 722 | | defer io_instance.deinit(); |
| 723 | | const io = io_instance.io(); |
| 718 | const io = fuzz_init.threaded_instance.io(); |
| 724 | 719 | |
| 725 | 720 | fuzz_init.sample_instance.prepareFailures(smith); |
| 726 | 721 | |
| ... | ... | @@ -731,59 +726,57 @@ fn fuzzArenaAllocator(fuzz_init: FuzzContext.Init, smith: *std.testing.Smith) an |
| 731 | 726 | defer fuzz_init.arena_state.* = arena_instance.state; |
| 732 | 727 | |
| 733 | 728 | var ctx: FuzzContext = .init( |
| 734 | | io, |
| 735 | 729 | control_allocator, |
| 736 | 730 | arena_instance.allocator(), |
| 737 | | fuzz_init.allocs, |
| 738 | 731 | ); |
| 739 | 732 | defer ctx.deinit(); |
| 740 | 733 | |
| 741 | | ctx.rwl.lockUncancelable(io); |
| 742 | | |
| 743 | 734 | var group: std.Io.Group = .init; |
| 744 | 735 | defer group.cancel(io); |
| 745 | 736 | |
| 737 | var n_allocs: usize = 0; |
| 746 | 738 | var n_actions: usize = 0; |
| 747 | 739 | while (!smith.eosWeightedSimple(99, 1) and n_actions < FuzzContext.max_action_count) { |
| 748 | 740 | errdefer comptime unreachable; |
| 749 | 741 | |
| 750 | | const ActionTag = @typeInfo(FuzzContext.Action).@"union".tag_type.?; |
| 751 | | const weights: []const testing.Smith.Weight = weights: { |
| 752 | | if (ctx.allocs.len == ctx.allocs.capacity) |
| 753 | | break :weights &.{ |
| 754 | | .value(ActionTag, .resize, 1), |
| 755 | | .value(ActionTag, .remap, 1), |
| 756 | | .value(ActionTag, .free, 1), |
| 757 | | }; |
| 758 | | break :weights testing.Smith.baselineWeights(ActionTag) ++ |
| 759 | | .{testing.Smith.Weight.value(ActionTag, .alloc, 2)}; |
| 760 | | }; |
| 761 | | const action: FuzzContext.Action = switch (smith.valueWeighted(ActionTag, weights)) { |
| 762 | | .alloc => action: { |
| 763 | | const alloc_index = ctx.allocs.addOneBounded() catch continue; |
| 764 | | ctx.allocs.items(.len)[alloc_index] = .free; |
| 765 | | break :action .{ .alloc = .{ |
| 766 | | .len = nextLen(smith), |
| 767 | | .alignment = smith.valueRangeAtMost( |
| 742 | const weights: []const testing.Smith.Weight = if (n_allocs == FuzzContext.max_alloc_count) |
| 743 | &.{ |
| 744 | .value(FuzzContext.Action, .resize, 1), |
| 745 | .value(FuzzContext.Action, .remap, 1), |
| 746 | .value(FuzzContext.Action, .free, 1), |
| 747 | } |
| 748 | else |
| 749 | &.{ |
| 750 | .value(FuzzContext.Action, .resize, 1), |
| 751 | .value(FuzzContext.Action, .remap, 1), |
| 752 | .value(FuzzContext.Action, .free, 1), |
| 753 | .value(FuzzContext.Action, .alloc, 3), |
| 754 | }; |
| 755 | switch (smith.valueWeighted(FuzzContext.Action, weights)) { |
| 756 | .alloc => { |
| 757 | const alloc_index = n_allocs; |
| 758 | n_allocs += 1; |
| 759 | ctx.allocs[alloc_index].common.len = .free; |
| 760 | group.concurrent(io, FuzzContext.doOneAlloc, .{ |
| 761 | &ctx, |
| 762 | nextLen(smith), |
| 763 | smith.valueRangeAtMost( |
| 768 | 764 | Alignment, |
| 769 | 765 | .@"1", |
| 770 | 766 | .fromByteUnits(2 * std.heap.page_size_max), |
| 771 | 767 | ), |
| 772 | | .index = alloc_index, |
| 773 | | } }; |
| 768 | @enumFromInt(alloc_index), |
| 769 | }) catch unreachable; |
| 774 | 770 | }, |
| 775 | | .resize => .{ .resize = .{ .new_len = nextLen(smith) } }, |
| 776 | | .remap => .{ .remap = .{ .new_len = nextLen(smith) } }, |
| 777 | | .free => .free, |
| 778 | | }; |
| 779 | | group.concurrent(io, FuzzContext.doOneAction, .{ &ctx, action }) catch break; |
| 771 | .resize => group.concurrent(io, FuzzContext.doOneResize, .{ &ctx, nextLen(smith) }) catch unreachable, |
| 772 | .remap => group.concurrent(io, FuzzContext.doOneRemap, .{ &ctx, nextLen(smith) }) catch unreachable, |
| 773 | .free => group.concurrent(io, FuzzContext.doOneFree, .{&ctx}) catch unreachable, |
| 774 | } |
| 780 | 775 | n_actions += 1; |
| 781 | 776 | } |
| 782 | 777 | |
| 783 | | ctx.rwl.unlock(io); |
| 784 | | |
| 785 | 778 | try group.await(io); |
| 786 | | try ctx.check(); |
| 779 | try ctx.check(n_allocs); |
| 787 | 780 | |
| 788 | 781 | // This also covers the `deinit` logic since `free_all` uses it internally. |
| 789 | 782 | |
| ... | ... | @@ -806,73 +799,71 @@ fn fuzzArenaAllocator(fuzz_init: FuzzContext.Init, smith: *std.testing.Smith) an |
| 806 | 799 | } |
| 807 | 800 | |
| 808 | 801 | fuzz_init.control_instance.reset(); |
| 809 | | fuzz_init.allocs.clearRetainingCapacity(); |
| 810 | | } |
| 811 | | fn nextLen(smith: *std.testing.Smith) usize { |
| 812 | | @disableInstrumentation(); |
| 813 | | return usizeRange(smith, 1, 16 << 10 << 10); |
| 814 | 802 | } |
| 815 | | fn usizeRange(smith: *std.testing.Smith, at_least: usize, at_most: usize) usize { |
| 803 | fn nextLen(smith: *std.testing.Smith) @typeInfo(FuzzContext.Alloc.Len).@"enum".tag_type { |
| 816 | 804 | @disableInstrumentation(); |
| 817 | | const Int = @Int(.unsigned, @min(64, @bitSizeOf(usize))); |
| 818 | | return smith.valueRangeAtMost(Int, @intCast(at_least), @intCast(at_most)); |
| 805 | const BackingInt = @typeInfo(FuzzContext.Alloc.Len).@"enum".tag_type; |
| 806 | return smith.valueRangeAtMost(BackingInt, 1, FuzzContext.max_alloc_size); |
| 819 | 807 | } |
| 820 | 808 | |
| 821 | 809 | const FuzzContext = struct { |
| 822 | | io: std.Io, |
| 823 | | rwl: std.Io.RwLock, |
| 824 | | |
| 825 | 810 | control_allocator: Allocator, |
| 826 | 811 | sample_allocator: Allocator, |
| 827 | 812 | |
| 828 | | allocs: *Allocs, |
| 813 | last_alloc_index: Alloc.Index, |
| 814 | allocs: [max_alloc_count]Alloc, |
| 829 | 815 | |
| 830 | | const max_alloc_count = 4096; |
| 816 | const max_alloc_count = 64; |
| 831 | 817 | const max_action_count = 2 * max_alloc_count; |
| 832 | 818 | |
| 833 | | const Allocs = std.MultiArrayList(struct { |
| 819 | const max_alloc_size = 16 << 10; |
| 820 | |
| 821 | const Alloc = struct { |
| 834 | 822 | control_ptr: [*]u8, |
| 835 | 823 | sample_ptr: [*]u8, |
| 836 | | len: Len, |
| 837 | | alignment: Alignment, |
| 838 | | }); |
| 824 | common: packed struct(usize) { |
| 825 | len: Len, |
| 826 | alignment: Alignment, |
| 827 | _: @Int(.unsigned, padding_bits) = 0, |
| 828 | }, |
| 829 | |
| 830 | const Len = enum(@Int(.unsigned, len_bits)) { |
| 831 | free = (1 << len_bits) - 1, |
| 832 | _, |
| 833 | }; |
| 834 | const len_bits = @min(64, @bitSizeOf(usize)) - @bitSizeOf(Alignment); |
| 835 | const padding_bits = @bitSizeOf(usize) - (len_bits + @bitSizeOf(Alignment)); |
| 839 | 836 | |
| 840 | | const Len = enum(usize) { |
| 841 | | free = std.math.maxInt(usize), |
| 842 | | _, |
| 837 | const Index = enum(usize) { |
| 838 | none = std.math.maxInt(usize), |
| 839 | _, |
| 840 | }; |
| 843 | 841 | }; |
| 844 | 842 | |
| 845 | | const Action = union(enum(u8)) { |
| 846 | | alloc: struct { len: usize, alignment: Alignment, index: usize }, |
| 847 | | resize: struct { new_len: usize }, |
| 848 | | remap: struct { new_len: usize }, |
| 843 | const Action = enum { |
| 844 | alloc, |
| 845 | resize, |
| 846 | remap, |
| 849 | 847 | free, |
| 850 | 848 | }; |
| 851 | 849 | |
| 852 | | threadlocal var tls_next: u8 = 0; |
| 853 | | threadlocal var tls_last_index: ?usize = null; |
| 854 | | |
| 855 | 850 | const Init = struct { |
| 856 | | gpa: Allocator, |
| 857 | | allocs: *FuzzContext.Allocs, |
| 851 | threaded_instance: *std.Io.Threaded, |
| 858 | 852 | arena_state: *ArenaAllocator.State, |
| 859 | 853 | control_instance: *std.heap.FixedBufferAllocator, |
| 860 | 854 | sample_instance: *FuzzAllocator, |
| 861 | 855 | }; |
| 862 | 856 | |
| 863 | 857 | fn init( |
| 864 | | io: std.Io, |
| 865 | 858 | control_allocator: Allocator, |
| 866 | 859 | sample_allocator: Allocator, |
| 867 | | allocs: *Allocs, |
| 868 | 860 | ) FuzzContext { |
| 869 | 861 | @disableInstrumentation(); |
| 870 | 862 | return .{ |
| 871 | | .io = io, |
| 872 | | .rwl = .init, |
| 873 | 863 | .control_allocator = control_allocator, |
| 874 | 864 | .sample_allocator = sample_allocator, |
| 875 | | .allocs = allocs, |
| 865 | .last_alloc_index = .none, |
| 866 | .allocs = undefined, |
| 876 | 867 | }; |
| 877 | 868 | } |
| 878 | 869 | |
| ... | ... | @@ -881,35 +872,22 @@ const FuzzContext = struct { |
| 881 | 872 | ctx.* = undefined; |
| 882 | 873 | } |
| 883 | 874 | |
| 884 | | fn check(ctx: *const FuzzContext) !void { |
| 875 | fn check(ctx: *const FuzzContext, n_allocs: usize) !void { |
| 885 | 876 | @disableInstrumentation(); |
| 886 | | for (0..ctx.allocs.len) |index| { |
| 887 | | const len: usize = switch (ctx.allocs.items(.len)[index]) { |
| 877 | for (ctx.allocs[0..n_allocs]) |allocation| { |
| 878 | const len: usize = switch (allocation.common.len) { |
| 888 | 879 | .free => continue, |
| 889 | 880 | _ => |len| @intFromEnum(len), |
| 890 | 881 | }; |
| 891 | | const control = ctx.allocs.items(.control_ptr)[index][0..len]; |
| 892 | | const sample = ctx.allocs.items(.sample_ptr)[index][0..len]; |
| 882 | const control = allocation.control_ptr[0..len]; |
| 883 | const sample = allocation.sample_ptr[0..len]; |
| 893 | 884 | try std.testing.expectEqualSlices(u8, control, sample); |
| 894 | 885 | } |
| 895 | 886 | } |
| 896 | 887 | |
| 897 | | fn doOneAction(ctx: *FuzzContext, action: Action) std.Io.Cancelable!void { |
| 898 | | @disableInstrumentation(); |
| 899 | | ctx.rwl.lockSharedUncancelable(ctx.io); |
| 900 | | defer ctx.rwl.unlockShared(ctx.io); |
| 901 | | |
| 902 | | switch (action) { |
| 903 | | .alloc => |act| ctx.doOneAlloc(act.len, act.alignment, act.index), |
| 904 | | .resize => |act| ctx.doOneResize(act.new_len), |
| 905 | | .remap => |act| ctx.doOneRemap(act.new_len), |
| 906 | | .free => ctx.doOneFree(), |
| 907 | | } |
| 908 | | } |
| 909 | | |
| 910 | | fn doOneAlloc(ctx: *FuzzContext, len: usize, alignment: Alignment, index: usize) void { |
| 888 | fn doOneAlloc(ctx: *FuzzContext, len: usize, alignment: Alignment, index: Alloc.Index) void { |
| 911 | 889 | @disableInstrumentation(); |
| 912 | | assert(ctx.allocs.items(.len)[index] == .free); |
| 890 | assert(ctx.allocs[@intFromEnum(index)].common.len == .free); |
| 913 | 891 | |
| 914 | 892 | const control_ptr = ctx.control_allocator.rawAlloc(len, alignment, @returnAddress()) orelse |
| 915 | 893 | return; |
| ... | ... | @@ -918,38 +896,42 @@ const FuzzContext = struct { |
| 918 | 896 | return; |
| 919 | 897 | }; |
| 920 | 898 | |
| 921 | | ctx.allocs.set(index, .{ |
| 899 | ctx.allocs[@intFromEnum(index)] = .{ |
| 922 | 900 | .control_ptr = control_ptr, |
| 923 | 901 | .sample_ptr = sample_ptr, |
| 924 | | .len = @enumFromInt(len), |
| 925 | | .alignment = alignment, |
| 926 | | }); |
| 927 | | |
| 928 | | for (control_ptr[0..len], sample_ptr[0..len]) |*control, *sample| { |
| 929 | | control.* = tls_next; |
| 930 | | sample.* = tls_next; |
| 931 | | tls_next +%= 1; |
| 902 | .common = .{ |
| 903 | .len = @enumFromInt(len), |
| 904 | .alignment = alignment, |
| 905 | }, |
| 906 | }; |
| 907 | |
| 908 | for (control_ptr[0..len], sample_ptr[0..len], 0..) |*control, *sample, i| { |
| 909 | control.* = @truncate(i); |
| 910 | sample.* = @truncate(i); |
| 932 | 911 | } |
| 933 | 912 | |
| 934 | | tls_last_index = index; |
| 913 | @atomicStore(Alloc.Index, &ctx.last_alloc_index, index, .release); |
| 935 | 914 | } |
| 936 | 915 | fn doOneResize(ctx: *FuzzContext, new_len: usize) void { |
| 937 | 916 | @disableInstrumentation(); |
| 938 | | const index = tls_last_index orelse return; |
| 939 | | const len = ctx.allocs.items(.len)[index]; |
| 940 | | assert(len != .free); |
| 941 | | const memory = ctx.allocs.items(.sample_ptr)[index][0..@intFromEnum(len)]; |
| 942 | | const alignment = ctx.allocs.items(.alignment)[index]; |
| 943 | 917 | |
| 944 | | assert(alignment.check(@intFromPtr(ctx.allocs.items(.control_ptr)[index]))); |
| 945 | | assert(alignment.check(@intFromPtr(ctx.allocs.items(.sample_ptr)[index]))); |
| 918 | const index = @atomicRmw(Alloc.Index, &ctx.last_alloc_index, .Xchg, .none, .acquire); |
| 919 | if (index == .none) return; |
| 920 | |
| 921 | const allocation = &ctx.allocs[@intFromEnum(index)]; |
| 922 | assert(allocation.common.len != .free); |
| 923 | const memory = allocation.sample_ptr[0..@intFromEnum(allocation.common.len)]; |
| 924 | const alignment = allocation.common.alignment; |
| 925 | |
| 926 | assert(alignment.check(@intFromPtr(allocation.control_ptr))); |
| 927 | assert(alignment.check(@intFromPtr(allocation.sample_ptr))); |
| 946 | 928 | |
| 947 | 929 | // Since `resize` is fallible, we have to ensure that `control_allocator` |
| 948 | 930 | // is always successful by reserving the memory we need beforehand. |
| 949 | 931 | const new_control_ptr = ctx.control_allocator.rawAlloc(new_len, alignment, @returnAddress()) orelse |
| 950 | 932 | return; |
| 951 | 933 | if (ctx.sample_allocator.rawResize(memory, alignment, new_len, @returnAddress())) { |
| 952 | | const old_control = ctx.allocs.items(.control_ptr)[index][0..memory.len]; |
| 934 | const old_control = allocation.control_ptr[0..memory.len]; |
| 953 | 935 | const overlap = @min(memory.len, new_len); |
| 954 | 936 | @memcpy(new_control_ptr[0..overlap], old_control[0..overlap]); |
| 955 | 937 | ctx.control_allocator.rawFree(old_control, alignment, @returnAddress()); |
| ... | ... | @@ -958,23 +940,27 @@ const FuzzContext = struct { |
| 958 | 940 | return; |
| 959 | 941 | } |
| 960 | 942 | |
| 961 | | ctx.allocs.set(index, .{ |
| 943 | ctx.allocs[@intFromEnum(index)] = .{ |
| 962 | 944 | .control_ptr = new_control_ptr, |
| 963 | 945 | .sample_ptr = memory.ptr, |
| 964 | | .len = @enumFromInt(new_len), |
| 965 | | .alignment = alignment, |
| 966 | | }); |
| 946 | .common = .{ |
| 947 | .len = @enumFromInt(new_len), |
| 948 | .alignment = alignment, |
| 949 | }, |
| 950 | }; |
| 967 | 951 | |
| 968 | 952 | if (new_len > memory.len) { |
| 969 | 953 | for ( |
| 970 | | ctx.allocs.items(.control_ptr)[index][memory.len..new_len], |
| 971 | | ctx.allocs.items(.sample_ptr)[index][memory.len..new_len], |
| 972 | | ) |*control, *sample| { |
| 973 | | control.* = tls_next; |
| 974 | | sample.* = tls_next; |
| 975 | | tls_next +%= 1; |
| 954 | allocation.control_ptr[memory.len..new_len], |
| 955 | allocation.sample_ptr[memory.len..new_len], |
| 956 | 0.., |
| 957 | ) |*control, *sample, i| { |
| 958 | control.* = @truncate(i); |
| 959 | sample.* = @truncate(i); |
| 976 | 960 | } |
| 977 | 961 | } |
| 962 | |
| 963 | @atomicStore(Alloc.Index, &ctx.last_alloc_index, index, .release); |
| 978 | 964 | } |
| 979 | 965 | fn doOneRemap(ctx: *FuzzContext, new_len: usize) void { |
| 980 | 966 | @disableInstrumentation(); |
| ... | ... | @@ -982,26 +968,29 @@ const FuzzContext = struct { |
| 982 | 968 | } |
| 983 | 969 | fn doOneFree(ctx: *FuzzContext) void { |
| 984 | 970 | @disableInstrumentation(); |
| 985 | | const index = tls_last_index orelse return; |
| 986 | | const len = ctx.allocs.items(.len)[index]; |
| 987 | | assert(len != .free); |
| 988 | | const memory = ctx.allocs.items(.sample_ptr)[index][0..@intFromEnum(len)]; |
| 989 | | const alignment = ctx.allocs.items(.alignment)[index]; |
| 990 | 971 | |
| 991 | | assert(alignment.check(@intFromPtr(ctx.allocs.items(.control_ptr)[index]))); |
| 992 | | assert(alignment.check(@intFromPtr(ctx.allocs.items(.sample_ptr)[index]))); |
| 972 | const index = @atomicRmw(Alloc.Index, &ctx.last_alloc_index, .Xchg, .none, .acquire); |
| 973 | if (index == .none) return; |
| 993 | 974 | |
| 994 | | ctx.control_allocator.rawFree(ctx.allocs.items(.control_ptr)[index][0..memory.len], alignment, @returnAddress()); |
| 995 | | ctx.sample_allocator.rawFree(ctx.allocs.items(.sample_ptr)[index][0..memory.len], alignment, @returnAddress()); |
| 975 | const allocation = &ctx.allocs[@intFromEnum(index)]; |
| 976 | assert(allocation.common.len != .free); |
| 977 | const len: usize = @intFromEnum(allocation.common.len); |
| 978 | const alignment = allocation.common.alignment; |
| 996 | 979 | |
| 997 | | ctx.allocs.set(index, .{ |
| 980 | assert(alignment.check(@intFromPtr(allocation.control_ptr))); |
| 981 | assert(alignment.check(@intFromPtr(allocation.sample_ptr))); |
| 982 | |
| 983 | ctx.control_allocator.rawFree(allocation.control_ptr[0..len], alignment, @returnAddress()); |
| 984 | ctx.sample_allocator.rawFree(allocation.sample_ptr[0..len], alignment, @returnAddress()); |
| 985 | |
| 986 | ctx.allocs[@intFromEnum(index)] = .{ |
| 998 | 987 | .control_ptr = undefined, |
| 999 | 988 | .sample_ptr = undefined, |
| 1000 | | .len = .free, |
| 1001 | | .alignment = undefined, |
| 1002 | | }); |
| 1003 | | |
| 1004 | | tls_last_index = null; |
| 989 | .common = .{ |
| 990 | .len = .free, |
| 991 | .alignment = .@"1", |
| 992 | }, |
| 993 | }; |
| 1005 | 994 | } |
| 1006 | 995 | }; |
| 1007 | 996 | |