authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-09-20 18:54:23+02:00
committergravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-11-05 16:10:15+01:00
log806097c165ce26d0a1817a0f2189b20ad5ef796d
treeda000cf35fcd10f6ab17de15dea188d509e60d82
parent17837affd22a6055c65a14252fa38610fdeabc3a

std: Make C allocator respect the required alignment

Use posix_memalign where available and the _aligned_{malloc,free} API on Windows. Closes #3783

3 files changed, 147 insertions(+), 84 deletions(-)

lib/std/c.zig+1-2
......@@ -245,7 +245,6 @@ pub extern "c" fn setregid(rgid: gid_t, egid: gid_t) c_int;
245245pub extern "c" fn setresuid(ruid: uid_t, euid: uid_t, suid: uid_t) c_int;
246246pub extern "c" fn setresgid(rgid: gid_t, egid: gid_t, sgid: gid_t) c_int;
247247
248pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?*c_void;
249248pub extern "c" fn malloc(usize) ?*c_void;
250249
251250pub usingnamespace switch (builtin.os.tag) {
......@@ -260,7 +259,7 @@ pub usingnamespace switch (builtin.os.tag) {
260259
261260pub extern "c" fn realloc(?*c_void, usize) ?*c_void;
262261pub extern "c" fn free(*c_void) void;
263pub extern "c" fn posix_memalign(memptr: **c_void, alignment: usize, size: usize) c_int;
262pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int;
264263
265264pub extern "c" fn futimes(fd: fd_t, times: *[2]timeval) c_int;
266265pub extern "c" fn utimes(path: [*:0]const u8, times: *[2]timeval) c_int;
lib/std/c/windows.zig+4
......@@ -4,3 +4,7 @@
44// The MIT license requires this copyright notice to be included in all copies
55// and substantial portions of the software.
66pub extern "c" fn _errno() *c_int;
7
8pub extern "c" fn _aligned_free(memblock: ?*c_void) void;
9pub extern "c" fn _aligned_malloc(size: usize, alignment: usize) ?*c_void;
10pub extern "c" fn _aligned_realloc(memblock: ?*c_void, size: usize, alignment: usize) ?*c_void;
lib/std/heap.zig+142-82
......@@ -21,67 +21,119 @@ pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig"
2121
2222const Allocator = mem.Allocator;
2323
24usingnamespace if (comptime @hasDecl(c, "malloc_size"))
25 struct {
26 pub const supports_malloc_size = true;
27 pub const malloc_size = c.malloc_size;
28 }
29else if (comptime @hasDecl(c, "malloc_usable_size"))
30 struct {
31 pub const supports_malloc_size = true;
32 pub const malloc_size = c.malloc_usable_size;
24const CAllocator = struct {
25 comptime {
26 if (!builtin.link_libc) {
27 @compileError("C allocator is only available when linking against libc");
28 }
3329 }
34else
35 struct {
36 pub const supports_malloc_size = false;
37 };
3830
39pub const c_allocator = &c_allocator_state;
40var c_allocator_state = Allocator{
41 .allocFn = cAlloc,
42 .resizeFn = cResize,
43};
31 usingnamespace if (comptime @hasDecl(c, "malloc_size"))
32 struct {
33 pub const supports_malloc_size = true;
34 pub const malloc_size = c.malloc_size;
35 }
36 else if (comptime @hasDecl(c, "malloc_usable_size"))
37 struct {
38 pub const supports_malloc_size = true;
39 pub const malloc_size = c.malloc_usable_size;
40 }
41 else
42 struct {
43 pub const supports_malloc_size = false;
44 };
4445
45fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Allocator.Error![]u8 {
46 assert(ptr_align <= @alignOf(c_longdouble));
47 const ptr = @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory);
48 if (len_align == 0) {
49 return ptr[0..len];
50 }
51 const full_len = init: {
52 if (supports_malloc_size) {
53 const s = malloc_size(ptr);
54 assert(s >= len);
55 break :init s;
46 // The alignment guaranteed by malloc, the value matches the result of the C
47 // expression `alignof(max_alignment_t)`
48 const min_ptr_alignment = comptime std.math.max(
49 @alignOf(c_longdouble),
50 @alignOf(c_longlong),
51 );
52
53 fn aligned_alloc(len: usize, alignment: usize) ?*c_void {
54 // The minimum alignment supported by both APIs is the size of a pointer
55 const eff_alignment = std.math.max(alignment, @sizeOf(usize));
56
57 if (builtin.os.tag == .windows) {
58 return c._aligned_malloc(len, eff_alignment);
5659 }
57 break :init len;
58 };
59 return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
60}
6160
62fn cResize(
63 self: *Allocator,
64 buf: []u8,
65 old_align: u29,
66 new_len: usize,
67 len_align: u29,
68 ret_addr: usize,
69) Allocator.Error!usize {
70 if (new_len == 0) {
71 c.free(buf.ptr);
72 return 0;
61 var aligned_ptr: ?*c_void = undefined;
62 if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0)
63 return null;
64 return aligned_ptr;
65 }
66
67 fn aligned_free(ptr: *c_void) void {
68 if (builtin.os.tag == .windows) {
69 return c._aligned_free(ptr);
70 }
71 c.free(ptr);
7372 }
74 if (new_len <= buf.len) {
75 return mem.alignAllocLen(buf.len, new_len, len_align);
73
74 fn alloc(
75 allocator: *Allocator,
76 len: usize,
77 alignment: u29,
78 len_align: u29,
79 return_address: usize,
80 ) error{OutOfMemory}![]u8 {
81 assert(len > 0);
82 assert(std.math.isPowerOfTwo(alignment));
83
84 var ptr = if (alignment <= min_ptr_alignment)
85 @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory)
86 else
87 @ptrCast([*]u8, aligned_alloc(len, alignment) orelse return error.OutOfMemory);
88
89 if (len_align == 0)
90 return ptr[0..len];
91
92 const full_len = init: {
93 if (supports_malloc_size) {
94 const s = malloc_size(ptr);
95 assert(s >= len);
96 break :init s;
97 }
98 break :init len;
99 };
100
101 return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
76102 }
77 if (supports_malloc_size) {
78 const full_len = malloc_size(buf.ptr);
79 if (new_len <= full_len) {
80 return mem.alignAllocLen(full_len, new_len, len_align);
103
104 fn resize(
105 allocator: *Allocator,
106 buf: []u8,
107 buf_align: u29,
108 new_len: usize,
109 len_align: u29,
110 return_address: usize,
111 ) Allocator.Error!usize {
112 if (new_len == 0) {
113 if (buf_align <= min_ptr_alignment)
114 c.free(buf.ptr)
115 else
116 aligned_free(buf.ptr);
117 return 0;
118 }
119 if (new_len <= buf.len) {
120 return mem.alignAllocLen(buf.len, new_len, len_align);
121 }
122 if (supports_malloc_size) {
123 const full_len = malloc_size(buf.ptr);
124 if (new_len <= full_len) {
125 return mem.alignAllocLen(full_len, new_len, len_align);
126 }
81127 }
128 return error.OutOfMemory;
82129 }
83 return error.OutOfMemory;
84}
130};
131
132pub const c_allocator = &c_allocator_state;
133var c_allocator_state = Allocator{
134 .allocFn = CAllocator.alloc,
135 .resizeFn = CAllocator.resize,
136};
85137
86138/// This allocator makes a syscall directly for every allocation and free.
87139/// Thread-safe and lock-free.
......@@ -726,9 +778,10 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
726778
727779test "c_allocator" {
728780 if (builtin.link_libc) {
729 var slice = try c_allocator.alloc(u8, 50);
730 defer c_allocator.free(slice);
731 slice = try c_allocator.realloc(slice, 100);
781 try testAllocator(c_allocator);
782 try testAllocatorAligned(c_allocator);
783 try testAllocatorLargeAlignment(c_allocator);
784 try testAllocatorAlignedShrink(c_allocator);
732785 }
733786}
734787
......@@ -772,7 +825,7 @@ test "WasmPageAllocator internals" {
772825test "PageAllocator" {
773826 const allocator = page_allocator;
774827 try testAllocator(allocator);
775 try testAllocatorAligned(allocator, 16);
828 try testAllocatorAligned(allocator);
776829 if (!std.Target.current.isWasm()) {
777830 try testAllocatorLargeAlignment(allocator);
778831 try testAllocatorAlignedShrink(allocator);
......@@ -802,7 +855,7 @@ test "HeapAllocator" {
802855
803856 const allocator = &heap_allocator.allocator;
804857 try testAllocator(allocator);
805 try testAllocatorAligned(allocator, 16);
858 try testAllocatorAligned(allocator);
806859 try testAllocatorLargeAlignment(allocator);
807860 try testAllocatorAlignedShrink(allocator);
808861 }
......@@ -813,7 +866,7 @@ test "ArenaAllocator" {
813866 defer arena_allocator.deinit();
814867
815868 try testAllocator(&arena_allocator.allocator);
816 try testAllocatorAligned(&arena_allocator.allocator, 16);
869 try testAllocatorAligned(&arena_allocator.allocator);
817870 try testAllocatorLargeAlignment(&arena_allocator.allocator);
818871 try testAllocatorAlignedShrink(&arena_allocator.allocator);
819872}
......@@ -823,7 +876,7 @@ test "FixedBufferAllocator" {
823876 var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
824877
825878 try testAllocator(&fixed_buffer_allocator.allocator);
826 try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);
879 try testAllocatorAligned(&fixed_buffer_allocator.allocator);
827880 try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);
828881 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
829882}
......@@ -881,7 +934,7 @@ test "ThreadSafeFixedBufferAllocator" {
881934 var fixed_buffer_allocator = ThreadSafeFixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
882935
883936 try testAllocator(&fixed_buffer_allocator.allocator);
884 try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);
937 try testAllocatorAligned(&fixed_buffer_allocator.allocator);
885938 try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);
886939 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
887940}
......@@ -916,6 +969,10 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void {
916969
917970 allocator.free(slice);
918971
972 // Zero-length allocation
973 var empty = try allocator.alloc(u8, 0);
974 allocator.free(empty);
975 // Allocation with zero-sized types
919976 const zero_bit_ptr = try allocator.create(u0);
920977 zero_bit_ptr.* = 0;
921978 allocator.destroy(zero_bit_ptr);
......@@ -928,31 +985,34 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void {
928985 allocator.free(oversize);
929986}
930987
931pub fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void {
988pub fn testAllocatorAligned(base_allocator: *mem.Allocator) !void {
932989 var validationAllocator = mem.validationWrap(base_allocator);
933990 const allocator = &validationAllocator.allocator;
934991
935 // initial
936 var slice = try allocator.alignedAlloc(u8, alignment, 10);
937 testing.expect(slice.len == 10);
938 // grow
939 slice = try allocator.realloc(slice, 100);
940 testing.expect(slice.len == 100);
941 // shrink
942 slice = allocator.shrink(slice, 10);
943 testing.expect(slice.len == 10);
944 // go to zero
945 slice = allocator.shrink(slice, 0);
946 testing.expect(slice.len == 0);
947 // realloc from zero
948 slice = try allocator.realloc(slice, 100);
949 testing.expect(slice.len == 100);
950 // shrink with shrink
951 slice = allocator.shrink(slice, 10);
952 testing.expect(slice.len == 10);
953 // shrink to zero
954 slice = allocator.shrink(slice, 0);
955 testing.expect(slice.len == 0);
992 // Test a few alignment values, smaller and bigger than the type's one
993 inline for ([_]u29{ 1, 2, 4, 8, 16, 32, 64 }) |alignment| {
994 // initial
995 var slice = try allocator.alignedAlloc(u8, alignment, 10);
996 testing.expect(slice.len == 10);
997 // grow
998 slice = try allocator.realloc(slice, 100);
999 testing.expect(slice.len == 100);
1000 // shrink
1001 slice = allocator.shrink(slice, 10);
1002 testing.expect(slice.len == 10);
1003 // go to zero
1004 slice = allocator.shrink(slice, 0);
1005 testing.expect(slice.len == 0);
1006 // realloc from zero
1007 slice = try allocator.realloc(slice, 100);
1008 testing.expect(slice.len == 100);
1009 // shrink with shrink
1010 slice = allocator.shrink(slice, 10);
1011 testing.expect(slice.len == 10);
1012 // shrink to zero
1013 slice = allocator.shrink(slice, 0);
1014 testing.expect(slice.len == 0);
1015 }
9561016}
9571017
9581018pub fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void {