authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-09-20 18:54:23+02:00
committergravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-11-05 16:10:15+01:00
log806097c165ce26d0a1817a0f2189b20ad5ef796d
treeda000cf35fcd10f6ab17de15dea188d509e60d82
parent17837affd22a6055c65a14252fa38610fdeabc3a

std: Make C allocator respect the required alignment

Use posix_memalign where available and the _aligned_{malloc,free} API on Windows. Closes #3783

3 files changed, 147 insertions(+), 84 deletions(-)

lib/std/c.zig+1-2
...@@ -245,7 +245,6 @@ pub extern "c" fn setregid(rgid: gid_t, egid: gid_t) c_int;...@@ -245,7 +245,6 @@ pub extern "c" fn setregid(rgid: gid_t, egid: gid_t) c_int;
245pub extern "c" fn setresuid(ruid: uid_t, euid: uid_t, suid: uid_t) c_int;245pub extern "c" fn setresuid(ruid: uid_t, euid: uid_t, suid: uid_t) c_int;
246pub extern "c" fn setresgid(rgid: gid_t, egid: gid_t, sgid: gid_t) c_int;246pub extern "c" fn setresgid(rgid: gid_t, egid: gid_t, sgid: gid_t) c_int;
247247
248pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?*c_void;
249pub extern "c" fn malloc(usize) ?*c_void;248pub extern "c" fn malloc(usize) ?*c_void;
250249
251pub usingnamespace switch (builtin.os.tag) {250pub usingnamespace switch (builtin.os.tag) {
...@@ -260,7 +259,7 @@ pub usingnamespace switch (builtin.os.tag) {...@@ -260,7 +259,7 @@ pub usingnamespace switch (builtin.os.tag) {
260259
261pub extern "c" fn realloc(?*c_void, usize) ?*c_void;260pub extern "c" fn realloc(?*c_void, usize) ?*c_void;
262pub extern "c" fn free(*c_void) void;261pub extern "c" fn free(*c_void) void;
263pub extern "c" fn posix_memalign(memptr: **c_void, alignment: usize, size: usize) c_int;262pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int;
264263
265pub extern "c" fn futimes(fd: fd_t, times: *[2]timeval) c_int;264pub extern "c" fn futimes(fd: fd_t, times: *[2]timeval) c_int;
266pub extern "c" fn utimes(path: [*:0]const u8, times: *[2]timeval) c_int;265pub extern "c" fn utimes(path: [*:0]const u8, times: *[2]timeval) c_int;
lib/std/c/windows.zig+4
...@@ -4,3 +4,7 @@...@@ -4,3 +4,7 @@
4// The MIT license requires this copyright notice to be included in all copies4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.5// and substantial portions of the software.
6pub extern "c" fn _errno() *c_int;6pub extern "c" fn _errno() *c_int;
7
8pub extern "c" fn _aligned_free(memblock: ?*c_void) void;
9pub extern "c" fn _aligned_malloc(size: usize, alignment: usize) ?*c_void;
10pub extern "c" fn _aligned_realloc(memblock: ?*c_void, size: usize, alignment: usize) ?*c_void;
lib/std/heap.zig+142-82
...@@ -21,67 +21,119 @@ pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig"...@@ -21,67 +21,119 @@ pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig"
2121
22const Allocator = mem.Allocator;22const Allocator = mem.Allocator;
2323
24usingnamespace if (comptime @hasDecl(c, "malloc_size"))24const CAllocator = struct {
25 struct {25 comptime {
26 pub const supports_malloc_size = true;26 if (!builtin.link_libc) {
27 pub const malloc_size = c.malloc_size;27 @compileError("C allocator is only available when linking against libc");
28 }28 }
29else if (comptime @hasDecl(c, "malloc_usable_size"))
30 struct {
31 pub const supports_malloc_size = true;
32 pub const malloc_size = c.malloc_usable_size;
33 }29 }
34else
35 struct {
36 pub const supports_malloc_size = false;
37 };
3830
39pub const c_allocator = &c_allocator_state;31 usingnamespace if (comptime @hasDecl(c, "malloc_size"))
40var c_allocator_state = Allocator{32 struct {
41 .allocFn = cAlloc,33 pub const supports_malloc_size = true;
42 .resizeFn = cResize,34 pub const malloc_size = c.malloc_size;
43};35 }
36 else if (comptime @hasDecl(c, "malloc_usable_size"))
37 struct {
38 pub const supports_malloc_size = true;
39 pub const malloc_size = c.malloc_usable_size;
40 }
41 else
42 struct {
43 pub const supports_malloc_size = false;
44 };
4445
45fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Allocator.Error![]u8 {46 // The alignment guaranteed by malloc, the value matches the result of the C
46 assert(ptr_align <= @alignOf(c_longdouble));47 // expression `alignof(max_alignment_t)`
47 const ptr = @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory);48 const min_ptr_alignment = comptime std.math.max(
48 if (len_align == 0) {49 @alignOf(c_longdouble),
49 return ptr[0..len];50 @alignOf(c_longlong),
50 }51 );
51 const full_len = init: {52
52 if (supports_malloc_size) {53 fn aligned_alloc(len: usize, alignment: usize) ?*c_void {
53 const s = malloc_size(ptr);54 // The minimum alignment supported by both APIs is the size of a pointer
54 assert(s >= len);55 const eff_alignment = std.math.max(alignment, @sizeOf(usize));
55 break :init s;56
57 if (builtin.os.tag == .windows) {
58 return c._aligned_malloc(len, eff_alignment);
56 }59 }
57 break :init len;
58 };
59 return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
60}
6160
62fn cResize(61 var aligned_ptr: ?*c_void = undefined;
63 self: *Allocator,62 if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0)
64 buf: []u8,63 return null;
65 old_align: u29,64 return aligned_ptr;
66 new_len: usize,65 }
67 len_align: u29,66
68 ret_addr: usize,67 fn aligned_free(ptr: *c_void) void {
69) Allocator.Error!usize {68 if (builtin.os.tag == .windows) {
70 if (new_len == 0) {69 return c._aligned_free(ptr);
71 c.free(buf.ptr);70 }
72 return 0;71 c.free(ptr);
73 }72 }
74 if (new_len <= buf.len) {73
75 return mem.alignAllocLen(buf.len, new_len, len_align);74 fn alloc(
75 allocator: *Allocator,
76 len: usize,
77 alignment: u29,
78 len_align: u29,
79 return_address: usize,
80 ) error{OutOfMemory}![]u8 {
81 assert(len > 0);
82 assert(std.math.isPowerOfTwo(alignment));
83
84 var ptr = if (alignment <= min_ptr_alignment)
85 @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory)
86 else
87 @ptrCast([*]u8, aligned_alloc(len, alignment) orelse return error.OutOfMemory);
88
89 if (len_align == 0)
90 return ptr[0..len];
91
92 const full_len = init: {
93 if (supports_malloc_size) {
94 const s = malloc_size(ptr);
95 assert(s >= len);
96 break :init s;
97 }
98 break :init len;
99 };
100
101 return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
76 }102 }
77 if (supports_malloc_size) {103
78 const full_len = malloc_size(buf.ptr);104 fn resize(
79 if (new_len <= full_len) {105 allocator: *Allocator,
80 return mem.alignAllocLen(full_len, new_len, len_align);106 buf: []u8,
107 buf_align: u29,
108 new_len: usize,
109 len_align: u29,
110 return_address: usize,
111 ) Allocator.Error!usize {
112 if (new_len == 0) {
113 if (buf_align <= min_ptr_alignment)
114 c.free(buf.ptr)
115 else
116 aligned_free(buf.ptr);
117 return 0;
118 }
119 if (new_len <= buf.len) {
120 return mem.alignAllocLen(buf.len, new_len, len_align);
121 }
122 if (supports_malloc_size) {
123 const full_len = malloc_size(buf.ptr);
124 if (new_len <= full_len) {
125 return mem.alignAllocLen(full_len, new_len, len_align);
126 }
81 }127 }
128 return error.OutOfMemory;
82 }129 }
83 return error.OutOfMemory;130};
84}131
132pub const c_allocator = &c_allocator_state;
133var c_allocator_state = Allocator{
134 .allocFn = CAllocator.alloc,
135 .resizeFn = CAllocator.resize,
136};
85137
86/// This allocator makes a syscall directly for every allocation and free.138/// This allocator makes a syscall directly for every allocation and free.
87/// Thread-safe and lock-free.139/// Thread-safe and lock-free.
...@@ -726,9 +778,10 @@ pub fn StackFallbackAllocator(comptime size: usize) type {...@@ -726,9 +778,10 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
726778
727test "c_allocator" {779test "c_allocator" {
728 if (builtin.link_libc) {780 if (builtin.link_libc) {
729 var slice = try c_allocator.alloc(u8, 50);781 try testAllocator(c_allocator);
730 defer c_allocator.free(slice);782 try testAllocatorAligned(c_allocator);
731 slice = try c_allocator.realloc(slice, 100);783 try testAllocatorLargeAlignment(c_allocator);
784 try testAllocatorAlignedShrink(c_allocator);
732 }785 }
733}786}
734787
...@@ -772,7 +825,7 @@ test "WasmPageAllocator internals" {...@@ -772,7 +825,7 @@ test "WasmPageAllocator internals" {
772test "PageAllocator" {825test "PageAllocator" {
773 const allocator = page_allocator;826 const allocator = page_allocator;
774 try testAllocator(allocator);827 try testAllocator(allocator);
775 try testAllocatorAligned(allocator, 16);828 try testAllocatorAligned(allocator);
776 if (!std.Target.current.isWasm()) {829 if (!std.Target.current.isWasm()) {
777 try testAllocatorLargeAlignment(allocator);830 try testAllocatorLargeAlignment(allocator);
778 try testAllocatorAlignedShrink(allocator);831 try testAllocatorAlignedShrink(allocator);
...@@ -802,7 +855,7 @@ test "HeapAllocator" {...@@ -802,7 +855,7 @@ test "HeapAllocator" {
802855
803 const allocator = &heap_allocator.allocator;856 const allocator = &heap_allocator.allocator;
804 try testAllocator(allocator);857 try testAllocator(allocator);
805 try testAllocatorAligned(allocator, 16);858 try testAllocatorAligned(allocator);
806 try testAllocatorLargeAlignment(allocator);859 try testAllocatorLargeAlignment(allocator);
807 try testAllocatorAlignedShrink(allocator);860 try testAllocatorAlignedShrink(allocator);
808 }861 }
...@@ -813,7 +866,7 @@ test "ArenaAllocator" {...@@ -813,7 +866,7 @@ test "ArenaAllocator" {
813 defer arena_allocator.deinit();866 defer arena_allocator.deinit();
814867
815 try testAllocator(&arena_allocator.allocator);868 try testAllocator(&arena_allocator.allocator);
816 try testAllocatorAligned(&arena_allocator.allocator, 16);869 try testAllocatorAligned(&arena_allocator.allocator);
817 try testAllocatorLargeAlignment(&arena_allocator.allocator);870 try testAllocatorLargeAlignment(&arena_allocator.allocator);
818 try testAllocatorAlignedShrink(&arena_allocator.allocator);871 try testAllocatorAlignedShrink(&arena_allocator.allocator);
819}872}
...@@ -823,7 +876,7 @@ test "FixedBufferAllocator" {...@@ -823,7 +876,7 @@ test "FixedBufferAllocator" {
823 var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));876 var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
824877
825 try testAllocator(&fixed_buffer_allocator.allocator);878 try testAllocator(&fixed_buffer_allocator.allocator);
826 try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);879 try testAllocatorAligned(&fixed_buffer_allocator.allocator);
827 try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);880 try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);
828 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);881 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
829}882}
...@@ -881,7 +934,7 @@ test "ThreadSafeFixedBufferAllocator" {...@@ -881,7 +934,7 @@ test "ThreadSafeFixedBufferAllocator" {
881 var fixed_buffer_allocator = ThreadSafeFixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);934 var fixed_buffer_allocator = ThreadSafeFixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
882935
883 try testAllocator(&fixed_buffer_allocator.allocator);936 try testAllocator(&fixed_buffer_allocator.allocator);
884 try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);937 try testAllocatorAligned(&fixed_buffer_allocator.allocator);
885 try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);938 try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);
886 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);939 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
887}940}
...@@ -916,6 +969,10 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void {...@@ -916,6 +969,10 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void {
916969
917 allocator.free(slice);970 allocator.free(slice);
918971
972 // Zero-length allocation
973 var empty = try allocator.alloc(u8, 0);
974 allocator.free(empty);
975 // Allocation with zero-sized types
919 const zero_bit_ptr = try allocator.create(u0);976 const zero_bit_ptr = try allocator.create(u0);
920 zero_bit_ptr.* = 0;977 zero_bit_ptr.* = 0;
921 allocator.destroy(zero_bit_ptr);978 allocator.destroy(zero_bit_ptr);
...@@ -928,31 +985,34 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void {...@@ -928,31 +985,34 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void {
928 allocator.free(oversize);985 allocator.free(oversize);
929}986}
930987
931pub fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void {988pub fn testAllocatorAligned(base_allocator: *mem.Allocator) !void {
932 var validationAllocator = mem.validationWrap(base_allocator);989 var validationAllocator = mem.validationWrap(base_allocator);
933 const allocator = &validationAllocator.allocator;990 const allocator = &validationAllocator.allocator;
934991
935 // initial992 // Test a few alignment values, smaller and bigger than the type's one
936 var slice = try allocator.alignedAlloc(u8, alignment, 10);993 inline for ([_]u29{ 1, 2, 4, 8, 16, 32, 64 }) |alignment| {
937 testing.expect(slice.len == 10);994 // initial
938 // grow995 var slice = try allocator.alignedAlloc(u8, alignment, 10);
939 slice = try allocator.realloc(slice, 100);996 testing.expect(slice.len == 10);
940 testing.expect(slice.len == 100);997 // grow
941 // shrink998 slice = try allocator.realloc(slice, 100);
942 slice = allocator.shrink(slice, 10);999 testing.expect(slice.len == 100);
943 testing.expect(slice.len == 10);1000 // shrink
944 // go to zero1001 slice = allocator.shrink(slice, 10);
945 slice = allocator.shrink(slice, 0);1002 testing.expect(slice.len == 10);
946 testing.expect(slice.len == 0);1003 // go to zero
947 // realloc from zero1004 slice = allocator.shrink(slice, 0);
948 slice = try allocator.realloc(slice, 100);1005 testing.expect(slice.len == 0);
949 testing.expect(slice.len == 100);1006 // realloc from zero
950 // shrink with shrink1007 slice = try allocator.realloc(slice, 100);
951 slice = allocator.shrink(slice, 10);1008 testing.expect(slice.len == 100);
952 testing.expect(slice.len == 10);1009 // shrink with shrink
953 // shrink to zero1010 slice = allocator.shrink(slice, 10);
954 slice = allocator.shrink(slice, 0);1011 testing.expect(slice.len == 10);
955 testing.expect(slice.len == 0);1012 // shrink to zero
1013 slice = allocator.shrink(slice, 0);
1014 testing.expect(slice.len == 0);
1015 }
956}1016}
9571017
958pub fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void {1018pub fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void {